ominfra 0.0.0.dev147__py3-none-any.whl → 0.0.0.dev149__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. ominfra/clouds/aws/cli.py +1 -1
  2. ominfra/configs.py +30 -5
  3. ominfra/journald/messages.py +1 -1
  4. ominfra/manage/commands/base.py +5 -5
  5. ominfra/manage/commands/execution.py +2 -2
  6. ominfra/manage/commands/inject.py +1 -1
  7. ominfra/manage/commands/interp.py +2 -2
  8. ominfra/manage/commands/subprocess.py +22 -14
  9. ominfra/manage/deploy/command.py +1 -1
  10. ominfra/manage/deploy/paths.py +2 -2
  11. ominfra/manage/main.py +48 -32
  12. ominfra/manage/remote/_main.py +172 -0
  13. ominfra/manage/remote/channel.py +41 -16
  14. ominfra/manage/remote/config.py +10 -0
  15. ominfra/manage/remote/connection.py +106 -0
  16. ominfra/manage/remote/execution.py +244 -155
  17. ominfra/manage/remote/inject.py +7 -3
  18. ominfra/manage/remote/spawning.py +51 -33
  19. ominfra/pyremote.py +28 -3
  20. ominfra/scripts/journald2aws.py +195 -91
  21. ominfra/scripts/manage.py +1366 -486
  22. ominfra/scripts/supervisor.py +533 -479
  23. ominfra/supervisor/dispatchers.py +1 -1
  24. ominfra/supervisor/http.py +2 -2
  25. ominfra/supervisor/inject.py +4 -4
  26. ominfra/supervisor/io.py +1 -1
  27. ominfra/supervisor/spawningimpl.py +1 -1
  28. ominfra/supervisor/supervisor.py +1 -1
  29. ominfra/supervisor/types.py +1 -1
  30. ominfra/tailscale/cli.py +1 -1
  31. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/METADATA +3 -3
  32. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/RECORD +36 -34
  33. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/LICENSE +0 -0
  34. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/WHEEL +0 -0
  35. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/entry_points.txt +0 -0
  36. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py CHANGED
@@ -3,16 +3,20 @@
3
3
  # @omlish-lite
4
4
  # @omlish-script
5
5
  # @omlish-amalg-output ../manage/main.py
6
- # ruff: noqa: N802 UP006 UP007 UP036
6
+ # ruff: noqa: N802 TC003 UP006 UP007 UP036
7
7
  """
8
8
  manage.py -s 'docker run -i python:3.12'
9
9
  manage.py -s 'ssh -i /foo/bar.pem foo@bar.baz' -q --python=python3.8
10
10
  """
11
11
  import abc
12
+ import asyncio
13
+ import asyncio.base_subprocess
14
+ import asyncio.subprocess
12
15
  import base64
13
16
  import collections
14
17
  import collections.abc
15
18
  import contextlib
19
+ import ctypes as ct
16
20
  import dataclasses as dc
17
21
  import datetime
18
22
  import decimal
@@ -30,6 +34,7 @@ import pwd
30
34
  import re
31
35
  import shlex
32
36
  import shutil
37
+ import signal
33
38
  import site
34
39
  import struct
35
40
  import subprocess
@@ -62,6 +67,9 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
62
67
  VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
63
68
  VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
64
69
 
70
+ # ../../omlish/lite/asyncio/asyncio.py
71
+ AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
72
+
65
73
  # ../../omlish/lite/cached.py
66
74
  T = ta.TypeVar('T')
67
75
  CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
@@ -822,10 +830,8 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
822
830
  with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
823
831
  main_src = sf.read()
824
832
 
825
- # Restore original argv0
833
+ # Restore vars
826
834
  sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
827
-
828
- # Grab context name
829
835
  context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
830
836
 
831
837
  # Write third ack
@@ -1000,12 +1006,102 @@ class PyremoteBootstrapDriver:
1000
1006
  else:
1001
1007
  raise TypeError(go)
1002
1008
 
1009
+ async def async_run(
1010
+ self,
1011
+ input: ta.Any, # asyncio.StreamWriter # noqa
1012
+ output: ta.Any, # asyncio.StreamReader
1013
+ ) -> Result:
1014
+ gen = self.gen()
1015
+
1016
+ gi: ta.Optional[bytes] = None
1017
+ while True:
1018
+ try:
1019
+ if gi is not None:
1020
+ go = gen.send(gi)
1021
+ else:
1022
+ go = next(gen)
1023
+ except StopIteration as e:
1024
+ return e.value
1025
+
1026
+ if isinstance(go, self.Read):
1027
+ if len(gi := await input.read(go.sz)) != go.sz:
1028
+ raise EOFError
1029
+ elif isinstance(go, self.Write):
1030
+ gi = None
1031
+ output.write(go.d)
1032
+ await output.drain()
1033
+ else:
1034
+ raise TypeError(go)
1035
+
1036
+
1037
+ ########################################
1038
+ # ../../../omlish/lite/asyncio/asyncio.py
1039
+
1040
+
1041
+ ##
1042
+
1043
+
1044
+ ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
1045
+
1046
+
1047
+ async def asyncio_open_stream_reader(
1048
+ f: ta.IO,
1049
+ loop: ta.Any = None,
1050
+ *,
1051
+ limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1052
+ ) -> asyncio.StreamReader:
1053
+ if loop is None:
1054
+ loop = asyncio.get_running_loop()
1055
+
1056
+ reader = asyncio.StreamReader(limit=limit, loop=loop)
1057
+ await loop.connect_read_pipe(
1058
+ lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1059
+ f,
1060
+ )
1061
+
1062
+ return reader
1063
+
1064
+
1065
+ async def asyncio_open_stream_writer(
1066
+ f: ta.IO,
1067
+ loop: ta.Any = None,
1068
+ ) -> asyncio.StreamWriter:
1069
+ if loop is None:
1070
+ loop = asyncio.get_running_loop()
1071
+
1072
+ writer_transport, writer_protocol = await loop.connect_write_pipe(
1073
+ lambda: asyncio.streams.FlowControlMixin(loop=loop),
1074
+ f,
1075
+ )
1076
+
1077
+ return asyncio.streams.StreamWriter(
1078
+ writer_transport,
1079
+ writer_protocol,
1080
+ None,
1081
+ loop,
1082
+ )
1083
+
1084
+
1085
+ ##
1086
+
1087
+
1088
+ def asyncio_maybe_timeout(
1089
+ fut: AwaitableT,
1090
+ timeout: ta.Optional[float] = None,
1091
+ ) -> AwaitableT:
1092
+ if timeout is not None:
1093
+ fut = asyncio.wait_for(fut, timeout) # type: ignore
1094
+ return fut
1095
+
1003
1096
 
1004
1097
  ########################################
1005
1098
  # ../../../omlish/lite/cached.py
1006
1099
 
1007
1100
 
1008
- class _cached_nullary: # noqa
1101
+ ##
1102
+
1103
+
1104
+ class _AbstractCachedNullary:
1009
1105
  def __init__(self, fn):
1010
1106
  super().__init__()
1011
1107
  self._fn = fn
@@ -1013,17 +1109,25 @@ class _cached_nullary: # noqa
1013
1109
  functools.update_wrapper(self, fn)
1014
1110
 
1015
1111
  def __call__(self, *args, **kwargs): # noqa
1016
- if self._value is self._missing:
1017
- self._value = self._fn()
1018
- return self._value
1112
+ raise TypeError
1019
1113
 
1020
1114
  def __get__(self, instance, owner): # noqa
1021
1115
  bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
1022
1116
  return bound
1023
1117
 
1024
1118
 
1119
+ ##
1120
+
1121
+
1122
+ class _CachedNullary(_AbstractCachedNullary):
1123
+ def __call__(self, *args, **kwargs): # noqa
1124
+ if self._value is self._missing:
1125
+ self._value = self._fn()
1126
+ return self._value
1127
+
1128
+
1025
1129
  def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
1026
- return _cached_nullary(fn)
1130
+ return _CachedNullary(fn)
1027
1131
 
1028
1132
 
1029
1133
  def static_init(fn: CallableT) -> CallableT:
@@ -1032,6 +1136,20 @@ def static_init(fn: CallableT) -> CallableT:
1032
1136
  return fn
1033
1137
 
1034
1138
 
1139
+ ##
1140
+
1141
+
1142
+ class _AsyncCachedNullary(_AbstractCachedNullary):
1143
+ async def __call__(self, *args, **kwargs):
1144
+ if self._value is self._missing:
1145
+ self._value = await self._fn()
1146
+ return self._value
1147
+
1148
+
1149
+ def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
1150
+ return _AsyncCachedNullary(fn)
1151
+
1152
+
1035
1153
  ########################################
1036
1154
  # ../../../omlish/lite/check.py
1037
1155
 
@@ -1071,6 +1189,11 @@ def check_non_empty_str(v: ta.Optional[str]) -> str:
1071
1189
  return v
1072
1190
 
1073
1191
 
1192
+ def check_arg(v: bool, msg: str = 'Illegal argument') -> None:
1193
+ if not v:
1194
+ raise ValueError(msg)
1195
+
1196
+
1074
1197
  def check_state(v: bool, msg: str = 'Illegal state') -> None:
1075
1198
  if not v:
1076
1199
  raise ValueError(msg)
@@ -1123,12 +1246,36 @@ def check_empty(v: SizedT) -> SizedT:
1123
1246
  return v
1124
1247
 
1125
1248
 
1126
- def check_non_empty(v: SizedT) -> SizedT:
1249
+ def check_not_empty(v: SizedT) -> SizedT:
1127
1250
  if not len(v):
1128
1251
  raise ValueError(v)
1129
1252
  return v
1130
1253
 
1131
1254
 
1255
+ ########################################
1256
+ # ../../../omlish/lite/deathsig.py
1257
+
1258
+
1259
+ LINUX_PR_SET_PDEATHSIG = 1 # Second arg is a signal
1260
+ LINUX_PR_GET_PDEATHSIG = 2 # Second arg is a ptr to return the signal
1261
+
1262
+
1263
+ def set_process_deathsig(sig: int) -> bool:
1264
+ if sys.platform == 'linux':
1265
+ libc = ct.CDLL('libc.so.6')
1266
+
1267
+ # int prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5);
1268
+ libc.prctl.restype = ct.c_int
1269
+ libc.prctl.argtypes = [ct.c_int, ct.c_ulong, ct.c_ulong, ct.c_ulong, ct.c_ulong]
1270
+
1271
+ libc.prctl(LINUX_PR_SET_PDEATHSIG, sig, 0, 0, 0, 0)
1272
+
1273
+ return True
1274
+
1275
+ else:
1276
+ return False
1277
+
1278
+
1132
1279
  ########################################
1133
1280
  # ../../../omlish/lite/json.py
1134
1281
 
@@ -1910,8 +2057,8 @@ class Command(abc.ABC, ta.Generic[CommandOutputT]):
1910
2057
  pass
1911
2058
 
1912
2059
  @ta.final
1913
- def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
1914
- return check_isinstance(executor.execute(self), self.Output) # type: ignore[return-value]
2060
+ async def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
2061
+ return check_isinstance(await executor.execute(self), self.Output) # type: ignore[return-value]
1915
2062
 
1916
2063
 
1917
2064
  ##
@@ -1972,10 +2119,10 @@ class CommandOutputOrExceptionData(CommandOutputOrException):
1972
2119
 
1973
2120
  class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
1974
2121
  @abc.abstractmethod
1975
- def execute(self, cmd: CommandT) -> CommandOutputT:
2122
+ def execute(self, cmd: CommandT) -> ta.Awaitable[CommandOutputT]:
1976
2123
  raise NotImplementedError
1977
2124
 
1978
- def try_execute(
2125
+ async def try_execute(
1979
2126
  self,
1980
2127
  cmd: CommandT,
1981
2128
  *,
@@ -1983,7 +2130,7 @@ class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
1983
2130
  omit_exc_object: bool = False,
1984
2131
  ) -> CommandOutputOrException[CommandOutputT]:
1985
2132
  try:
1986
- o = self.execute(cmd)
2133
+ o = await self.execute(cmd)
1987
2134
 
1988
2135
  except Exception as e: # noqa
1989
2136
  if log is not None:
@@ -2054,8 +2201,18 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
2054
2201
  class RemoteConfig:
2055
2202
  payload_file: ta.Optional[str] = None
2056
2203
 
2204
+ set_pgid: bool = True
2205
+
2206
+ deathsig: ta.Optional[str] = 'KILL'
2207
+
2057
2208
  pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
2058
2209
 
2210
+ forward_logging: bool = True
2211
+
2212
+ timebomb_delay_s: ta.Optional[float] = 60 * 60.
2213
+
2214
+ heartbeat_interval_s: float = 3.
2215
+
2059
2216
 
2060
2217
  ########################################
2061
2218
  # ../remote/payload.py
@@ -3788,6 +3945,8 @@ class InterpSpecifier:
3788
3945
  s, o = InterpOpts.parse_suffix(s)
3789
3946
  if not any(s.startswith(o) for o in Specifier.OPERATORS):
3790
3947
  s = '~=' + s
3948
+ if s.count('.') < 2:
3949
+ s += '.0'
3791
3950
  return cls(
3792
3951
  specifier=Specifier(s),
3793
3952
  opts=o,
@@ -3834,9 +3993,9 @@ class LocalCommandExecutor(CommandExecutor):
3834
3993
 
3835
3994
  self._command_executors = command_executors
3836
3995
 
3837
- def execute(self, cmd: Command) -> Command.Output:
3996
+ async def execute(self, cmd: Command) -> Command.Output:
3838
3997
  ce: CommandExecutor = self._command_executors[type(cmd)]
3839
- return ce.execute(cmd)
3998
+ return await ce.execute(cmd)
3840
3999
 
3841
4000
 
3842
4001
  ########################################
@@ -3882,7 +4041,7 @@ class DeployCommand(Command['DeployCommand.Output']):
3882
4041
 
3883
4042
 
3884
4043
  class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
3885
- def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
4044
+ async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
3886
4045
  log.info('Deploying!')
3887
4046
 
3888
4047
  return DeployCommand.Output()
@@ -3904,11 +4063,30 @@ ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMar
3904
4063
  # ../remote/channel.py
3905
4064
 
3906
4065
 
3907
- class RemoteChannel:
4066
+ ##
4067
+
4068
+
4069
+ class RemoteChannel(abc.ABC):
4070
+ @abc.abstractmethod
4071
+ def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
4072
+ raise NotImplementedError
4073
+
4074
+ @abc.abstractmethod
4075
+ def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
4076
+ raise NotImplementedError
4077
+
4078
+ def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
4079
+ pass
4080
+
4081
+
4082
+ ##
4083
+
4084
+
4085
+ class RemoteChannelImpl(RemoteChannel):
3908
4086
  def __init__(
3909
4087
  self,
3910
- input: ta.IO, # noqa
3911
- output: ta.IO,
4088
+ input: asyncio.StreamReader, # noqa
4089
+ output: asyncio.StreamWriter,
3912
4090
  *,
3913
4091
  msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
3914
4092
  ) -> None:
@@ -3918,41 +4096,46 @@ class RemoteChannel:
3918
4096
  self._output = output
3919
4097
  self._msh = msh
3920
4098
 
3921
- self._lock = threading.RLock()
4099
+ self._input_lock = asyncio.Lock()
4100
+ self._output_lock = asyncio.Lock()
3922
4101
 
3923
4102
  def set_marshaler(self, msh: ObjMarshalerManager) -> None:
3924
4103
  self._msh = msh
3925
4104
 
3926
- def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
4105
+ #
4106
+
4107
+ async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
3927
4108
  j = json_dumps_compact(self._msh.marshal_obj(o, ty))
3928
4109
  d = j.encode('utf-8')
3929
4110
 
3930
4111
  self._output.write(struct.pack('<I', len(d)))
3931
4112
  self._output.write(d)
3932
- self._output.flush()
4113
+ await self._output.drain()
3933
4114
 
3934
- def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
3935
- with self._lock:
3936
- return self._send_obj(o, ty)
4115
+ async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
4116
+ async with self._output_lock:
4117
+ return await self._send_obj(o, ty)
4118
+
4119
+ #
3937
4120
 
3938
- def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
3939
- d = self._input.read(4)
4121
+ async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
4122
+ d = await self._input.read(4)
3940
4123
  if not d:
3941
4124
  return None
3942
4125
  if len(d) != 4:
3943
4126
  raise EOFError
3944
4127
 
3945
4128
  sz = struct.unpack('<I', d)[0]
3946
- d = self._input.read(sz)
4129
+ d = await self._input.read(sz)
3947
4130
  if len(d) != sz:
3948
4131
  raise EOFError
3949
4132
 
3950
4133
  j = json.loads(d.decode('utf-8'))
3951
4134
  return self._msh.unmarshal_obj(j, ty)
3952
4135
 
3953
- def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
3954
- with self._lock:
3955
- return self._recv_obj(ty)
4136
+ async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
4137
+ async with self._input_lock:
4138
+ return await self._recv_obj(ty)
3956
4139
 
3957
4140
 
3958
4141
  ########################################
@@ -3986,7 +4169,7 @@ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
3986
4169
  return args
3987
4170
 
3988
4171
 
3989
- def _prepare_subprocess_invocation(
4172
+ def prepare_subprocess_invocation(
3990
4173
  *args: str,
3991
4174
  env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
3992
4175
  extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
@@ -3994,9 +4177,9 @@ def _prepare_subprocess_invocation(
3994
4177
  shell: bool = False,
3995
4178
  **kwargs: ta.Any,
3996
4179
  ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
3997
- log.debug(args)
4180
+ log.debug('prepare_subprocess_invocation: args=%r', args)
3998
4181
  if extra_env:
3999
- log.debug(extra_env)
4182
+ log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
4000
4183
 
4001
4184
  if extra_env:
4002
4185
  env = {**(env if env is not None else os.environ), **extra_env}
@@ -4015,14 +4198,46 @@ def _prepare_subprocess_invocation(
4015
4198
  )
4016
4199
 
4017
4200
 
4018
- def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
4019
- args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
4020
- return subprocess.check_call(args, **kwargs) # type: ignore
4201
+ ##
4202
+
4203
+
4204
+ @contextlib.contextmanager
4205
+ def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
4206
+ start_time = time.time()
4207
+ try:
4208
+ log.debug('subprocess_common_context.try: args=%r', args)
4209
+ yield
4210
+
4211
+ except Exception as exc: # noqa
4212
+ log.debug('subprocess_common_context.except: exc=%r', exc)
4213
+ raise
4214
+
4215
+ finally:
4216
+ end_time = time.time()
4217
+ elapsed_s = end_time - start_time
4218
+ log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
4219
+
4220
+
4221
+ ##
4222
+
4223
+
4224
+ def subprocess_check_call(
4225
+ *args: str,
4226
+ stdout: ta.Any = sys.stderr,
4227
+ **kwargs: ta.Any,
4228
+ ) -> None:
4229
+ args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
4230
+ with subprocess_common_context(*args, **kwargs):
4231
+ return subprocess.check_call(args, **kwargs) # type: ignore
4021
4232
 
4022
4233
 
4023
- def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
4024
- args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
4025
- return subprocess.check_output(args, **kwargs)
4234
+ def subprocess_check_output(
4235
+ *args: str,
4236
+ **kwargs: ta.Any,
4237
+ ) -> bytes:
4238
+ args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
4239
+ with subprocess_common_context(*args, **kwargs):
4240
+ return subprocess.check_output(args, **kwargs)
4026
4241
 
4027
4242
 
4028
4243
  def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
@@ -4038,16 +4253,31 @@ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
4038
4253
  )
4039
4254
 
4040
4255
 
4041
- def subprocess_try_call(
4042
- *args: str,
4256
+ def _subprocess_try_run(
4257
+ fn: ta.Callable[..., T],
4258
+ *args: ta.Any,
4043
4259
  try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4044
4260
  **kwargs: ta.Any,
4045
- ) -> bool:
4261
+ ) -> ta.Union[T, Exception]:
4046
4262
  try:
4047
- subprocess_check_call(*args, **kwargs)
4263
+ return fn(*args, **kwargs)
4048
4264
  except try_exceptions as e: # noqa
4049
4265
  if log.isEnabledFor(logging.DEBUG):
4050
4266
  log.exception('command failed')
4267
+ return e
4268
+
4269
+
4270
+ def subprocess_try_call(
4271
+ *args: str,
4272
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4273
+ **kwargs: ta.Any,
4274
+ ) -> bool:
4275
+ if isinstance(_subprocess_try_run(
4276
+ subprocess_check_call,
4277
+ *args,
4278
+ try_exceptions=try_exceptions,
4279
+ **kwargs,
4280
+ ), Exception):
4051
4281
  return False
4052
4282
  else:
4053
4283
  return True
@@ -4058,12 +4288,15 @@ def subprocess_try_output(
4058
4288
  try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4059
4289
  **kwargs: ta.Any,
4060
4290
  ) -> ta.Optional[bytes]:
4061
- try:
4062
- return subprocess_check_output(*args, **kwargs)
4063
- except try_exceptions as e: # noqa
4064
- if log.isEnabledFor(logging.DEBUG):
4065
- log.exception('command failed')
4291
+ if isinstance(ret := _subprocess_try_run(
4292
+ subprocess_check_output,
4293
+ *args,
4294
+ try_exceptions=try_exceptions,
4295
+ **kwargs,
4296
+ ), Exception):
4066
4297
  return None
4298
+ else:
4299
+ return ret
4067
4300
 
4068
4301
 
4069
4302
  def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
@@ -4090,175 +4323,922 @@ def subprocess_close(
4090
4323
 
4091
4324
 
4092
4325
  ########################################
4093
- # ../../../omdev/interp/inspect.py
4326
+ # ../remote/execution.py
4094
4327
 
4095
4328
 
4096
- @dc.dataclass(frozen=True)
4097
- class InterpInspection:
4098
- exe: str
4099
- version: Version
4329
+ ##
4100
4330
 
4101
- version_str: str
4102
- config_vars: ta.Mapping[str, str]
4103
- prefix: str
4104
- base_prefix: str
4105
4331
 
4106
- @property
4107
- def opts(self) -> InterpOpts:
4108
- return InterpOpts(
4109
- threaded=bool(self.config_vars.get('Py_GIL_DISABLED')),
4110
- debug=bool(self.config_vars.get('Py_DEBUG')),
4111
- )
4332
+ class _RemoteProtocol:
4333
+ class Message(abc.ABC): # noqa
4334
+ async def send(self, chan: RemoteChannel) -> None:
4335
+ await chan.send_obj(self, _RemoteProtocol.Message)
4112
4336
 
4113
- @property
4114
- def iv(self) -> InterpVersion:
4115
- return InterpVersion(
4116
- version=self.version,
4117
- opts=self.opts,
4118
- )
4337
+ @classmethod
4338
+ async def recv(cls: ta.Type[T], chan: RemoteChannel) -> ta.Optional[T]:
4339
+ return await chan.recv_obj(cls)
4119
4340
 
4120
- @property
4121
- def is_venv(self) -> bool:
4122
- return self.prefix != self.base_prefix
4341
+ #
4123
4342
 
4343
+ class Request(Message, abc.ABC): # noqa
4344
+ pass
4124
4345
 
4125
- class InterpInspector:
4346
+ @dc.dataclass(frozen=True)
4347
+ class CommandRequest(Request):
4348
+ seq: int
4349
+ cmd: Command
4126
4350
 
4127
- def __init__(self) -> None:
4128
- super().__init__()
4351
+ @dc.dataclass(frozen=True)
4352
+ class PingRequest(Request):
4353
+ time: float
4129
4354
 
4130
- self._cache: ta.Dict[str, ta.Optional[InterpInspection]] = {}
4355
+ #
4131
4356
 
4132
- _RAW_INSPECTION_CODE = """
4133
- __import__('json').dumps(dict(
4134
- version_str=__import__('sys').version,
4135
- prefix=__import__('sys').prefix,
4136
- base_prefix=__import__('sys').base_prefix,
4137
- config_vars=__import__('sysconfig').get_config_vars(),
4138
- ))"""
4357
+ class Response(Message, abc.ABC): # noqa
4358
+ pass
4139
4359
 
4140
- _INSPECTION_CODE = ''.join(l.strip() for l in _RAW_INSPECTION_CODE.splitlines())
4360
+ @dc.dataclass(frozen=True)
4361
+ class LogResponse(Response):
4362
+ s: str
4141
4363
 
4142
- @staticmethod
4143
- def _build_inspection(
4144
- exe: str,
4145
- output: str,
4146
- ) -> InterpInspection:
4147
- dct = json.loads(output)
4364
+ @dc.dataclass(frozen=True)
4365
+ class CommandResponse(Response):
4366
+ seq: int
4367
+ res: CommandOutputOrExceptionData
4148
4368
 
4149
- version = Version(dct['version_str'].split()[0])
4369
+ @dc.dataclass(frozen=True)
4370
+ class PingResponse(Response):
4371
+ time: float
4150
4372
 
4151
- return InterpInspection(
4152
- exe=exe,
4153
- version=version,
4154
- **{k: dct[k] for k in (
4155
- 'version_str',
4156
- 'prefix',
4157
- 'base_prefix',
4158
- 'config_vars',
4159
- )},
4160
- )
4161
4373
 
4162
- @classmethod
4163
- def running(cls) -> 'InterpInspection':
4164
- return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
4374
+ ##
4165
4375
 
4166
- def _inspect(self, exe: str) -> InterpInspection:
4167
- output = subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
4168
- return self._build_inspection(exe, output.decode())
4169
4376
 
4170
- def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
4171
- try:
4172
- return self._cache[exe]
4173
- except KeyError:
4174
- ret: ta.Optional[InterpInspection]
4175
- try:
4176
- ret = self._inspect(exe)
4177
- except Exception as e: # noqa
4178
- if log.isEnabledFor(logging.DEBUG):
4179
- log.exception('Failed to inspect interp: %s', exe)
4180
- ret = None
4181
- self._cache[exe] = ret
4182
- return ret
4377
+ class _RemoteLogHandler(logging.Handler):
4378
+ def __init__(
4379
+ self,
4380
+ chan: RemoteChannel,
4381
+ loop: ta.Any = None,
4382
+ ) -> None:
4383
+ super().__init__()
4183
4384
 
4385
+ self._chan = chan
4386
+ self._loop = loop
4184
4387
 
4185
- INTERP_INSPECTOR = InterpInspector()
4388
+ def emit(self, record):
4389
+ msg = self.format(record)
4186
4390
 
4391
+ async def inner():
4392
+ await _RemoteProtocol.LogResponse(msg).send(self._chan)
4187
4393
 
4188
- ########################################
4189
- # ../commands/subprocess.py
4394
+ loop = self._loop
4395
+ if loop is None:
4396
+ loop = asyncio.get_running_loop()
4397
+ if loop is not None:
4398
+ asyncio.run_coroutine_threadsafe(inner(), loop)
4190
4399
 
4191
4400
 
4192
4401
  ##
4193
4402
 
4194
4403
 
4195
- @dc.dataclass(frozen=True)
4196
- class SubprocessCommand(Command['SubprocessCommand.Output']):
4197
- cmd: ta.Sequence[str]
4404
+ class _RemoteCommandHandler:
4405
+ def __init__(
4406
+ self,
4407
+ chan: RemoteChannel,
4408
+ executor: CommandExecutor,
4409
+ *,
4410
+ stop: ta.Optional[asyncio.Event] = None,
4411
+ ) -> None:
4412
+ super().__init__()
4198
4413
 
4199
- shell: bool = False
4200
- cwd: ta.Optional[str] = None
4201
- env: ta.Optional[ta.Mapping[str, str]] = None
4414
+ self._chan = chan
4415
+ self._executor = executor
4416
+ self._stop = stop if stop is not None else asyncio.Event()
4202
4417
 
4203
- stdout: str = 'pipe' # SubprocessChannelOption
4204
- stderr: str = 'pipe' # SubprocessChannelOption
4418
+ self._cmds_by_seq: ta.Dict[int, _RemoteCommandHandler._Command] = {}
4419
+
4420
+ @dc.dataclass(frozen=True)
4421
+ class _Command:
4422
+ req: _RemoteProtocol.CommandRequest
4423
+ fut: asyncio.Future
4424
+
4425
+ async def run(self) -> None:
4426
+ stop_task = asyncio.create_task(self._stop.wait())
4427
+ recv_task: ta.Optional[asyncio.Task] = None
4428
+
4429
+ while not self._stop.is_set():
4430
+ if recv_task is None:
4431
+ recv_task = asyncio.create_task(_RemoteProtocol.Request.recv(self._chan))
4432
+
4433
+ done, pending = await asyncio.wait([
4434
+ stop_task,
4435
+ recv_task,
4436
+ ], return_when=asyncio.FIRST_COMPLETED)
4437
+
4438
+ if recv_task in done:
4439
+ msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
4440
+ recv_task.result(),
4441
+ (_RemoteProtocol.Message, type(None)),
4442
+ )
4443
+ recv_task = None
4444
+
4445
+ if msg is None:
4446
+ break
4447
+
4448
+ await self._handle_message(msg)
4449
+
4450
+ async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
4451
+ if isinstance(msg, _RemoteProtocol.PingRequest):
4452
+ log.debug('Ping: %r', msg)
4453
+ await _RemoteProtocol.PingResponse(
4454
+ time=msg.time,
4455
+ ).send(self._chan)
4456
+
4457
+ elif isinstance(msg, _RemoteProtocol.CommandRequest):
4458
+ fut = asyncio.create_task(self._handle_command_request(msg))
4459
+ self._cmds_by_seq[msg.seq] = _RemoteCommandHandler._Command(
4460
+ req=msg,
4461
+ fut=fut,
4462
+ )
4463
+
4464
+ else:
4465
+ raise TypeError(msg)
4466
+
4467
+ async def _handle_command_request(self, req: _RemoteProtocol.CommandRequest) -> None:
4468
+ res = await self._executor.try_execute(
4469
+ req.cmd,
4470
+ log=log,
4471
+ omit_exc_object=True,
4472
+ )
4473
+
4474
+ await _RemoteProtocol.CommandResponse(
4475
+ seq=req.seq,
4476
+ res=CommandOutputOrExceptionData(
4477
+ output=res.output,
4478
+ exception=res.exception,
4479
+ ),
4480
+ ).send(self._chan)
4481
+
4482
+ self._cmds_by_seq.pop(req.seq) # noqa
4483
+
4484
+
4485
+ ##
4486
+
4487
+
4488
+ @dc.dataclass()
4489
+ class RemoteCommandError(Exception):
4490
+ e: CommandException
4491
+
4492
+
4493
+ class RemoteCommandExecutor(CommandExecutor):
4494
+ def __init__(self, chan: RemoteChannel) -> None:
4495
+ super().__init__()
4496
+
4497
+ self._chan = chan
4498
+
4499
+ self._cmd_seq = itertools.count()
4500
+ self._queue: asyncio.Queue = asyncio.Queue() # asyncio.Queue[RemoteCommandExecutor._Request]
4501
+ self._stop = asyncio.Event()
4502
+ self._loop_task: ta.Optional[asyncio.Task] = None
4503
+ self._reqs_by_seq: ta.Dict[int, RemoteCommandExecutor._Request] = {}
4504
+
4505
+ #
4506
+
4507
+ async def start(self) -> None:
4508
+ check_none(self._loop_task)
4509
+ check_state(not self._stop.is_set())
4510
+ self._loop_task = asyncio.create_task(self._loop())
4511
+
4512
+ async def aclose(self) -> None:
4513
+ self._stop.set()
4514
+ if self._loop_task is not None:
4515
+ await self._loop_task
4516
+
4517
+ #
4518
+
4519
+ @dc.dataclass(frozen=True)
4520
+ class _Request:
4521
+ seq: int
4522
+ cmd: Command
4523
+ fut: asyncio.Future
4524
+
4525
+ async def _loop(self) -> None:
4526
+ log.debug('RemoteCommandExecutor loop start: %r', self)
4527
+
4528
+ stop_task = asyncio.create_task(self._stop.wait())
4529
+ queue_task: ta.Optional[asyncio.Task] = None
4530
+ recv_task: ta.Optional[asyncio.Task] = None
4531
+
4532
+ while not self._stop.is_set():
4533
+ if queue_task is None:
4534
+ queue_task = asyncio.create_task(self._queue.get())
4535
+ if recv_task is None:
4536
+ recv_task = asyncio.create_task(_RemoteProtocol.Message.recv(self._chan))
4537
+
4538
+ done, pending = await asyncio.wait([
4539
+ stop_task,
4540
+ queue_task,
4541
+ recv_task,
4542
+ ], return_when=asyncio.FIRST_COMPLETED)
4543
+
4544
+ if queue_task in done:
4545
+ req = check_isinstance(queue_task.result(), RemoteCommandExecutor._Request)
4546
+ queue_task = None
4547
+ await self._handle_request(req)
4548
+
4549
+ if recv_task in done:
4550
+ msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
4551
+ recv_task.result(),
4552
+ (_RemoteProtocol.Message, type(None)),
4553
+ )
4554
+ recv_task = None
4555
+
4556
+ if msg is None:
4557
+ log.debug('RemoteCommandExecutor got eof: %r', self)
4558
+ break
4559
+
4560
+ await self._handle_message(msg)
4561
+
4562
+ log.debug('RemoteCommandExecutor loop stopping: %r', self)
4563
+
4564
+ for task in [
4565
+ stop_task,
4566
+ queue_task,
4567
+ recv_task,
4568
+ ]:
4569
+ if task is not None and not task.done():
4570
+ task.cancel()
4571
+
4572
+ for req in self._reqs_by_seq.values():
4573
+ req.fut.cancel()
4574
+
4575
+ log.debug('RemoteCommandExecutor loop exited: %r', self)
4576
+
4577
+ async def _handle_request(self, req: _Request) -> None:
4578
+ self._reqs_by_seq[req.seq] = req
4579
+ await _RemoteProtocol.CommandRequest(
4580
+ seq=req.seq,
4581
+ cmd=req.cmd,
4582
+ ).send(self._chan)
4583
+
4584
+ async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
4585
+ if isinstance(msg, _RemoteProtocol.PingRequest):
4586
+ log.debug('Ping: %r', msg)
4587
+ await _RemoteProtocol.PingResponse(
4588
+ time=msg.time,
4589
+ ).send(self._chan)
4590
+
4591
+ elif isinstance(msg, _RemoteProtocol.LogResponse):
4592
+ log.info(msg.s)
4593
+
4594
+ elif isinstance(msg, _RemoteProtocol.CommandResponse):
4595
+ req = self._reqs_by_seq.pop(msg.seq)
4596
+ req.fut.set_result(msg.res)
4597
+
4598
+ else:
4599
+ raise TypeError(msg)
4600
+
4601
+ #
4602
+
4603
+ async def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
4604
+ req = RemoteCommandExecutor._Request(
4605
+ seq=next(self._cmd_seq),
4606
+ cmd=cmd,
4607
+ fut=asyncio.Future(),
4608
+ )
4609
+ await self._queue.put(req)
4610
+ return await req.fut
4611
+
4612
+ # @ta.override
4613
+ async def execute(self, cmd: Command) -> Command.Output:
4614
+ r = await self._remote_execute(cmd)
4615
+ if (e := r.exception) is not None:
4616
+ raise RemoteCommandError(e)
4617
+ else:
4618
+ return check_not_none(r.output)
4619
+
4620
+ # @ta.override
4621
+ async def try_execute(
4622
+ self,
4623
+ cmd: Command,
4624
+ *,
4625
+ log: ta.Optional[logging.Logger] = None,
4626
+ omit_exc_object: bool = False,
4627
+ ) -> CommandOutputOrException:
4628
+ try:
4629
+ r = await self._remote_execute(cmd)
4630
+
4631
+ except Exception as e: # noqa
4632
+ if log is not None:
4633
+ log.exception('Exception executing remote command: %r', type(cmd))
4634
+
4635
+ return CommandOutputOrExceptionData(exception=CommandException.of(
4636
+ e,
4637
+ omit_exc_object=omit_exc_object,
4638
+ cmd=cmd,
4639
+ ))
4640
+
4641
+ else:
4642
+ return r
4643
+
4644
+
4645
+ ########################################
4646
+ # ../../../omlish/lite/asyncio/subprocesses.py
4647
+
4648
+
4649
+ ##
4650
+
4651
+
4652
+ @contextlib.asynccontextmanager
4653
+ async def asyncio_subprocess_popen(
4654
+ *cmd: str,
4655
+ shell: bool = False,
4656
+ timeout: ta.Optional[float] = None,
4657
+ **kwargs: ta.Any,
4658
+ ) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
4659
+ fac: ta.Any
4660
+ if shell:
4661
+ fac = functools.partial(
4662
+ asyncio.create_subprocess_shell,
4663
+ check_single(cmd),
4664
+ )
4665
+ else:
4666
+ fac = functools.partial(
4667
+ asyncio.create_subprocess_exec,
4668
+ *cmd,
4669
+ )
4670
+
4671
+ with subprocess_common_context(
4672
+ *cmd,
4673
+ shell=shell,
4674
+ timeout=timeout,
4675
+ **kwargs,
4676
+ ):
4677
+ proc: asyncio.subprocess.Process
4678
+ proc = await fac(**kwargs)
4679
+ try:
4680
+ yield proc
4681
+
4682
+ finally:
4683
+ await asyncio_maybe_timeout(proc.wait(), timeout)
4684
+
4685
+
4686
+ ##
4687
+
4688
+
4689
+ class AsyncioProcessCommunicator:
4690
+ def __init__(
4691
+ self,
4692
+ proc: asyncio.subprocess.Process,
4693
+ loop: ta.Optional[ta.Any] = None,
4694
+ ) -> None:
4695
+ super().__init__()
4696
+
4697
+ if loop is None:
4698
+ loop = asyncio.get_running_loop()
4699
+
4700
+ self._proc = proc
4701
+ self._loop = loop
4702
+
4703
+ self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check_isinstance(
4704
+ proc._transport, # type: ignore # noqa
4705
+ asyncio.base_subprocess.BaseSubprocessTransport,
4706
+ )
4707
+
4708
+ @property
4709
+ def _debug(self) -> bool:
4710
+ return self._loop.get_debug()
4711
+
4712
+ async def _feed_stdin(self, input: bytes) -> None: # noqa
4713
+ stdin = check_not_none(self._proc.stdin)
4714
+ try:
4715
+ if input is not None:
4716
+ stdin.write(input)
4717
+ if self._debug:
4718
+ log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
4719
+
4720
+ await stdin.drain()
4721
+
4722
+ except (BrokenPipeError, ConnectionResetError) as exc:
4723
+ # communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
4724
+ # exceptions.
4725
+ if self._debug:
4726
+ log.debug('%r communicate: stdin got %r', self, exc)
4727
+
4728
+ if self._debug:
4729
+ log.debug('%r communicate: close stdin', self)
4730
+
4731
+ stdin.close()
4732
+
4733
+ async def _noop(self) -> None:
4734
+ return None
4735
+
4736
+ async def _read_stream(self, fd: int) -> bytes:
4737
+ transport: ta.Any = check_not_none(self._transport.get_pipe_transport(fd))
4738
+
4739
+ if fd == 2:
4740
+ stream = check_not_none(self._proc.stderr)
4741
+ else:
4742
+ check_equal(fd, 1)
4743
+ stream = check_not_none(self._proc.stdout)
4744
+
4745
+ if self._debug:
4746
+ name = 'stdout' if fd == 1 else 'stderr'
4747
+ log.debug('%r communicate: read %s', self, name)
4748
+
4749
+ output = await stream.read()
4750
+
4751
+ if self._debug:
4752
+ name = 'stdout' if fd == 1 else 'stderr'
4753
+ log.debug('%r communicate: close %s', self, name)
4754
+
4755
+ transport.close()
4756
+
4757
+ return output
4758
+
4759
+ class Communication(ta.NamedTuple):
4760
+ stdout: ta.Optional[bytes]
4761
+ stderr: ta.Optional[bytes]
4762
+
4763
+ async def _communicate(
4764
+ self,
4765
+ input: ta.Any = None, # noqa
4766
+ ) -> Communication:
4767
+ stdin_fut: ta.Any
4768
+ if self._proc.stdin is not None:
4769
+ stdin_fut = self._feed_stdin(input)
4770
+ else:
4771
+ stdin_fut = self._noop()
4772
+
4773
+ stdout_fut: ta.Any
4774
+ if self._proc.stdout is not None:
4775
+ stdout_fut = self._read_stream(1)
4776
+ else:
4777
+ stdout_fut = self._noop()
4778
+
4779
+ stderr_fut: ta.Any
4780
+ if self._proc.stderr is not None:
4781
+ stderr_fut = self._read_stream(2)
4782
+ else:
4783
+ stderr_fut = self._noop()
4784
+
4785
+ stdin_res, stdout_res, stderr_res = await asyncio.gather(stdin_fut, stdout_fut, stderr_fut)
4786
+
4787
+ await self._proc.wait()
4788
+
4789
+ return AsyncioProcessCommunicator.Communication(stdout_res, stderr_res)
4790
+
4791
+ async def communicate(
4792
+ self,
4793
+ input: ta.Any = None, # noqa
4794
+ timeout: ta.Optional[float] = None,
4795
+ ) -> Communication:
4796
+ return await asyncio_maybe_timeout(self._communicate(input), timeout)
4797
+
4798
+
4799
+ async def asyncio_subprocess_communicate(
4800
+ proc: asyncio.subprocess.Process,
4801
+ input: ta.Any = None, # noqa
4802
+ timeout: ta.Optional[float] = None,
4803
+ ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
4804
+ return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
4805
+
4806
+
4807
+ ##
4808
+
4809
+
4810
+ async def _asyncio_subprocess_check_run(
4811
+ *args: str,
4812
+ input: ta.Any = None, # noqa
4813
+ timeout: ta.Optional[float] = None,
4814
+ **kwargs: ta.Any,
4815
+ ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
4816
+ args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
4817
+
4818
+ proc: asyncio.subprocess.Process
4819
+ async with asyncio_subprocess_popen(*args, **kwargs) as proc:
4820
+ stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
4821
+
4822
+ if proc.returncode:
4823
+ raise subprocess.CalledProcessError(
4824
+ proc.returncode,
4825
+ args,
4826
+ output=stdout,
4827
+ stderr=stderr,
4828
+ )
4829
+
4830
+ return stdout, stderr
4831
+
4832
+
4833
+ async def asyncio_subprocess_check_call(
4834
+ *args: str,
4835
+ stdout: ta.Any = sys.stderr,
4836
+ input: ta.Any = None, # noqa
4837
+ timeout: ta.Optional[float] = None,
4838
+ **kwargs: ta.Any,
4839
+ ) -> None:
4840
+ _, _ = await _asyncio_subprocess_check_run(
4841
+ *args,
4842
+ stdout=stdout,
4843
+ input=input,
4844
+ timeout=timeout,
4845
+ **kwargs,
4846
+ )
4847
+
4848
+
4849
+ async def asyncio_subprocess_check_output(
4850
+ *args: str,
4851
+ input: ta.Any = None, # noqa
4852
+ timeout: ta.Optional[float] = None,
4853
+ **kwargs: ta.Any,
4854
+ ) -> bytes:
4855
+ stdout, stderr = await _asyncio_subprocess_check_run(
4856
+ *args,
4857
+ stdout=asyncio.subprocess.PIPE,
4858
+ input=input,
4859
+ timeout=timeout,
4860
+ **kwargs,
4861
+ )
4862
+
4863
+ return check_not_none(stdout)
4864
+
4865
+
4866
+ async def asyncio_subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
4867
+ return (await asyncio_subprocess_check_output(*args, **kwargs)).decode().strip()
4868
+
4869
+
4870
+ ##
4871
+
4872
+
4873
+ async def _asyncio_subprocess_try_run(
4874
+ fn: ta.Callable[..., ta.Awaitable[T]],
4875
+ *args: ta.Any,
4876
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4877
+ **kwargs: ta.Any,
4878
+ ) -> ta.Union[T, Exception]:
4879
+ try:
4880
+ return await fn(*args, **kwargs)
4881
+ except try_exceptions as e: # noqa
4882
+ if log.isEnabledFor(logging.DEBUG):
4883
+ log.exception('command failed')
4884
+ return e
4885
+
4886
+
4887
+ async def asyncio_subprocess_try_call(
4888
+ *args: str,
4889
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4890
+ **kwargs: ta.Any,
4891
+ ) -> bool:
4892
+ if isinstance(await _asyncio_subprocess_try_run(
4893
+ asyncio_subprocess_check_call,
4894
+ *args,
4895
+ try_exceptions=try_exceptions,
4896
+ **kwargs,
4897
+ ), Exception):
4898
+ return False
4899
+ else:
4900
+ return True
4901
+
4902
+
4903
+ async def asyncio_subprocess_try_output(
4904
+ *args: str,
4905
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4906
+ **kwargs: ta.Any,
4907
+ ) -> ta.Optional[bytes]:
4908
+ if isinstance(ret := await _asyncio_subprocess_try_run(
4909
+ asyncio_subprocess_check_output,
4910
+ *args,
4911
+ try_exceptions=try_exceptions,
4912
+ **kwargs,
4913
+ ), Exception):
4914
+ return None
4915
+ else:
4916
+ return ret
4917
+
4918
+
4919
+ async def asyncio_subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
4920
+ out = await asyncio_subprocess_try_output(*args, **kwargs)
4921
+ return out.decode().strip() if out is not None else None
4922
+
4923
+
4924
+ ########################################
4925
+ # ../../../omdev/interp/inspect.py
4926
+
4927
+
4928
+ @dc.dataclass(frozen=True)
4929
+ class InterpInspection:
4930
+ exe: str
4931
+ version: Version
4932
+
4933
+ version_str: str
4934
+ config_vars: ta.Mapping[str, str]
4935
+ prefix: str
4936
+ base_prefix: str
4937
+
4938
+ @property
4939
+ def opts(self) -> InterpOpts:
4940
+ return InterpOpts(
4941
+ threaded=bool(self.config_vars.get('Py_GIL_DISABLED')),
4942
+ debug=bool(self.config_vars.get('Py_DEBUG')),
4943
+ )
4944
+
4945
+ @property
4946
+ def iv(self) -> InterpVersion:
4947
+ return InterpVersion(
4948
+ version=self.version,
4949
+ opts=self.opts,
4950
+ )
4951
+
4952
+ @property
4953
+ def is_venv(self) -> bool:
4954
+ return self.prefix != self.base_prefix
4955
+
4956
+
4957
+ class InterpInspector:
4958
+ def __init__(self) -> None:
4959
+ super().__init__()
4960
+
4961
+ self._cache: ta.Dict[str, ta.Optional[InterpInspection]] = {}
4962
+
4963
+ _RAW_INSPECTION_CODE = """
4964
+ __import__('json').dumps(dict(
4965
+ version_str=__import__('sys').version,
4966
+ prefix=__import__('sys').prefix,
4967
+ base_prefix=__import__('sys').base_prefix,
4968
+ config_vars=__import__('sysconfig').get_config_vars(),
4969
+ ))"""
4970
+
4971
+ _INSPECTION_CODE = ''.join(l.strip() for l in _RAW_INSPECTION_CODE.splitlines())
4972
+
4973
+ @staticmethod
4974
+ def _build_inspection(
4975
+ exe: str,
4976
+ output: str,
4977
+ ) -> InterpInspection:
4978
+ dct = json.loads(output)
4979
+
4980
+ version = Version(dct['version_str'].split()[0])
4981
+
4982
+ return InterpInspection(
4983
+ exe=exe,
4984
+ version=version,
4985
+ **{k: dct[k] for k in (
4986
+ 'version_str',
4987
+ 'prefix',
4988
+ 'base_prefix',
4989
+ 'config_vars',
4990
+ )},
4991
+ )
4992
+
4993
+ @classmethod
4994
+ def running(cls) -> 'InterpInspection':
4995
+ return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
4996
+
4997
+ async def _inspect(self, exe: str) -> InterpInspection:
4998
+ output = await asyncio_subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
4999
+ return self._build_inspection(exe, output.decode())
5000
+
5001
+ async def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
5002
+ try:
5003
+ return self._cache[exe]
5004
+ except KeyError:
5005
+ ret: ta.Optional[InterpInspection]
5006
+ try:
5007
+ ret = await self._inspect(exe)
5008
+ except Exception as e: # noqa
5009
+ if log.isEnabledFor(logging.DEBUG):
5010
+ log.exception('Failed to inspect interp: %s', exe)
5011
+ ret = None
5012
+ self._cache[exe] = ret
5013
+ return ret
5014
+
5015
+
5016
+ INTERP_INSPECTOR = InterpInspector()
5017
+
5018
+
5019
+ ########################################
5020
+ # ../commands/subprocess.py
5021
+
5022
+
5023
+ ##
5024
+
5025
+
5026
+ @dc.dataclass(frozen=True)
5027
+ class SubprocessCommand(Command['SubprocessCommand.Output']):
5028
+ cmd: ta.Sequence[str]
5029
+
5030
+ shell: bool = False
5031
+ cwd: ta.Optional[str] = None
5032
+ env: ta.Optional[ta.Mapping[str, str]] = None
5033
+
5034
+ stdout: str = 'pipe' # SubprocessChannelOption
5035
+ stderr: str = 'pipe' # SubprocessChannelOption
4205
5036
 
4206
5037
  input: ta.Optional[bytes] = None
4207
5038
  timeout: ta.Optional[float] = None
4208
5039
 
4209
- def __post_init__(self) -> None:
4210
- check_not_isinstance(self.cmd, str)
5040
+ def __post_init__(self) -> None:
5041
+ check_not_isinstance(self.cmd, str)
5042
+
5043
+ @dc.dataclass(frozen=True)
5044
+ class Output(Command.Output):
5045
+ rc: int
5046
+ pid: int
5047
+
5048
+ elapsed_s: float
5049
+
5050
+ stdout: ta.Optional[bytes] = None
5051
+ stderr: ta.Optional[bytes] = None
5052
+
5053
+
5054
+ ##
5055
+
5056
+
5057
+ class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
5058
+ async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
5059
+ proc: asyncio.subprocess.Process
5060
+ async with asyncio_subprocess_popen(
5061
+ *subprocess_maybe_shell_wrap_exec(*cmd.cmd),
5062
+
5063
+ shell=cmd.shell,
5064
+ cwd=cmd.cwd,
5065
+ env={**os.environ, **(cmd.env or {})},
5066
+
5067
+ stdin=subprocess.PIPE if cmd.input is not None else None,
5068
+ stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stdout)],
5069
+ stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stderr)],
5070
+
5071
+ timeout=cmd.timeout,
5072
+ ) as proc:
5073
+ start_time = time.time()
5074
+ stdout, stderr = await asyncio_subprocess_communicate(
5075
+ proc,
5076
+ input=cmd.input,
5077
+ timeout=cmd.timeout,
5078
+ )
5079
+ end_time = time.time()
5080
+
5081
+ return SubprocessCommand.Output(
5082
+ rc=check_not_none(proc.returncode),
5083
+ pid=proc.pid,
5084
+
5085
+ elapsed_s=end_time - start_time,
5086
+
5087
+ stdout=stdout, # noqa
5088
+ stderr=stderr, # noqa
5089
+ )
5090
+
5091
+
5092
+ ########################################
5093
+ # ../remote/_main.py
5094
+
5095
+
5096
+ ##
5097
+
5098
+
5099
+ class _RemoteExecutionLogHandler(logging.Handler):
5100
+ def __init__(self, fn: ta.Callable[[str], None]) -> None:
5101
+ super().__init__()
5102
+ self._fn = fn
5103
+
5104
+ def emit(self, record):
5105
+ msg = self.format(record)
5106
+ self._fn(msg)
5107
+
5108
+
5109
+ ##
5110
+
5111
+
5112
+ class _RemoteExecutionMain:
5113
+ def __init__(
5114
+ self,
5115
+ chan: RemoteChannel,
5116
+ ) -> None:
5117
+ super().__init__()
5118
+
5119
+ self._chan = chan
5120
+
5121
+ self.__bootstrap: ta.Optional[MainBootstrap] = None
5122
+ self.__injector: ta.Optional[Injector] = None
5123
+
5124
+ @property
5125
+ def _bootstrap(self) -> MainBootstrap:
5126
+ return check_not_none(self.__bootstrap)
5127
+
5128
+ @property
5129
+ def _injector(self) -> Injector:
5130
+ return check_not_none(self.__injector)
5131
+
5132
+ #
5133
+
5134
+ def _timebomb_main(
5135
+ self,
5136
+ delay_s: float,
5137
+ *,
5138
+ sig: int = signal.SIGINT,
5139
+ code: int = 1,
5140
+ ) -> None:
5141
+ time.sleep(delay_s)
5142
+
5143
+ if (pgid := os.getpgid(0)) == os.getpid():
5144
+ os.killpg(pgid, sig)
5145
+
5146
+ os._exit(code) # noqa
5147
+
5148
+ @cached_nullary
5149
+ def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
5150
+ if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
5151
+ return None
5152
+
5153
+ thr = threading.Thread(
5154
+ target=functools.partial(self._timebomb_main, tbd),
5155
+ name=f'{self.__class__.__name__}.timebomb',
5156
+ daemon=True,
5157
+ )
5158
+
5159
+ thr.start()
4211
5160
 
4212
- @dc.dataclass(frozen=True)
4213
- class Output(Command.Output):
4214
- rc: int
4215
- pid: int
5161
+ log.debug('Started timebomb thread: %r', thr)
4216
5162
 
4217
- elapsed_s: float
5163
+ return thr
4218
5164
 
4219
- stdout: ta.Optional[bytes] = None
4220
- stderr: ta.Optional[bytes] = None
5165
+ #
4221
5166
 
5167
+ @cached_nullary
5168
+ def _log_handler(self) -> _RemoteLogHandler:
5169
+ return _RemoteLogHandler(self._chan)
4222
5170
 
4223
- ##
5171
+ #
4224
5172
 
5173
+ async def _setup(self) -> None:
5174
+ check_none(self.__bootstrap)
5175
+ check_none(self.__injector)
4225
5176
 
4226
- class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
4227
- def execute(self, inp: SubprocessCommand) -> SubprocessCommand.Output:
4228
- with subprocess.Popen(
4229
- subprocess_maybe_shell_wrap_exec(*inp.cmd),
5177
+ # Bootstrap
4230
5178
 
4231
- shell=inp.shell,
4232
- cwd=inp.cwd,
4233
- env={**os.environ, **(inp.env or {})},
5179
+ self.__bootstrap = check_not_none(await self._chan.recv_obj(MainBootstrap))
4234
5180
 
4235
- stdin=subprocess.PIPE if inp.input is not None else None,
4236
- stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stdout)],
4237
- stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stderr)],
4238
- ) as proc:
4239
- start_time = time.time()
4240
- stdout, stderr = proc.communicate(
4241
- input=inp.input,
4242
- timeout=inp.timeout,
4243
- )
4244
- end_time = time.time()
5181
+ if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
5182
+ pycharm_debug_connect(prd)
4245
5183
 
4246
- return SubprocessCommand.Output(
4247
- rc=proc.returncode,
4248
- pid=proc.pid,
5184
+ self.__injector = main_bootstrap(self._bootstrap)
4249
5185
 
4250
- elapsed_s=end_time - start_time,
5186
+ self._chan.set_marshaler(self._injector[ObjMarshalerManager])
4251
5187
 
4252
- stdout=stdout, # noqa
4253
- stderr=stderr, # noqa
5188
+ # Post-bootstrap
5189
+
5190
+ if self._bootstrap.remote_config.set_pgid:
5191
+ if os.getpgid(0) != os.getpid():
5192
+ log.debug('Setting pgid')
5193
+ os.setpgid(0, 0)
5194
+
5195
+ if (ds := self._bootstrap.remote_config.deathsig) is not None:
5196
+ log.debug('Setting deathsig: %s', ds)
5197
+ set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
5198
+
5199
+ self._timebomb_thread()
5200
+
5201
+ if self._bootstrap.remote_config.forward_logging:
5202
+ log.debug('Installing log forwarder')
5203
+ logging.root.addHandler(self._log_handler())
5204
+
5205
+ #
5206
+
5207
+ async def run(self) -> None:
5208
+ await self._setup()
5209
+
5210
+ executor = self._injector[LocalCommandExecutor]
5211
+
5212
+ handler = _RemoteCommandHandler(self._chan, executor)
5213
+
5214
+ await handler.run()
5215
+
5216
+
5217
+ def _remote_execution_main() -> None:
5218
+ rt = pyremote_bootstrap_finalize() # noqa
5219
+
5220
+ async def inner() -> None:
5221
+ input = await asyncio_open_stream_reader(rt.input) # noqa
5222
+ output = await asyncio_open_stream_writer(rt.output)
5223
+
5224
+ chan = RemoteChannelImpl(
5225
+ input,
5226
+ output,
4254
5227
  )
4255
5228
 
5229
+ await _RemoteExecutionMain(chan).run()
5230
+
5231
+ asyncio.run(inner())
5232
+
4256
5233
 
4257
5234
  ########################################
4258
5235
  # ../remote/spawning.py
4259
5236
 
4260
5237
 
4261
- class RemoteSpawning:
5238
+ ##
5239
+
5240
+
5241
+ class RemoteSpawning(abc.ABC):
4262
5242
  @dc.dataclass(frozen=True)
4263
5243
  class Target:
4264
5244
  shell: ta.Optional[str] = None
@@ -4269,15 +5249,35 @@ class RemoteSpawning:
4269
5249
 
4270
5250
  stderr: ta.Optional[str] = None # SubprocessChannelOption
4271
5251
 
4272
- #
5252
+ @dc.dataclass(frozen=True)
5253
+ class Spawned:
5254
+ stdin: asyncio.StreamWriter
5255
+ stdout: asyncio.StreamReader
5256
+ stderr: ta.Optional[asyncio.StreamReader]
5257
+
5258
+ @abc.abstractmethod
5259
+ def spawn(
5260
+ self,
5261
+ tgt: Target,
5262
+ src: str,
5263
+ *,
5264
+ timeout: ta.Optional[float] = None,
5265
+ debug: bool = False,
5266
+ ) -> ta.AsyncContextManager[Spawned]:
5267
+ raise NotImplementedError
4273
5268
 
4274
- class _PreparedCmd(ta.NamedTuple):
5269
+
5270
+ ##
5271
+
5272
+
5273
+ class SubprocessRemoteSpawning(RemoteSpawning):
5274
+ class _PreparedCmd(ta.NamedTuple): # noqa
4275
5275
  cmd: ta.Sequence[str]
4276
5276
  shell: bool
4277
5277
 
4278
5278
  def _prepare_cmd(
4279
5279
  self,
4280
- tgt: Target,
5280
+ tgt: RemoteSpawning.Target,
4281
5281
  src: str,
4282
5282
  ) -> _PreparedCmd:
4283
5283
  if tgt.shell is not None:
@@ -4285,44 +5285,38 @@ class RemoteSpawning:
4285
5285
  if tgt.shell_quote:
4286
5286
  sh_src = shlex.quote(sh_src)
4287
5287
  sh_cmd = f'{tgt.shell} {sh_src}'
4288
- return RemoteSpawning._PreparedCmd(
4289
- cmd=[sh_cmd],
4290
- shell=True,
4291
- )
5288
+ return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
4292
5289
 
4293
5290
  else:
4294
- return RemoteSpawning._PreparedCmd(
4295
- cmd=[tgt.python, '-c', src],
4296
- shell=False,
4297
- )
5291
+ return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
4298
5292
 
4299
5293
  #
4300
5294
 
4301
- @dc.dataclass(frozen=True)
4302
- class Spawned:
4303
- stdin: ta.IO
4304
- stdout: ta.IO
4305
- stderr: ta.Optional[ta.IO]
4306
-
4307
- @contextlib.contextmanager
4308
- def spawn(
5295
+ @contextlib.asynccontextmanager
5296
+ async def spawn(
4309
5297
  self,
4310
- tgt: Target,
5298
+ tgt: RemoteSpawning.Target,
4311
5299
  src: str,
4312
5300
  *,
4313
5301
  timeout: ta.Optional[float] = None,
4314
- ) -> ta.Generator[Spawned, None, None]:
5302
+ debug: bool = False,
5303
+ ) -> ta.AsyncGenerator[RemoteSpawning.Spawned, None]:
4315
5304
  pc = self._prepare_cmd(tgt, src)
4316
5305
 
4317
- with subprocess.Popen(
4318
- subprocess_maybe_shell_wrap_exec(*pc.cmd),
4319
- shell=pc.shell,
4320
- stdin=subprocess.PIPE,
4321
- stdout=subprocess.PIPE,
4322
- stderr=(
4323
- SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
4324
- if tgt.stderr is not None else None
4325
- ),
5306
+ cmd = pc.cmd
5307
+ if not debug:
5308
+ cmd = subprocess_maybe_shell_wrap_exec(*cmd)
5309
+
5310
+ async with asyncio_subprocess_popen(
5311
+ *cmd,
5312
+ shell=pc.shell,
5313
+ stdin=subprocess.PIPE,
5314
+ stdout=subprocess.PIPE,
5315
+ stderr=(
5316
+ SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
5317
+ if tgt.stderr is not None else None
5318
+ ),
5319
+ timeout=timeout,
4326
5320
  ) as proc:
4327
5321
  stdin = check_not_none(proc.stdin)
4328
5322
  stdout = check_not_none(proc.stdout)
@@ -4340,8 +5334,6 @@ class RemoteSpawning:
4340
5334
  except BrokenPipeError:
4341
5335
  pass
4342
5336
 
4343
- proc.wait(timeout)
4344
-
4345
5337
 
4346
5338
  ########################################
4347
5339
  # ../../../omdev/interp/providers.py
@@ -4370,17 +5362,17 @@ class InterpProvider(abc.ABC):
4370
5362
  setattr(cls, 'name', snake_case(cls.__name__[:-len(sfx)]))
4371
5363
 
4372
5364
  @abc.abstractmethod
4373
- def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5365
+ def get_installed_versions(self, spec: InterpSpecifier) -> ta.Awaitable[ta.Sequence[InterpVersion]]:
4374
5366
  raise NotImplementedError
4375
5367
 
4376
5368
  @abc.abstractmethod
4377
- def get_installed_version(self, version: InterpVersion) -> Interp:
5369
+ def get_installed_version(self, version: InterpVersion) -> ta.Awaitable[Interp]:
4378
5370
  raise NotImplementedError
4379
5371
 
4380
- def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5372
+ async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
4381
5373
  return []
4382
5374
 
4383
- def install_version(self, version: InterpVersion) -> Interp:
5375
+ async def install_version(self, version: InterpVersion) -> Interp:
4384
5376
  raise TypeError
4385
5377
 
4386
5378
 
@@ -4392,10 +5384,10 @@ class RunningInterpProvider(InterpProvider):
4392
5384
  def version(self) -> InterpVersion:
4393
5385
  return InterpInspector.running().iv
4394
5386
 
4395
- def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5387
+ async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
4396
5388
  return [self.version()]
4397
5389
 
4398
- def get_installed_version(self, version: InterpVersion) -> Interp:
5390
+ async def get_installed_version(self, version: InterpVersion) -> Interp:
4399
5391
  if version != self.version():
4400
5392
  raise KeyError(version)
4401
5393
  return Interp(
@@ -4405,159 +5397,26 @@ class RunningInterpProvider(InterpProvider):
4405
5397
 
4406
5398
 
4407
5399
  ########################################
4408
- # ../remote/execution.py
4409
-
4410
-
4411
- ##
4412
-
4413
-
4414
- class _RemoteExecutionLogHandler(logging.Handler):
4415
- def __init__(self, fn: ta.Callable[[str], None]) -> None:
4416
- super().__init__()
4417
- self._fn = fn
4418
-
4419
- def emit(self, record):
4420
- msg = self.format(record)
4421
- self._fn(msg)
4422
-
4423
-
4424
- @dc.dataclass(frozen=True)
4425
- class _RemoteExecutionRequest:
4426
- c: Command
4427
-
4428
-
4429
- @dc.dataclass(frozen=True)
4430
- class _RemoteExecutionLog:
4431
- s: str
4432
-
4433
-
4434
- @dc.dataclass(frozen=True)
4435
- class _RemoteExecutionResponse:
4436
- r: ta.Optional[CommandOutputOrExceptionData] = None
4437
- l: ta.Optional[_RemoteExecutionLog] = None
4438
-
4439
-
4440
- def _remote_execution_main() -> None:
4441
- rt = pyremote_bootstrap_finalize() # noqa
4442
-
4443
- chan = RemoteChannel(
4444
- rt.input,
4445
- rt.output,
4446
- )
4447
-
4448
- bs = check_not_none(chan.recv_obj(MainBootstrap))
4449
-
4450
- if (prd := bs.remote_config.pycharm_remote_debug) is not None:
4451
- pycharm_debug_connect(prd)
4452
-
4453
- injector = main_bootstrap(bs)
4454
-
4455
- chan.set_marshaler(injector[ObjMarshalerManager])
4456
-
4457
- #
4458
-
4459
- log_lock = threading.RLock()
4460
- send_logs = False
4461
-
4462
- def log_fn(s: str) -> None:
4463
- with log_lock:
4464
- if send_logs:
4465
- chan.send_obj(_RemoteExecutionResponse(l=_RemoteExecutionLog(s)))
4466
-
4467
- log_handler = _RemoteExecutionLogHandler(log_fn)
4468
- logging.root.addHandler(log_handler)
4469
-
4470
- #
4471
-
4472
- ce = injector[LocalCommandExecutor]
4473
-
4474
- while True:
4475
- req = chan.recv_obj(_RemoteExecutionRequest)
4476
- if req is None:
4477
- break
4478
-
4479
- with log_lock:
4480
- send_logs = True
4481
-
4482
- r = ce.try_execute(
4483
- req.c,
4484
- log=log,
4485
- omit_exc_object=True,
4486
- )
4487
-
4488
- with log_lock:
4489
- send_logs = False
4490
-
4491
- chan.send_obj(_RemoteExecutionResponse(r=CommandOutputOrExceptionData(
4492
- output=r.output,
4493
- exception=r.exception,
4494
- )))
5400
+ # ../remote/connection.py
4495
5401
 
4496
5402
 
4497
5403
  ##
4498
5404
 
4499
5405
 
4500
- @dc.dataclass()
4501
- class RemoteCommandError(Exception):
4502
- e: CommandException
4503
-
4504
-
4505
- class RemoteCommandExecutor(CommandExecutor):
4506
- def __init__(self, chan: RemoteChannel) -> None:
4507
- super().__init__()
4508
-
4509
- self._chan = chan
4510
-
4511
- def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
4512
- self._chan.send_obj(_RemoteExecutionRequest(cmd))
4513
-
4514
- while True:
4515
- if (r := self._chan.recv_obj(_RemoteExecutionResponse)) is None:
4516
- raise EOFError
4517
-
4518
- if r.l is not None:
4519
- log.info(r.l.s)
4520
-
4521
- if r.r is not None:
4522
- return r.r
4523
-
4524
- # @ta.override
4525
- def execute(self, cmd: Command) -> Command.Output:
4526
- r = self._remote_execute(cmd)
4527
- if (e := r.exception) is not None:
4528
- raise RemoteCommandError(e)
4529
- else:
4530
- return check_not_none(r.output)
4531
-
4532
- # @ta.override
4533
- def try_execute(
5406
+ class RemoteExecutionConnector(abc.ABC):
5407
+ @abc.abstractmethod
5408
+ def connect(
4534
5409
  self,
4535
- cmd: Command,
4536
- *,
4537
- log: ta.Optional[logging.Logger] = None,
4538
- omit_exc_object: bool = False,
4539
- ) -> CommandOutputOrException:
4540
- try:
4541
- r = self._remote_execute(cmd)
4542
-
4543
- except Exception as e: # noqa
4544
- if log is not None:
4545
- log.exception('Exception executing remote command: %r', type(cmd))
4546
-
4547
- return CommandOutputOrExceptionData(exception=CommandException.of(
4548
- e,
4549
- omit_exc_object=omit_exc_object,
4550
- cmd=cmd,
4551
- ))
4552
-
4553
- else:
4554
- return r
5410
+ tgt: RemoteSpawning.Target,
5411
+ bs: MainBootstrap,
5412
+ ) -> ta.AsyncContextManager[RemoteCommandExecutor]:
5413
+ raise NotImplementedError
4555
5414
 
4556
5415
 
4557
5416
  ##
4558
5417
 
4559
5418
 
4560
- class RemoteExecution:
5419
+ class PyremoteRemoteExecutionConnector(RemoteExecutionConnector):
4561
5420
  def __init__(
4562
5421
  self,
4563
5422
  *,
@@ -4590,38 +5449,43 @@ class RemoteExecution:
4590
5449
 
4591
5450
  #
4592
5451
 
4593
- @contextlib.contextmanager
4594
- def connect(
5452
+ @contextlib.asynccontextmanager
5453
+ async def connect(
4595
5454
  self,
4596
5455
  tgt: RemoteSpawning.Target,
4597
5456
  bs: MainBootstrap,
4598
- ) -> ta.Generator[RemoteCommandExecutor, None, None]:
5457
+ ) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
4599
5458
  spawn_src = self._spawn_src()
4600
5459
  remote_src = self._remote_src()
4601
5460
 
4602
- with self._spawning.spawn(
5461
+ async with self._spawning.spawn(
4603
5462
  tgt,
4604
5463
  spawn_src,
5464
+ debug=bs.main_config.debug,
4605
5465
  ) as proc:
4606
- res = PyremoteBootstrapDriver( # noqa
5466
+ res = await PyremoteBootstrapDriver( # noqa
4607
5467
  remote_src,
4608
5468
  PyremoteBootstrapOptions(
4609
5469
  debug=bs.main_config.debug,
4610
5470
  ),
4611
- ).run(
5471
+ ).async_run(
4612
5472
  proc.stdout,
4613
5473
  proc.stdin,
4614
5474
  )
4615
5475
 
4616
- chan = RemoteChannel(
5476
+ chan = RemoteChannelImpl(
4617
5477
  proc.stdout,
4618
5478
  proc.stdin,
4619
5479
  msh=self._msh,
4620
5480
  )
4621
5481
 
4622
- chan.send_obj(bs)
5482
+ await chan.send_obj(bs)
5483
+
5484
+ rce: RemoteCommandExecutor
5485
+ async with contextlib.aclosing(RemoteCommandExecutor(chan)) as rce:
5486
+ await rce.start()
4623
5487
 
4624
- yield RemoteCommandExecutor(chan)
5488
+ yield rce
4625
5489
 
4626
5490
 
4627
5491
  ########################################
@@ -4643,7 +5507,6 @@ TODO:
4643
5507
 
4644
5508
 
4645
5509
  class Pyenv:
4646
-
4647
5510
  def __init__(
4648
5511
  self,
4649
5512
  *,
@@ -4656,13 +5519,13 @@ class Pyenv:
4656
5519
 
4657
5520
  self._root_kw = root
4658
5521
 
4659
- @cached_nullary
4660
- def root(self) -> ta.Optional[str]:
5522
+ @async_cached_nullary
5523
+ async def root(self) -> ta.Optional[str]:
4661
5524
  if self._root_kw is not None:
4662
5525
  return self._root_kw
4663
5526
 
4664
5527
  if shutil.which('pyenv'):
4665
- return subprocess_check_output_str('pyenv', 'root')
5528
+ return await asyncio_subprocess_check_output_str('pyenv', 'root')
4666
5529
 
4667
5530
  d = os.path.expanduser('~/.pyenv')
4668
5531
  if os.path.isdir(d) and os.path.isfile(os.path.join(d, 'bin', 'pyenv')):
@@ -4670,12 +5533,12 @@ class Pyenv:
4670
5533
 
4671
5534
  return None
4672
5535
 
4673
- @cached_nullary
4674
- def exe(self) -> str:
4675
- return os.path.join(check_not_none(self.root()), 'bin', 'pyenv')
5536
+ @async_cached_nullary
5537
+ async def exe(self) -> str:
5538
+ return os.path.join(check_not_none(await self.root()), 'bin', 'pyenv')
4676
5539
 
4677
- def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
4678
- if (root := self.root()) is None:
5540
+ async def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
5541
+ if (root := await self.root()) is None:
4679
5542
  return []
4680
5543
  ret = []
4681
5544
  vp = os.path.join(root, 'versions')
@@ -4687,11 +5550,11 @@ class Pyenv:
4687
5550
  ret.append((dn, ep))
4688
5551
  return ret
4689
5552
 
4690
- def installable_versions(self) -> ta.List[str]:
4691
- if self.root() is None:
5553
+ async def installable_versions(self) -> ta.List[str]:
5554
+ if await self.root() is None:
4692
5555
  return []
4693
5556
  ret = []
4694
- s = subprocess_check_output_str(self.exe(), 'install', '--list')
5557
+ s = await asyncio_subprocess_check_output_str(await self.exe(), 'install', '--list')
4695
5558
  for l in s.splitlines():
4696
5559
  if not l.startswith(' '):
4697
5560
  continue
@@ -4701,12 +5564,12 @@ class Pyenv:
4701
5564
  ret.append(l)
4702
5565
  return ret
4703
5566
 
4704
- def update(self) -> bool:
4705
- if (root := self.root()) is None:
5567
+ async def update(self) -> bool:
5568
+ if (root := await self.root()) is None:
4706
5569
  return False
4707
5570
  if not os.path.isdir(os.path.join(root, '.git')):
4708
5571
  return False
4709
- subprocess_check_call('git', 'pull', cwd=root)
5572
+ await asyncio_subprocess_check_call('git', 'pull', cwd=root)
4710
5573
  return True
4711
5574
 
4712
5575
 
@@ -4767,17 +5630,16 @@ THREADED_PYENV_INSTALL_OPTS = PyenvInstallOpts(conf_opts=['--disable-gil'])
4767
5630
 
4768
5631
  class PyenvInstallOptsProvider(abc.ABC):
4769
5632
  @abc.abstractmethod
4770
- def opts(self) -> PyenvInstallOpts:
5633
+ def opts(self) -> ta.Awaitable[PyenvInstallOpts]:
4771
5634
  raise NotImplementedError
4772
5635
 
4773
5636
 
4774
5637
  class LinuxPyenvInstallOpts(PyenvInstallOptsProvider):
4775
- def opts(self) -> PyenvInstallOpts:
5638
+ async def opts(self) -> PyenvInstallOpts:
4776
5639
  return PyenvInstallOpts()
4777
5640
 
4778
5641
 
4779
5642
  class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
4780
-
4781
5643
  @cached_nullary
4782
5644
  def framework_opts(self) -> PyenvInstallOpts:
4783
5645
  return PyenvInstallOpts(conf_opts=['--enable-framework'])
@@ -4793,12 +5655,12 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
4793
5655
  'zlib',
4794
5656
  ]
4795
5657
 
4796
- @cached_nullary
4797
- def brew_deps_opts(self) -> PyenvInstallOpts:
5658
+ @async_cached_nullary
5659
+ async def brew_deps_opts(self) -> PyenvInstallOpts:
4798
5660
  cflags = []
4799
5661
  ldflags = []
4800
5662
  for dep in self.BREW_DEPS:
4801
- dep_prefix = subprocess_check_output_str('brew', '--prefix', dep)
5663
+ dep_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', dep)
4802
5664
  cflags.append(f'-I{dep_prefix}/include')
4803
5665
  ldflags.append(f'-L{dep_prefix}/lib')
4804
5666
  return PyenvInstallOpts(
@@ -4806,13 +5668,13 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
4806
5668
  ldflags=ldflags,
4807
5669
  )
4808
5670
 
4809
- @cached_nullary
4810
- def brew_tcl_opts(self) -> PyenvInstallOpts:
4811
- if subprocess_try_output('brew', '--prefix', 'tcl-tk') is None:
5671
+ @async_cached_nullary
5672
+ async def brew_tcl_opts(self) -> PyenvInstallOpts:
5673
+ if await asyncio_subprocess_try_output('brew', '--prefix', 'tcl-tk') is None:
4812
5674
  return PyenvInstallOpts()
4813
5675
 
4814
- tcl_tk_prefix = subprocess_check_output_str('brew', '--prefix', 'tcl-tk')
4815
- tcl_tk_ver_str = subprocess_check_output_str('brew', 'ls', '--versions', 'tcl-tk')
5676
+ tcl_tk_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', 'tcl-tk')
5677
+ tcl_tk_ver_str = await asyncio_subprocess_check_output_str('brew', 'ls', '--versions', 'tcl-tk')
4816
5678
  tcl_tk_ver = '.'.join(tcl_tk_ver_str.split()[1].split('.')[:2])
4817
5679
 
4818
5680
  return PyenvInstallOpts(conf_opts=[
@@ -4827,11 +5689,11 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
4827
5689
  # pkg_config_path += ':' + os.environ['PKG_CONFIG_PATH']
4828
5690
  # return PyenvInstallOpts(env={'PKG_CONFIG_PATH': pkg_config_path})
4829
5691
 
4830
- def opts(self) -> PyenvInstallOpts:
5692
+ async def opts(self) -> PyenvInstallOpts:
4831
5693
  return PyenvInstallOpts().merge(
4832
5694
  self.framework_opts(),
4833
- self.brew_deps_opts(),
4834
- self.brew_tcl_opts(),
5695
+ await self.brew_deps_opts(),
5696
+ await self.brew_tcl_opts(),
4835
5697
  # self.brew_ssl_opts(),
4836
5698
  )
4837
5699
 
@@ -4863,20 +5725,8 @@ class PyenvVersionInstaller:
4863
5725
  ) -> None:
4864
5726
  super().__init__()
4865
5727
 
4866
- if no_default_opts:
4867
- if opts is None:
4868
- opts = PyenvInstallOpts()
4869
- else:
4870
- lst = [opts if opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
4871
- if interp_opts.debug:
4872
- lst.append(DEBUG_PYENV_INSTALL_OPTS)
4873
- if interp_opts.threaded:
4874
- lst.append(THREADED_PYENV_INSTALL_OPTS)
4875
- lst.append(PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
4876
- opts = PyenvInstallOpts().merge(*lst)
4877
-
4878
5728
  self._version = version
4879
- self._opts = opts
5729
+ self._given_opts = opts
4880
5730
  self._interp_opts = interp_opts
4881
5731
  self._given_install_name = install_name
4882
5732
 
@@ -4887,9 +5737,21 @@ class PyenvVersionInstaller:
4887
5737
  def version(self) -> str:
4888
5738
  return self._version
4889
5739
 
4890
- @property
4891
- def opts(self) -> PyenvInstallOpts:
4892
- return self._opts
5740
+ @async_cached_nullary
5741
+ async def opts(self) -> PyenvInstallOpts:
5742
+ opts = self._given_opts
5743
+ if self._no_default_opts:
5744
+ if opts is None:
5745
+ opts = PyenvInstallOpts()
5746
+ else:
5747
+ lst = [self._given_opts if self._given_opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
5748
+ if self._interp_opts.debug:
5749
+ lst.append(DEBUG_PYENV_INSTALL_OPTS)
5750
+ if self._interp_opts.threaded:
5751
+ lst.append(THREADED_PYENV_INSTALL_OPTS)
5752
+ lst.append(await PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
5753
+ opts = PyenvInstallOpts().merge(*lst)
5754
+ return opts
4893
5755
 
4894
5756
  @cached_nullary
4895
5757
  def install_name(self) -> str:
@@ -4897,17 +5759,18 @@ class PyenvVersionInstaller:
4897
5759
  return self._given_install_name
4898
5760
  return self._version + ('-debug' if self._interp_opts.debug else '')
4899
5761
 
4900
- @cached_nullary
4901
- def install_dir(self) -> str:
4902
- return str(os.path.join(check_not_none(self._pyenv.root()), 'versions', self.install_name()))
5762
+ @async_cached_nullary
5763
+ async def install_dir(self) -> str:
5764
+ return str(os.path.join(check_not_none(await self._pyenv.root()), 'versions', self.install_name()))
4903
5765
 
4904
- @cached_nullary
4905
- def install(self) -> str:
4906
- env = {**os.environ, **self._opts.env}
5766
+ @async_cached_nullary
5767
+ async def install(self) -> str:
5768
+ opts = await self.opts()
5769
+ env = {**os.environ, **opts.env}
4907
5770
  for k, l in [
4908
- ('CFLAGS', self._opts.cflags),
4909
- ('LDFLAGS', self._opts.ldflags),
4910
- ('PYTHON_CONFIGURE_OPTS', self._opts.conf_opts),
5771
+ ('CFLAGS', opts.cflags),
5772
+ ('LDFLAGS', opts.ldflags),
5773
+ ('PYTHON_CONFIGURE_OPTS', opts.conf_opts),
4911
5774
  ]:
4912
5775
  v = ' '.join(l)
4913
5776
  if k in os.environ:
@@ -4915,13 +5778,13 @@ class PyenvVersionInstaller:
4915
5778
  env[k] = v
4916
5779
 
4917
5780
  conf_args = [
4918
- *self._opts.opts,
5781
+ *opts.opts,
4919
5782
  self._version,
4920
5783
  ]
4921
5784
 
4922
5785
  if self._given_install_name is not None:
4923
5786
  full_args = [
4924
- os.path.join(check_not_none(self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'),
5787
+ os.path.join(check_not_none(await self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'), # noqa
4925
5788
  *conf_args,
4926
5789
  self.install_dir(),
4927
5790
  ]
@@ -4932,12 +5795,12 @@ class PyenvVersionInstaller:
4932
5795
  *conf_args,
4933
5796
  ]
4934
5797
 
4935
- subprocess_check_call(
5798
+ await asyncio_subprocess_check_call(
4936
5799
  *full_args,
4937
5800
  env=env,
4938
5801
  )
4939
5802
 
4940
- exe = os.path.join(self.install_dir(), 'bin', 'python')
5803
+ exe = os.path.join(await self.install_dir(), 'bin', 'python')
4941
5804
  if not os.path.isfile(exe):
4942
5805
  raise RuntimeError(f'Interpreter not found: {exe}')
4943
5806
  return exe
@@ -4947,7 +5810,6 @@ class PyenvVersionInstaller:
4947
5810
 
4948
5811
 
4949
5812
  class PyenvInterpProvider(InterpProvider):
4950
-
4951
5813
  def __init__(
4952
5814
  self,
4953
5815
  pyenv: Pyenv = Pyenv(),
@@ -4990,11 +5852,11 @@ class PyenvInterpProvider(InterpProvider):
4990
5852
  exe: str
4991
5853
  version: InterpVersion
4992
5854
 
4993
- def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
5855
+ async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
4994
5856
  iv: ta.Optional[InterpVersion]
4995
5857
  if self._inspect:
4996
5858
  try:
4997
- iv = check_not_none(self._inspector.inspect(ep)).iv
5859
+ iv = check_not_none(await self._inspector.inspect(ep)).iv
4998
5860
  except Exception as e: # noqa
4999
5861
  return None
5000
5862
  else:
@@ -5007,10 +5869,10 @@ class PyenvInterpProvider(InterpProvider):
5007
5869
  version=iv,
5008
5870
  )
5009
5871
 
5010
- def installed(self) -> ta.Sequence[Installed]:
5872
+ async def installed(self) -> ta.Sequence[Installed]:
5011
5873
  ret: ta.List[PyenvInterpProvider.Installed] = []
5012
- for vn, ep in self._pyenv.version_exes():
5013
- if (i := self._make_installed(vn, ep)) is None:
5874
+ for vn, ep in await self._pyenv.version_exes():
5875
+ if (i := await self._make_installed(vn, ep)) is None:
5014
5876
  log.debug('Invalid pyenv version: %s', vn)
5015
5877
  continue
5016
5878
  ret.append(i)
@@ -5018,11 +5880,11 @@ class PyenvInterpProvider(InterpProvider):
5018
5880
 
5019
5881
  #
5020
5882
 
5021
- def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5022
- return [i.version for i in self.installed()]
5883
+ async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5884
+ return [i.version for i in await self.installed()]
5023
5885
 
5024
- def get_installed_version(self, version: InterpVersion) -> Interp:
5025
- for i in self.installed():
5886
+ async def get_installed_version(self, version: InterpVersion) -> Interp:
5887
+ for i in await self.installed():
5026
5888
  if i.version == version:
5027
5889
  return Interp(
5028
5890
  exe=i.exe,
@@ -5032,10 +5894,10 @@ class PyenvInterpProvider(InterpProvider):
5032
5894
 
5033
5895
  #
5034
5896
 
5035
- def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5897
+ async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5036
5898
  lst = []
5037
5899
 
5038
- for vs in self._pyenv.installable_versions():
5900
+ for vs in await self._pyenv.installable_versions():
5039
5901
  if (iv := self.guess_version(vs)) is None:
5040
5902
  continue
5041
5903
  if iv.opts.debug:
@@ -5045,16 +5907,16 @@ class PyenvInterpProvider(InterpProvider):
5045
5907
 
5046
5908
  return lst
5047
5909
 
5048
- def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5049
- lst = self._get_installable_versions(spec)
5910
+ async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5911
+ lst = await self._get_installable_versions(spec)
5050
5912
 
5051
5913
  if self._try_update and not any(v in spec for v in lst):
5052
5914
  if self._pyenv.update():
5053
- lst = self._get_installable_versions(spec)
5915
+ lst = await self._get_installable_versions(spec)
5054
5916
 
5055
5917
  return lst
5056
5918
 
5057
- def install_version(self, version: InterpVersion) -> Interp:
5919
+ async def install_version(self, version: InterpVersion) -> Interp:
5058
5920
  inst_version = str(version.version)
5059
5921
  inst_opts = version.opts
5060
5922
  if inst_opts.threaded:
@@ -5066,7 +5928,7 @@ class PyenvInterpProvider(InterpProvider):
5066
5928
  interp_opts=inst_opts,
5067
5929
  )
5068
5930
 
5069
- exe = installer.install()
5931
+ exe = await installer.install()
5070
5932
  return Interp(exe, version)
5071
5933
 
5072
5934
 
@@ -5145,7 +6007,7 @@ class SystemInterpProvider(InterpProvider):
5145
6007
 
5146
6008
  #
5147
6009
 
5148
- def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
6010
+ async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
5149
6011
  if not self.inspect:
5150
6012
  s = os.path.basename(exe)
5151
6013
  if s.startswith('python'):
@@ -5155,13 +6017,13 @@ class SystemInterpProvider(InterpProvider):
5155
6017
  return InterpVersion.parse(s)
5156
6018
  except InvalidVersion:
5157
6019
  pass
5158
- ii = self.inspector.inspect(exe)
6020
+ ii = await self.inspector.inspect(exe)
5159
6021
  return ii.iv if ii is not None else None
5160
6022
 
5161
- def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
6023
+ async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
5162
6024
  lst = []
5163
6025
  for e in self.exes():
5164
- if (ev := self.get_exe_version(e)) is None:
6026
+ if (ev := await self.get_exe_version(e)) is None:
5165
6027
  log.debug('Invalid system version: %s', e)
5166
6028
  continue
5167
6029
  lst.append((e, ev))
@@ -5169,11 +6031,11 @@ class SystemInterpProvider(InterpProvider):
5169
6031
 
5170
6032
  #
5171
6033
 
5172
- def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5173
- return [ev for e, ev in self.exe_versions()]
6034
+ async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
6035
+ return [ev for e, ev in await self.exe_versions()]
5174
6036
 
5175
- def get_installed_version(self, version: InterpVersion) -> Interp:
5176
- for e, ev in self.exe_versions():
6037
+ async def get_installed_version(self, version: InterpVersion) -> Interp:
6038
+ for e, ev in await self.exe_versions():
5177
6039
  if ev != version:
5178
6040
  continue
5179
6041
  return Interp(
@@ -5194,9 +6056,11 @@ def bind_remote(
5194
6056
  lst: ta.List[InjectorBindingOrBindings] = [
5195
6057
  inj.bind(remote_config),
5196
6058
 
5197
- inj.bind(RemoteSpawning, singleton=True),
6059
+ inj.bind(SubprocessRemoteSpawning, singleton=True),
6060
+ inj.bind(RemoteSpawning, to_key=SubprocessRemoteSpawning),
5198
6061
 
5199
- inj.bind(RemoteExecution, singleton=True),
6062
+ inj.bind(PyremoteRemoteExecutionConnector, singleton=True),
6063
+ inj.bind(RemoteExecutionConnector, to_key=PyremoteRemoteExecutionConnector),
5200
6064
  ]
5201
6065
 
5202
6066
  if (pf := remote_config.payload_file) is not None:
@@ -5220,13 +6084,14 @@ class InterpResolver:
5220
6084
  providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
5221
6085
  ) -> None:
5222
6086
  super().__init__()
6087
+
5223
6088
  self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
5224
6089
 
5225
- def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
6090
+ async def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
5226
6091
  lst = [
5227
6092
  (i, si)
5228
6093
  for i, p in enumerate(self._providers.values())
5229
- for si in p.get_installed_versions(spec)
6094
+ for si in await p.get_installed_versions(spec)
5230
6095
  if spec.contains(si)
5231
6096
  ]
5232
6097
 
@@ -5238,16 +6103,16 @@ class InterpResolver:
5238
6103
  bp = list(self._providers.values())[bi]
5239
6104
  return (bp, bv)
5240
6105
 
5241
- def resolve(
6106
+ async def resolve(
5242
6107
  self,
5243
6108
  spec: InterpSpecifier,
5244
6109
  *,
5245
6110
  install: bool = False,
5246
6111
  ) -> ta.Optional[Interp]:
5247
- tup = self._resolve_installed(spec)
6112
+ tup = await self._resolve_installed(spec)
5248
6113
  if tup is not None:
5249
6114
  bp, bv = tup
5250
- return bp.get_installed_version(bv)
6115
+ return await bp.get_installed_version(bv)
5251
6116
 
5252
6117
  if not install:
5253
6118
  return None
@@ -5255,21 +6120,21 @@ class InterpResolver:
5255
6120
  tp = list(self._providers.values())[0] # noqa
5256
6121
 
5257
6122
  sv = sorted(
5258
- [s for s in tp.get_installable_versions(spec) if s in spec],
6123
+ [s for s in await tp.get_installable_versions(spec) if s in spec],
5259
6124
  key=lambda s: s.version,
5260
6125
  )
5261
6126
  if not sv:
5262
6127
  return None
5263
6128
 
5264
6129
  bv = sv[-1]
5265
- return tp.install_version(bv)
6130
+ return await tp.install_version(bv)
5266
6131
 
5267
- def list(self, spec: InterpSpecifier) -> None:
6132
+ async def list(self, spec: InterpSpecifier) -> None:
5268
6133
  print('installed:')
5269
6134
  for n, p in self._providers.items():
5270
6135
  lst = [
5271
6136
  si
5272
- for si in p.get_installed_versions(spec)
6137
+ for si in await p.get_installed_versions(spec)
5273
6138
  if spec.contains(si)
5274
6139
  ]
5275
6140
  if lst:
@@ -5283,7 +6148,7 @@ class InterpResolver:
5283
6148
  for n, p in self._providers.items():
5284
6149
  lst = [
5285
6150
  si
5286
- for si in p.get_installable_versions(spec)
6151
+ for si in await p.get_installable_versions(spec)
5287
6152
  if spec.contains(si)
5288
6153
  ]
5289
6154
  if lst:
@@ -5325,9 +6190,9 @@ class InterpCommand(Command['InterpCommand.Output']):
5325
6190
 
5326
6191
 
5327
6192
  class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
5328
- def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
6193
+ async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
5329
6194
  i = InterpSpecifier.parse(check_not_none(cmd.spec))
5330
- o = check_not_none(DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
6195
+ o = check_not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
5331
6196
  return InterpCommand.Output(
5332
6197
  exe=o.exe,
5333
6198
  version=str(o.version.version),
@@ -5366,7 +6231,7 @@ def bind_command(
5366
6231
  class _FactoryCommandExecutor(CommandExecutor):
5367
6232
  factory: ta.Callable[[], CommandExecutor]
5368
6233
 
5369
- def execute(self, i: Command) -> Command.Output:
6234
+ def execute(self, i: Command) -> ta.Awaitable[Command.Output]:
5370
6235
  return self.factory().execute(i)
5371
6236
 
5372
6237
 
@@ -5521,31 +6386,7 @@ def main_bootstrap(bs: MainBootstrap) -> Injector:
5521
6386
  ##
5522
6387
 
5523
6388
 
5524
- def _main() -> None:
5525
- import argparse
5526
-
5527
- parser = argparse.ArgumentParser()
5528
-
5529
- parser.add_argument('--_payload-file')
5530
-
5531
- parser.add_argument('-s', '--shell')
5532
- parser.add_argument('-q', '--shell-quote', action='store_true')
5533
- parser.add_argument('--python', default='python3')
5534
-
5535
- parser.add_argument('--pycharm-debug-port', type=int)
5536
- parser.add_argument('--pycharm-debug-host')
5537
- parser.add_argument('--pycharm-debug-version')
5538
-
5539
- parser.add_argument('--debug', action='store_true')
5540
-
5541
- parser.add_argument('--local', action='store_true')
5542
-
5543
- parser.add_argument('command', nargs='+')
5544
-
5545
- args = parser.parse_args()
5546
-
5547
- #
5548
-
6389
+ async def _async_main(args: ta.Any) -> None:
5549
6390
  bs = MainBootstrap(
5550
6391
  main_config=MainConfig(
5551
6392
  log_level='DEBUG' if args.debug else 'INFO',
@@ -5558,12 +6399,16 @@ def _main() -> None:
5558
6399
 
5559
6400
  pycharm_remote_debug=PycharmRemoteDebug(
5560
6401
  port=args.pycharm_debug_port,
5561
- host=args.pycharm_debug_host,
6402
+ **(dict(host=args.pycharm_debug_host) if args.pycharm_debug_host is not None else {}),
5562
6403
  install_version=args.pycharm_debug_version,
5563
6404
  ) if args.pycharm_debug_port is not None else None,
6405
+
6406
+ timebomb_delay_s=args.remote_timebomb_delay_s,
5564
6407
  ),
5565
6408
  )
5566
6409
 
6410
+ #
6411
+
5567
6412
  injector = main_bootstrap(
5568
6413
  bs,
5569
6414
  )
@@ -5582,7 +6427,7 @@ def _main() -> None:
5582
6427
 
5583
6428
  #
5584
6429
 
5585
- with contextlib.ExitStack() as es:
6430
+ async with contextlib.AsyncExitStack() as es:
5586
6431
  ce: CommandExecutor
5587
6432
 
5588
6433
  if args.local:
@@ -5595,16 +6440,51 @@ def _main() -> None:
5595
6440
  python=args.python,
5596
6441
  )
5597
6442
 
5598
- ce = es.enter_context(injector[RemoteExecution].connect(tgt, bs)) # noqa
6443
+ ce = await es.enter_async_context(injector[RemoteExecutionConnector].connect(tgt, bs)) # noqa
5599
6444
 
5600
- for cmd in cmds:
5601
- r = ce.try_execute(
6445
+ async def run_command(cmd: Command) -> None:
6446
+ res = await ce.try_execute(
5602
6447
  cmd,
5603
6448
  log=log,
5604
6449
  omit_exc_object=True,
5605
6450
  )
5606
6451
 
5607
- print(msh.marshal_obj(r, opts=ObjMarshalOptions(raw_bytes=True)))
6452
+ print(msh.marshal_obj(res, opts=ObjMarshalOptions(raw_bytes=True)))
6453
+
6454
+ await asyncio.gather(*[
6455
+ run_command(cmd)
6456
+ for cmd in cmds
6457
+ ])
6458
+
6459
+
6460
+ def _main() -> None:
6461
+ import argparse
6462
+
6463
+ parser = argparse.ArgumentParser()
6464
+
6465
+ parser.add_argument('--_payload-file')
6466
+
6467
+ parser.add_argument('-s', '--shell')
6468
+ parser.add_argument('-q', '--shell-quote', action='store_true')
6469
+ parser.add_argument('--python', default='python3')
6470
+
6471
+ parser.add_argument('--pycharm-debug-port', type=int)
6472
+ parser.add_argument('--pycharm-debug-host')
6473
+ parser.add_argument('--pycharm-debug-version')
6474
+
6475
+ parser.add_argument('--remote-timebomb-delay-s', type=float)
6476
+
6477
+ parser.add_argument('--debug', action='store_true')
6478
+
6479
+ parser.add_argument('--local', action='store_true')
6480
+
6481
+ parser.add_argument('command', nargs='+')
6482
+
6483
+ args = parser.parse_args()
6484
+
6485
+ #
6486
+
6487
+ asyncio.run(_async_main(args))
5608
6488
 
5609
6489
 
5610
6490
  if __name__ == '__main__':