ominfra 0.0.0.dev147__py3-none-any.whl → 0.0.0.dev149__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. ominfra/clouds/aws/cli.py +1 -1
  2. ominfra/configs.py +30 -5
  3. ominfra/journald/messages.py +1 -1
  4. ominfra/manage/commands/base.py +5 -5
  5. ominfra/manage/commands/execution.py +2 -2
  6. ominfra/manage/commands/inject.py +1 -1
  7. ominfra/manage/commands/interp.py +2 -2
  8. ominfra/manage/commands/subprocess.py +22 -14
  9. ominfra/manage/deploy/command.py +1 -1
  10. ominfra/manage/deploy/paths.py +2 -2
  11. ominfra/manage/main.py +48 -32
  12. ominfra/manage/remote/_main.py +172 -0
  13. ominfra/manage/remote/channel.py +41 -16
  14. ominfra/manage/remote/config.py +10 -0
  15. ominfra/manage/remote/connection.py +106 -0
  16. ominfra/manage/remote/execution.py +244 -155
  17. ominfra/manage/remote/inject.py +7 -3
  18. ominfra/manage/remote/spawning.py +51 -33
  19. ominfra/pyremote.py +28 -3
  20. ominfra/scripts/journald2aws.py +195 -91
  21. ominfra/scripts/manage.py +1366 -486
  22. ominfra/scripts/supervisor.py +533 -479
  23. ominfra/supervisor/dispatchers.py +1 -1
  24. ominfra/supervisor/http.py +2 -2
  25. ominfra/supervisor/inject.py +4 -4
  26. ominfra/supervisor/io.py +1 -1
  27. ominfra/supervisor/spawningimpl.py +1 -1
  28. ominfra/supervisor/supervisor.py +1 -1
  29. ominfra/supervisor/types.py +1 -1
  30. ominfra/tailscale/cli.py +1 -1
  31. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/METADATA +3 -3
  32. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/RECORD +36 -34
  33. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/LICENSE +0 -0
  34. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/WHEEL +0 -0
  35. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/entry_points.txt +0 -0
  36. {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev149.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py CHANGED
@@ -3,16 +3,20 @@
3
3
  # @omlish-lite
4
4
  # @omlish-script
5
5
  # @omlish-amalg-output ../manage/main.py
6
- # ruff: noqa: N802 UP006 UP007 UP036
6
+ # ruff: noqa: N802 TC003 UP006 UP007 UP036
7
7
  """
8
8
  manage.py -s 'docker run -i python:3.12'
9
9
  manage.py -s 'ssh -i /foo/bar.pem foo@bar.baz' -q --python=python3.8
10
10
  """
11
11
  import abc
12
+ import asyncio
13
+ import asyncio.base_subprocess
14
+ import asyncio.subprocess
12
15
  import base64
13
16
  import collections
14
17
  import collections.abc
15
18
  import contextlib
19
+ import ctypes as ct
16
20
  import dataclasses as dc
17
21
  import datetime
18
22
  import decimal
@@ -30,6 +34,7 @@ import pwd
30
34
  import re
31
35
  import shlex
32
36
  import shutil
37
+ import signal
33
38
  import site
34
39
  import struct
35
40
  import subprocess
@@ -62,6 +67,9 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
62
67
  VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
63
68
  VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
64
69
 
70
+ # ../../omlish/lite/asyncio/asyncio.py
71
+ AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
72
+
65
73
  # ../../omlish/lite/cached.py
66
74
  T = ta.TypeVar('T')
67
75
  CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
@@ -822,10 +830,8 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
822
830
  with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
823
831
  main_src = sf.read()
824
832
 
825
- # Restore original argv0
833
+ # Restore vars
826
834
  sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
827
-
828
- # Grab context name
829
835
  context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
830
836
 
831
837
  # Write third ack
@@ -1000,12 +1006,102 @@ class PyremoteBootstrapDriver:
1000
1006
  else:
1001
1007
  raise TypeError(go)
1002
1008
 
1009
+ async def async_run(
1010
+ self,
1011
+ input: ta.Any, # asyncio.StreamWriter # noqa
1012
+ output: ta.Any, # asyncio.StreamReader
1013
+ ) -> Result:
1014
+ gen = self.gen()
1015
+
1016
+ gi: ta.Optional[bytes] = None
1017
+ while True:
1018
+ try:
1019
+ if gi is not None:
1020
+ go = gen.send(gi)
1021
+ else:
1022
+ go = next(gen)
1023
+ except StopIteration as e:
1024
+ return e.value
1025
+
1026
+ if isinstance(go, self.Read):
1027
+ if len(gi := await input.read(go.sz)) != go.sz:
1028
+ raise EOFError
1029
+ elif isinstance(go, self.Write):
1030
+ gi = None
1031
+ output.write(go.d)
1032
+ await output.drain()
1033
+ else:
1034
+ raise TypeError(go)
1035
+
1036
+
1037
+ ########################################
1038
+ # ../../../omlish/lite/asyncio/asyncio.py
1039
+
1040
+
1041
+ ##
1042
+
1043
+
1044
+ ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
1045
+
1046
+
1047
+ async def asyncio_open_stream_reader(
1048
+ f: ta.IO,
1049
+ loop: ta.Any = None,
1050
+ *,
1051
+ limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1052
+ ) -> asyncio.StreamReader:
1053
+ if loop is None:
1054
+ loop = asyncio.get_running_loop()
1055
+
1056
+ reader = asyncio.StreamReader(limit=limit, loop=loop)
1057
+ await loop.connect_read_pipe(
1058
+ lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1059
+ f,
1060
+ )
1061
+
1062
+ return reader
1063
+
1064
+
1065
+ async def asyncio_open_stream_writer(
1066
+ f: ta.IO,
1067
+ loop: ta.Any = None,
1068
+ ) -> asyncio.StreamWriter:
1069
+ if loop is None:
1070
+ loop = asyncio.get_running_loop()
1071
+
1072
+ writer_transport, writer_protocol = await loop.connect_write_pipe(
1073
+ lambda: asyncio.streams.FlowControlMixin(loop=loop),
1074
+ f,
1075
+ )
1076
+
1077
+ return asyncio.streams.StreamWriter(
1078
+ writer_transport,
1079
+ writer_protocol,
1080
+ None,
1081
+ loop,
1082
+ )
1083
+
1084
+
1085
+ ##
1086
+
1087
+
1088
+ def asyncio_maybe_timeout(
1089
+ fut: AwaitableT,
1090
+ timeout: ta.Optional[float] = None,
1091
+ ) -> AwaitableT:
1092
+ if timeout is not None:
1093
+ fut = asyncio.wait_for(fut, timeout) # type: ignore
1094
+ return fut
1095
+
1003
1096
 
1004
1097
  ########################################
1005
1098
  # ../../../omlish/lite/cached.py
1006
1099
 
1007
1100
 
1008
- class _cached_nullary: # noqa
1101
+ ##
1102
+
1103
+
1104
+ class _AbstractCachedNullary:
1009
1105
  def __init__(self, fn):
1010
1106
  super().__init__()
1011
1107
  self._fn = fn
@@ -1013,17 +1109,25 @@ class _cached_nullary: # noqa
1013
1109
  functools.update_wrapper(self, fn)
1014
1110
 
1015
1111
  def __call__(self, *args, **kwargs): # noqa
1016
- if self._value is self._missing:
1017
- self._value = self._fn()
1018
- return self._value
1112
+ raise TypeError
1019
1113
 
1020
1114
  def __get__(self, instance, owner): # noqa
1021
1115
  bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
1022
1116
  return bound
1023
1117
 
1024
1118
 
1119
+ ##
1120
+
1121
+
1122
+ class _CachedNullary(_AbstractCachedNullary):
1123
+ def __call__(self, *args, **kwargs): # noqa
1124
+ if self._value is self._missing:
1125
+ self._value = self._fn()
1126
+ return self._value
1127
+
1128
+
1025
1129
  def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
1026
- return _cached_nullary(fn)
1130
+ return _CachedNullary(fn)
1027
1131
 
1028
1132
 
1029
1133
  def static_init(fn: CallableT) -> CallableT:
@@ -1032,6 +1136,20 @@ def static_init(fn: CallableT) -> CallableT:
1032
1136
  return fn
1033
1137
 
1034
1138
 
1139
+ ##
1140
+
1141
+
1142
+ class _AsyncCachedNullary(_AbstractCachedNullary):
1143
+ async def __call__(self, *args, **kwargs):
1144
+ if self._value is self._missing:
1145
+ self._value = await self._fn()
1146
+ return self._value
1147
+
1148
+
1149
+ def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
1150
+ return _AsyncCachedNullary(fn)
1151
+
1152
+
1035
1153
  ########################################
1036
1154
  # ../../../omlish/lite/check.py
1037
1155
 
@@ -1071,6 +1189,11 @@ def check_non_empty_str(v: ta.Optional[str]) -> str:
1071
1189
  return v
1072
1190
 
1073
1191
 
1192
+ def check_arg(v: bool, msg: str = 'Illegal argument') -> None:
1193
+ if not v:
1194
+ raise ValueError(msg)
1195
+
1196
+
1074
1197
  def check_state(v: bool, msg: str = 'Illegal state') -> None:
1075
1198
  if not v:
1076
1199
  raise ValueError(msg)
@@ -1123,12 +1246,36 @@ def check_empty(v: SizedT) -> SizedT:
1123
1246
  return v
1124
1247
 
1125
1248
 
1126
- def check_non_empty(v: SizedT) -> SizedT:
1249
+ def check_not_empty(v: SizedT) -> SizedT:
1127
1250
  if not len(v):
1128
1251
  raise ValueError(v)
1129
1252
  return v
1130
1253
 
1131
1254
 
1255
+ ########################################
1256
+ # ../../../omlish/lite/deathsig.py
1257
+
1258
+
1259
+ LINUX_PR_SET_PDEATHSIG = 1 # Second arg is a signal
1260
+ LINUX_PR_GET_PDEATHSIG = 2 # Second arg is a ptr to return the signal
1261
+
1262
+
1263
+ def set_process_deathsig(sig: int) -> bool:
1264
+ if sys.platform == 'linux':
1265
+ libc = ct.CDLL('libc.so.6')
1266
+
1267
+ # int prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5);
1268
+ libc.prctl.restype = ct.c_int
1269
+ libc.prctl.argtypes = [ct.c_int, ct.c_ulong, ct.c_ulong, ct.c_ulong, ct.c_ulong]
1270
+
1271
+ libc.prctl(LINUX_PR_SET_PDEATHSIG, sig, 0, 0, 0, 0)
1272
+
1273
+ return True
1274
+
1275
+ else:
1276
+ return False
1277
+
1278
+
1132
1279
  ########################################
1133
1280
  # ../../../omlish/lite/json.py
1134
1281
 
@@ -1910,8 +2057,8 @@ class Command(abc.ABC, ta.Generic[CommandOutputT]):
1910
2057
  pass
1911
2058
 
1912
2059
  @ta.final
1913
- def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
1914
- return check_isinstance(executor.execute(self), self.Output) # type: ignore[return-value]
2060
+ async def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
2061
+ return check_isinstance(await executor.execute(self), self.Output) # type: ignore[return-value]
1915
2062
 
1916
2063
 
1917
2064
  ##
@@ -1972,10 +2119,10 @@ class CommandOutputOrExceptionData(CommandOutputOrException):
1972
2119
 
1973
2120
  class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
1974
2121
  @abc.abstractmethod
1975
- def execute(self, cmd: CommandT) -> CommandOutputT:
2122
+ def execute(self, cmd: CommandT) -> ta.Awaitable[CommandOutputT]:
1976
2123
  raise NotImplementedError
1977
2124
 
1978
- def try_execute(
2125
+ async def try_execute(
1979
2126
  self,
1980
2127
  cmd: CommandT,
1981
2128
  *,
@@ -1983,7 +2130,7 @@ class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
1983
2130
  omit_exc_object: bool = False,
1984
2131
  ) -> CommandOutputOrException[CommandOutputT]:
1985
2132
  try:
1986
- o = self.execute(cmd)
2133
+ o = await self.execute(cmd)
1987
2134
 
1988
2135
  except Exception as e: # noqa
1989
2136
  if log is not None:
@@ -2054,8 +2201,18 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
2054
2201
  class RemoteConfig:
2055
2202
  payload_file: ta.Optional[str] = None
2056
2203
 
2204
+ set_pgid: bool = True
2205
+
2206
+ deathsig: ta.Optional[str] = 'KILL'
2207
+
2057
2208
  pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
2058
2209
 
2210
+ forward_logging: bool = True
2211
+
2212
+ timebomb_delay_s: ta.Optional[float] = 60 * 60.
2213
+
2214
+ heartbeat_interval_s: float = 3.
2215
+
2059
2216
 
2060
2217
  ########################################
2061
2218
  # ../remote/payload.py
@@ -3788,6 +3945,8 @@ class InterpSpecifier:
3788
3945
  s, o = InterpOpts.parse_suffix(s)
3789
3946
  if not any(s.startswith(o) for o in Specifier.OPERATORS):
3790
3947
  s = '~=' + s
3948
+ if s.count('.') < 2:
3949
+ s += '.0'
3791
3950
  return cls(
3792
3951
  specifier=Specifier(s),
3793
3952
  opts=o,
@@ -3834,9 +3993,9 @@ class LocalCommandExecutor(CommandExecutor):
3834
3993
 
3835
3994
  self._command_executors = command_executors
3836
3995
 
3837
- def execute(self, cmd: Command) -> Command.Output:
3996
+ async def execute(self, cmd: Command) -> Command.Output:
3838
3997
  ce: CommandExecutor = self._command_executors[type(cmd)]
3839
- return ce.execute(cmd)
3998
+ return await ce.execute(cmd)
3840
3999
 
3841
4000
 
3842
4001
  ########################################
@@ -3882,7 +4041,7 @@ class DeployCommand(Command['DeployCommand.Output']):
3882
4041
 
3883
4042
 
3884
4043
  class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
3885
- def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
4044
+ async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
3886
4045
  log.info('Deploying!')
3887
4046
 
3888
4047
  return DeployCommand.Output()
@@ -3904,11 +4063,30 @@ ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMar
3904
4063
  # ../remote/channel.py
3905
4064
 
3906
4065
 
3907
- class RemoteChannel:
4066
+ ##
4067
+
4068
+
4069
+ class RemoteChannel(abc.ABC):
4070
+ @abc.abstractmethod
4071
+ def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
4072
+ raise NotImplementedError
4073
+
4074
+ @abc.abstractmethod
4075
+ def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
4076
+ raise NotImplementedError
4077
+
4078
+ def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
4079
+ pass
4080
+
4081
+
4082
+ ##
4083
+
4084
+
4085
+ class RemoteChannelImpl(RemoteChannel):
3908
4086
  def __init__(
3909
4087
  self,
3910
- input: ta.IO, # noqa
3911
- output: ta.IO,
4088
+ input: asyncio.StreamReader, # noqa
4089
+ output: asyncio.StreamWriter,
3912
4090
  *,
3913
4091
  msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
3914
4092
  ) -> None:
@@ -3918,41 +4096,46 @@ class RemoteChannel:
3918
4096
  self._output = output
3919
4097
  self._msh = msh
3920
4098
 
3921
- self._lock = threading.RLock()
4099
+ self._input_lock = asyncio.Lock()
4100
+ self._output_lock = asyncio.Lock()
3922
4101
 
3923
4102
  def set_marshaler(self, msh: ObjMarshalerManager) -> None:
3924
4103
  self._msh = msh
3925
4104
 
3926
- def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
4105
+ #
4106
+
4107
+ async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
3927
4108
  j = json_dumps_compact(self._msh.marshal_obj(o, ty))
3928
4109
  d = j.encode('utf-8')
3929
4110
 
3930
4111
  self._output.write(struct.pack('<I', len(d)))
3931
4112
  self._output.write(d)
3932
- self._output.flush()
4113
+ await self._output.drain()
3933
4114
 
3934
- def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
3935
- with self._lock:
3936
- return self._send_obj(o, ty)
4115
+ async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
4116
+ async with self._output_lock:
4117
+ return await self._send_obj(o, ty)
4118
+
4119
+ #
3937
4120
 
3938
- def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
3939
- d = self._input.read(4)
4121
+ async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
4122
+ d = await self._input.read(4)
3940
4123
  if not d:
3941
4124
  return None
3942
4125
  if len(d) != 4:
3943
4126
  raise EOFError
3944
4127
 
3945
4128
  sz = struct.unpack('<I', d)[0]
3946
- d = self._input.read(sz)
4129
+ d = await self._input.read(sz)
3947
4130
  if len(d) != sz:
3948
4131
  raise EOFError
3949
4132
 
3950
4133
  j = json.loads(d.decode('utf-8'))
3951
4134
  return self._msh.unmarshal_obj(j, ty)
3952
4135
 
3953
- def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
3954
- with self._lock:
3955
- return self._recv_obj(ty)
4136
+ async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
4137
+ async with self._input_lock:
4138
+ return await self._recv_obj(ty)
3956
4139
 
3957
4140
 
3958
4141
  ########################################
@@ -3986,7 +4169,7 @@ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
3986
4169
  return args
3987
4170
 
3988
4171
 
3989
- def _prepare_subprocess_invocation(
4172
+ def prepare_subprocess_invocation(
3990
4173
  *args: str,
3991
4174
  env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
3992
4175
  extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
@@ -3994,9 +4177,9 @@ def _prepare_subprocess_invocation(
3994
4177
  shell: bool = False,
3995
4178
  **kwargs: ta.Any,
3996
4179
  ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
3997
- log.debug(args)
4180
+ log.debug('prepare_subprocess_invocation: args=%r', args)
3998
4181
  if extra_env:
3999
- log.debug(extra_env)
4182
+ log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
4000
4183
 
4001
4184
  if extra_env:
4002
4185
  env = {**(env if env is not None else os.environ), **extra_env}
@@ -4015,14 +4198,46 @@ def _prepare_subprocess_invocation(
4015
4198
  )
4016
4199
 
4017
4200
 
4018
- def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
4019
- args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
4020
- return subprocess.check_call(args, **kwargs) # type: ignore
4201
+ ##
4202
+
4203
+
4204
+ @contextlib.contextmanager
4205
+ def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
4206
+ start_time = time.time()
4207
+ try:
4208
+ log.debug('subprocess_common_context.try: args=%r', args)
4209
+ yield
4210
+
4211
+ except Exception as exc: # noqa
4212
+ log.debug('subprocess_common_context.except: exc=%r', exc)
4213
+ raise
4214
+
4215
+ finally:
4216
+ end_time = time.time()
4217
+ elapsed_s = end_time - start_time
4218
+ log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
4219
+
4220
+
4221
+ ##
4222
+
4223
+
4224
+ def subprocess_check_call(
4225
+ *args: str,
4226
+ stdout: ta.Any = sys.stderr,
4227
+ **kwargs: ta.Any,
4228
+ ) -> None:
4229
+ args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
4230
+ with subprocess_common_context(*args, **kwargs):
4231
+ return subprocess.check_call(args, **kwargs) # type: ignore
4021
4232
 
4022
4233
 
4023
- def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
4024
- args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
4025
- return subprocess.check_output(args, **kwargs)
4234
+ def subprocess_check_output(
4235
+ *args: str,
4236
+ **kwargs: ta.Any,
4237
+ ) -> bytes:
4238
+ args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
4239
+ with subprocess_common_context(*args, **kwargs):
4240
+ return subprocess.check_output(args, **kwargs)
4026
4241
 
4027
4242
 
4028
4243
  def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
@@ -4038,16 +4253,31 @@ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
4038
4253
  )
4039
4254
 
4040
4255
 
4041
- def subprocess_try_call(
4042
- *args: str,
4256
+ def _subprocess_try_run(
4257
+ fn: ta.Callable[..., T],
4258
+ *args: ta.Any,
4043
4259
  try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4044
4260
  **kwargs: ta.Any,
4045
- ) -> bool:
4261
+ ) -> ta.Union[T, Exception]:
4046
4262
  try:
4047
- subprocess_check_call(*args, **kwargs)
4263
+ return fn(*args, **kwargs)
4048
4264
  except try_exceptions as e: # noqa
4049
4265
  if log.isEnabledFor(logging.DEBUG):
4050
4266
  log.exception('command failed')
4267
+ return e
4268
+
4269
+
4270
+ def subprocess_try_call(
4271
+ *args: str,
4272
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4273
+ **kwargs: ta.Any,
4274
+ ) -> bool:
4275
+ if isinstance(_subprocess_try_run(
4276
+ subprocess_check_call,
4277
+ *args,
4278
+ try_exceptions=try_exceptions,
4279
+ **kwargs,
4280
+ ), Exception):
4051
4281
  return False
4052
4282
  else:
4053
4283
  return True
@@ -4058,12 +4288,15 @@ def subprocess_try_output(
4058
4288
  try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4059
4289
  **kwargs: ta.Any,
4060
4290
  ) -> ta.Optional[bytes]:
4061
- try:
4062
- return subprocess_check_output(*args, **kwargs)
4063
- except try_exceptions as e: # noqa
4064
- if log.isEnabledFor(logging.DEBUG):
4065
- log.exception('command failed')
4291
+ if isinstance(ret := _subprocess_try_run(
4292
+ subprocess_check_output,
4293
+ *args,
4294
+ try_exceptions=try_exceptions,
4295
+ **kwargs,
4296
+ ), Exception):
4066
4297
  return None
4298
+ else:
4299
+ return ret
4067
4300
 
4068
4301
 
4069
4302
  def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
@@ -4090,175 +4323,922 @@ def subprocess_close(
4090
4323
 
4091
4324
 
4092
4325
  ########################################
4093
- # ../../../omdev/interp/inspect.py
4326
+ # ../remote/execution.py
4094
4327
 
4095
4328
 
4096
- @dc.dataclass(frozen=True)
4097
- class InterpInspection:
4098
- exe: str
4099
- version: Version
4329
+ ##
4100
4330
 
4101
- version_str: str
4102
- config_vars: ta.Mapping[str, str]
4103
- prefix: str
4104
- base_prefix: str
4105
4331
 
4106
- @property
4107
- def opts(self) -> InterpOpts:
4108
- return InterpOpts(
4109
- threaded=bool(self.config_vars.get('Py_GIL_DISABLED')),
4110
- debug=bool(self.config_vars.get('Py_DEBUG')),
4111
- )
4332
+ class _RemoteProtocol:
4333
+ class Message(abc.ABC): # noqa
4334
+ async def send(self, chan: RemoteChannel) -> None:
4335
+ await chan.send_obj(self, _RemoteProtocol.Message)
4112
4336
 
4113
- @property
4114
- def iv(self) -> InterpVersion:
4115
- return InterpVersion(
4116
- version=self.version,
4117
- opts=self.opts,
4118
- )
4337
+ @classmethod
4338
+ async def recv(cls: ta.Type[T], chan: RemoteChannel) -> ta.Optional[T]:
4339
+ return await chan.recv_obj(cls)
4119
4340
 
4120
- @property
4121
- def is_venv(self) -> bool:
4122
- return self.prefix != self.base_prefix
4341
+ #
4123
4342
 
4343
+ class Request(Message, abc.ABC): # noqa
4344
+ pass
4124
4345
 
4125
- class InterpInspector:
4346
+ @dc.dataclass(frozen=True)
4347
+ class CommandRequest(Request):
4348
+ seq: int
4349
+ cmd: Command
4126
4350
 
4127
- def __init__(self) -> None:
4128
- super().__init__()
4351
+ @dc.dataclass(frozen=True)
4352
+ class PingRequest(Request):
4353
+ time: float
4129
4354
 
4130
- self._cache: ta.Dict[str, ta.Optional[InterpInspection]] = {}
4355
+ #
4131
4356
 
4132
- _RAW_INSPECTION_CODE = """
4133
- __import__('json').dumps(dict(
4134
- version_str=__import__('sys').version,
4135
- prefix=__import__('sys').prefix,
4136
- base_prefix=__import__('sys').base_prefix,
4137
- config_vars=__import__('sysconfig').get_config_vars(),
4138
- ))"""
4357
+ class Response(Message, abc.ABC): # noqa
4358
+ pass
4139
4359
 
4140
- _INSPECTION_CODE = ''.join(l.strip() for l in _RAW_INSPECTION_CODE.splitlines())
4360
+ @dc.dataclass(frozen=True)
4361
+ class LogResponse(Response):
4362
+ s: str
4141
4363
 
4142
- @staticmethod
4143
- def _build_inspection(
4144
- exe: str,
4145
- output: str,
4146
- ) -> InterpInspection:
4147
- dct = json.loads(output)
4364
+ @dc.dataclass(frozen=True)
4365
+ class CommandResponse(Response):
4366
+ seq: int
4367
+ res: CommandOutputOrExceptionData
4148
4368
 
4149
- version = Version(dct['version_str'].split()[0])
4369
+ @dc.dataclass(frozen=True)
4370
+ class PingResponse(Response):
4371
+ time: float
4150
4372
 
4151
- return InterpInspection(
4152
- exe=exe,
4153
- version=version,
4154
- **{k: dct[k] for k in (
4155
- 'version_str',
4156
- 'prefix',
4157
- 'base_prefix',
4158
- 'config_vars',
4159
- )},
4160
- )
4161
4373
 
4162
- @classmethod
4163
- def running(cls) -> 'InterpInspection':
4164
- return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
4374
+ ##
4165
4375
 
4166
- def _inspect(self, exe: str) -> InterpInspection:
4167
- output = subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
4168
- return self._build_inspection(exe, output.decode())
4169
4376
 
4170
- def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
4171
- try:
4172
- return self._cache[exe]
4173
- except KeyError:
4174
- ret: ta.Optional[InterpInspection]
4175
- try:
4176
- ret = self._inspect(exe)
4177
- except Exception as e: # noqa
4178
- if log.isEnabledFor(logging.DEBUG):
4179
- log.exception('Failed to inspect interp: %s', exe)
4180
- ret = None
4181
- self._cache[exe] = ret
4182
- return ret
4377
+ class _RemoteLogHandler(logging.Handler):
4378
+ def __init__(
4379
+ self,
4380
+ chan: RemoteChannel,
4381
+ loop: ta.Any = None,
4382
+ ) -> None:
4383
+ super().__init__()
4183
4384
 
4385
+ self._chan = chan
4386
+ self._loop = loop
4184
4387
 
4185
- INTERP_INSPECTOR = InterpInspector()
4388
+ def emit(self, record):
4389
+ msg = self.format(record)
4186
4390
 
4391
+ async def inner():
4392
+ await _RemoteProtocol.LogResponse(msg).send(self._chan)
4187
4393
 
4188
- ########################################
4189
- # ../commands/subprocess.py
4394
+ loop = self._loop
4395
+ if loop is None:
4396
+ loop = asyncio.get_running_loop()
4397
+ if loop is not None:
4398
+ asyncio.run_coroutine_threadsafe(inner(), loop)
4190
4399
 
4191
4400
 
4192
4401
  ##
4193
4402
 
4194
4403
 
4195
- @dc.dataclass(frozen=True)
4196
- class SubprocessCommand(Command['SubprocessCommand.Output']):
4197
- cmd: ta.Sequence[str]
4404
+ class _RemoteCommandHandler:
4405
+ def __init__(
4406
+ self,
4407
+ chan: RemoteChannel,
4408
+ executor: CommandExecutor,
4409
+ *,
4410
+ stop: ta.Optional[asyncio.Event] = None,
4411
+ ) -> None:
4412
+ super().__init__()
4198
4413
 
4199
- shell: bool = False
4200
- cwd: ta.Optional[str] = None
4201
- env: ta.Optional[ta.Mapping[str, str]] = None
4414
+ self._chan = chan
4415
+ self._executor = executor
4416
+ self._stop = stop if stop is not None else asyncio.Event()
4202
4417
 
4203
- stdout: str = 'pipe' # SubprocessChannelOption
4204
- stderr: str = 'pipe' # SubprocessChannelOption
4418
+ self._cmds_by_seq: ta.Dict[int, _RemoteCommandHandler._Command] = {}
4419
+
4420
+ @dc.dataclass(frozen=True)
4421
+ class _Command:
4422
+ req: _RemoteProtocol.CommandRequest
4423
+ fut: asyncio.Future
4424
+
4425
+ async def run(self) -> None:
4426
+ stop_task = asyncio.create_task(self._stop.wait())
4427
+ recv_task: ta.Optional[asyncio.Task] = None
4428
+
4429
+ while not self._stop.is_set():
4430
+ if recv_task is None:
4431
+ recv_task = asyncio.create_task(_RemoteProtocol.Request.recv(self._chan))
4432
+
4433
+ done, pending = await asyncio.wait([
4434
+ stop_task,
4435
+ recv_task,
4436
+ ], return_when=asyncio.FIRST_COMPLETED)
4437
+
4438
+ if recv_task in done:
4439
+ msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
4440
+ recv_task.result(),
4441
+ (_RemoteProtocol.Message, type(None)),
4442
+ )
4443
+ recv_task = None
4444
+
4445
+ if msg is None:
4446
+ break
4447
+
4448
+ await self._handle_message(msg)
4449
+
4450
+ async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
4451
+ if isinstance(msg, _RemoteProtocol.PingRequest):
4452
+ log.debug('Ping: %r', msg)
4453
+ await _RemoteProtocol.PingResponse(
4454
+ time=msg.time,
4455
+ ).send(self._chan)
4456
+
4457
+ elif isinstance(msg, _RemoteProtocol.CommandRequest):
4458
+ fut = asyncio.create_task(self._handle_command_request(msg))
4459
+ self._cmds_by_seq[msg.seq] = _RemoteCommandHandler._Command(
4460
+ req=msg,
4461
+ fut=fut,
4462
+ )
4463
+
4464
+ else:
4465
+ raise TypeError(msg)
4466
+
4467
+ async def _handle_command_request(self, req: _RemoteProtocol.CommandRequest) -> None:
4468
+ res = await self._executor.try_execute(
4469
+ req.cmd,
4470
+ log=log,
4471
+ omit_exc_object=True,
4472
+ )
4473
+
4474
+ await _RemoteProtocol.CommandResponse(
4475
+ seq=req.seq,
4476
+ res=CommandOutputOrExceptionData(
4477
+ output=res.output,
4478
+ exception=res.exception,
4479
+ ),
4480
+ ).send(self._chan)
4481
+
4482
+ self._cmds_by_seq.pop(req.seq) # noqa
4483
+
4484
+
4485
+ ##
4486
+
4487
+
4488
+ @dc.dataclass()
4489
+ class RemoteCommandError(Exception):
4490
+ e: CommandException
4491
+
4492
+
4493
+ class RemoteCommandExecutor(CommandExecutor):
4494
+ def __init__(self, chan: RemoteChannel) -> None:
4495
+ super().__init__()
4496
+
4497
+ self._chan = chan
4498
+
4499
+ self._cmd_seq = itertools.count()
4500
+ self._queue: asyncio.Queue = asyncio.Queue() # asyncio.Queue[RemoteCommandExecutor._Request]
4501
+ self._stop = asyncio.Event()
4502
+ self._loop_task: ta.Optional[asyncio.Task] = None
4503
+ self._reqs_by_seq: ta.Dict[int, RemoteCommandExecutor._Request] = {}
4504
+
4505
+ #
4506
+
4507
+ async def start(self) -> None:
4508
+ check_none(self._loop_task)
4509
+ check_state(not self._stop.is_set())
4510
+ self._loop_task = asyncio.create_task(self._loop())
4511
+
4512
+ async def aclose(self) -> None:
4513
+ self._stop.set()
4514
+ if self._loop_task is not None:
4515
+ await self._loop_task
4516
+
4517
+ #
4518
+
4519
+ @dc.dataclass(frozen=True)
4520
+ class _Request:
4521
+ seq: int
4522
+ cmd: Command
4523
+ fut: asyncio.Future
4524
+
4525
+ async def _loop(self) -> None:
4526
+ log.debug('RemoteCommandExecutor loop start: %r', self)
4527
+
4528
+ stop_task = asyncio.create_task(self._stop.wait())
4529
+ queue_task: ta.Optional[asyncio.Task] = None
4530
+ recv_task: ta.Optional[asyncio.Task] = None
4531
+
4532
+ while not self._stop.is_set():
4533
+ if queue_task is None:
4534
+ queue_task = asyncio.create_task(self._queue.get())
4535
+ if recv_task is None:
4536
+ recv_task = asyncio.create_task(_RemoteProtocol.Message.recv(self._chan))
4537
+
4538
+ done, pending = await asyncio.wait([
4539
+ stop_task,
4540
+ queue_task,
4541
+ recv_task,
4542
+ ], return_when=asyncio.FIRST_COMPLETED)
4543
+
4544
+ if queue_task in done:
4545
+ req = check_isinstance(queue_task.result(), RemoteCommandExecutor._Request)
4546
+ queue_task = None
4547
+ await self._handle_request(req)
4548
+
4549
+ if recv_task in done:
4550
+ msg: ta.Optional[_RemoteProtocol.Message] = check_isinstance(
4551
+ recv_task.result(),
4552
+ (_RemoteProtocol.Message, type(None)),
4553
+ )
4554
+ recv_task = None
4555
+
4556
+ if msg is None:
4557
+ log.debug('RemoteCommandExecutor got eof: %r', self)
4558
+ break
4559
+
4560
+ await self._handle_message(msg)
4561
+
4562
+ log.debug('RemoteCommandExecutor loop stopping: %r', self)
4563
+
4564
+ for task in [
4565
+ stop_task,
4566
+ queue_task,
4567
+ recv_task,
4568
+ ]:
4569
+ if task is not None and not task.done():
4570
+ task.cancel()
4571
+
4572
+ for req in self._reqs_by_seq.values():
4573
+ req.fut.cancel()
4574
+
4575
+ log.debug('RemoteCommandExecutor loop exited: %r', self)
4576
+
4577
+ async def _handle_request(self, req: _Request) -> None:
4578
+ self._reqs_by_seq[req.seq] = req
4579
+ await _RemoteProtocol.CommandRequest(
4580
+ seq=req.seq,
4581
+ cmd=req.cmd,
4582
+ ).send(self._chan)
4583
+
4584
+ async def _handle_message(self, msg: _RemoteProtocol.Message) -> None:
4585
+ if isinstance(msg, _RemoteProtocol.PingRequest):
4586
+ log.debug('Ping: %r', msg)
4587
+ await _RemoteProtocol.PingResponse(
4588
+ time=msg.time,
4589
+ ).send(self._chan)
4590
+
4591
+ elif isinstance(msg, _RemoteProtocol.LogResponse):
4592
+ log.info(msg.s)
4593
+
4594
+ elif isinstance(msg, _RemoteProtocol.CommandResponse):
4595
+ req = self._reqs_by_seq.pop(msg.seq)
4596
+ req.fut.set_result(msg.res)
4597
+
4598
+ else:
4599
+ raise TypeError(msg)
4600
+
4601
+ #
4602
+
4603
+ async def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
4604
+ req = RemoteCommandExecutor._Request(
4605
+ seq=next(self._cmd_seq),
4606
+ cmd=cmd,
4607
+ fut=asyncio.Future(),
4608
+ )
4609
+ await self._queue.put(req)
4610
+ return await req.fut
4611
+
4612
+ # @ta.override
4613
+ async def execute(self, cmd: Command) -> Command.Output:
4614
+ r = await self._remote_execute(cmd)
4615
+ if (e := r.exception) is not None:
4616
+ raise RemoteCommandError(e)
4617
+ else:
4618
+ return check_not_none(r.output)
4619
+
4620
+ # @ta.override
4621
+ async def try_execute(
4622
+ self,
4623
+ cmd: Command,
4624
+ *,
4625
+ log: ta.Optional[logging.Logger] = None,
4626
+ omit_exc_object: bool = False,
4627
+ ) -> CommandOutputOrException:
4628
+ try:
4629
+ r = await self._remote_execute(cmd)
4630
+
4631
+ except Exception as e: # noqa
4632
+ if log is not None:
4633
+ log.exception('Exception executing remote command: %r', type(cmd))
4634
+
4635
+ return CommandOutputOrExceptionData(exception=CommandException.of(
4636
+ e,
4637
+ omit_exc_object=omit_exc_object,
4638
+ cmd=cmd,
4639
+ ))
4640
+
4641
+ else:
4642
+ return r
4643
+
4644
+
4645
+ ########################################
4646
+ # ../../../omlish/lite/asyncio/subprocesses.py
4647
+
4648
+
4649
+ ##
4650
+
4651
+
4652
+ @contextlib.asynccontextmanager
4653
+ async def asyncio_subprocess_popen(
4654
+ *cmd: str,
4655
+ shell: bool = False,
4656
+ timeout: ta.Optional[float] = None,
4657
+ **kwargs: ta.Any,
4658
+ ) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
4659
+ fac: ta.Any
4660
+ if shell:
4661
+ fac = functools.partial(
4662
+ asyncio.create_subprocess_shell,
4663
+ check_single(cmd),
4664
+ )
4665
+ else:
4666
+ fac = functools.partial(
4667
+ asyncio.create_subprocess_exec,
4668
+ *cmd,
4669
+ )
4670
+
4671
+ with subprocess_common_context(
4672
+ *cmd,
4673
+ shell=shell,
4674
+ timeout=timeout,
4675
+ **kwargs,
4676
+ ):
4677
+ proc: asyncio.subprocess.Process
4678
+ proc = await fac(**kwargs)
4679
+ try:
4680
+ yield proc
4681
+
4682
+ finally:
4683
+ await asyncio_maybe_timeout(proc.wait(), timeout)
4684
+
4685
+
4686
+ ##
4687
+
4688
+
4689
+ class AsyncioProcessCommunicator:
4690
+ def __init__(
4691
+ self,
4692
+ proc: asyncio.subprocess.Process,
4693
+ loop: ta.Optional[ta.Any] = None,
4694
+ ) -> None:
4695
+ super().__init__()
4696
+
4697
+ if loop is None:
4698
+ loop = asyncio.get_running_loop()
4699
+
4700
+ self._proc = proc
4701
+ self._loop = loop
4702
+
4703
+ self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check_isinstance(
4704
+ proc._transport, # type: ignore # noqa
4705
+ asyncio.base_subprocess.BaseSubprocessTransport,
4706
+ )
4707
+
4708
+ @property
4709
+ def _debug(self) -> bool:
4710
+ return self._loop.get_debug()
4711
+
4712
+ async def _feed_stdin(self, input: bytes) -> None: # noqa
4713
+ stdin = check_not_none(self._proc.stdin)
4714
+ try:
4715
+ if input is not None:
4716
+ stdin.write(input)
4717
+ if self._debug:
4718
+ log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
4719
+
4720
+ await stdin.drain()
4721
+
4722
+ except (BrokenPipeError, ConnectionResetError) as exc:
4723
+ # communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
4724
+ # exceptions.
4725
+ if self._debug:
4726
+ log.debug('%r communicate: stdin got %r', self, exc)
4727
+
4728
+ if self._debug:
4729
+ log.debug('%r communicate: close stdin', self)
4730
+
4731
+ stdin.close()
4732
+
4733
+ async def _noop(self) -> None:
4734
+ return None
4735
+
4736
+ async def _read_stream(self, fd: int) -> bytes:
4737
+ transport: ta.Any = check_not_none(self._transport.get_pipe_transport(fd))
4738
+
4739
+ if fd == 2:
4740
+ stream = check_not_none(self._proc.stderr)
4741
+ else:
4742
+ check_equal(fd, 1)
4743
+ stream = check_not_none(self._proc.stdout)
4744
+
4745
+ if self._debug:
4746
+ name = 'stdout' if fd == 1 else 'stderr'
4747
+ log.debug('%r communicate: read %s', self, name)
4748
+
4749
+ output = await stream.read()
4750
+
4751
+ if self._debug:
4752
+ name = 'stdout' if fd == 1 else 'stderr'
4753
+ log.debug('%r communicate: close %s', self, name)
4754
+
4755
+ transport.close()
4756
+
4757
+ return output
4758
+
4759
+ class Communication(ta.NamedTuple):
4760
+ stdout: ta.Optional[bytes]
4761
+ stderr: ta.Optional[bytes]
4762
+
4763
+ async def _communicate(
4764
+ self,
4765
+ input: ta.Any = None, # noqa
4766
+ ) -> Communication:
4767
+ stdin_fut: ta.Any
4768
+ if self._proc.stdin is not None:
4769
+ stdin_fut = self._feed_stdin(input)
4770
+ else:
4771
+ stdin_fut = self._noop()
4772
+
4773
+ stdout_fut: ta.Any
4774
+ if self._proc.stdout is not None:
4775
+ stdout_fut = self._read_stream(1)
4776
+ else:
4777
+ stdout_fut = self._noop()
4778
+
4779
+ stderr_fut: ta.Any
4780
+ if self._proc.stderr is not None:
4781
+ stderr_fut = self._read_stream(2)
4782
+ else:
4783
+ stderr_fut = self._noop()
4784
+
4785
+ stdin_res, stdout_res, stderr_res = await asyncio.gather(stdin_fut, stdout_fut, stderr_fut)
4786
+
4787
+ await self._proc.wait()
4788
+
4789
+ return AsyncioProcessCommunicator.Communication(stdout_res, stderr_res)
4790
+
4791
+ async def communicate(
4792
+ self,
4793
+ input: ta.Any = None, # noqa
4794
+ timeout: ta.Optional[float] = None,
4795
+ ) -> Communication:
4796
+ return await asyncio_maybe_timeout(self._communicate(input), timeout)
4797
+
4798
+
4799
+ async def asyncio_subprocess_communicate(
4800
+ proc: asyncio.subprocess.Process,
4801
+ input: ta.Any = None, # noqa
4802
+ timeout: ta.Optional[float] = None,
4803
+ ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
4804
+ return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
4805
+
4806
+
4807
+ ##
4808
+
4809
+
4810
+ async def _asyncio_subprocess_check_run(
4811
+ *args: str,
4812
+ input: ta.Any = None, # noqa
4813
+ timeout: ta.Optional[float] = None,
4814
+ **kwargs: ta.Any,
4815
+ ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
4816
+ args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
4817
+
4818
+ proc: asyncio.subprocess.Process
4819
+ async with asyncio_subprocess_popen(*args, **kwargs) as proc:
4820
+ stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
4821
+
4822
+ if proc.returncode:
4823
+ raise subprocess.CalledProcessError(
4824
+ proc.returncode,
4825
+ args,
4826
+ output=stdout,
4827
+ stderr=stderr,
4828
+ )
4829
+
4830
+ return stdout, stderr
4831
+
4832
+
4833
+ async def asyncio_subprocess_check_call(
4834
+ *args: str,
4835
+ stdout: ta.Any = sys.stderr,
4836
+ input: ta.Any = None, # noqa
4837
+ timeout: ta.Optional[float] = None,
4838
+ **kwargs: ta.Any,
4839
+ ) -> None:
4840
+ _, _ = await _asyncio_subprocess_check_run(
4841
+ *args,
4842
+ stdout=stdout,
4843
+ input=input,
4844
+ timeout=timeout,
4845
+ **kwargs,
4846
+ )
4847
+
4848
+
4849
+ async def asyncio_subprocess_check_output(
4850
+ *args: str,
4851
+ input: ta.Any = None, # noqa
4852
+ timeout: ta.Optional[float] = None,
4853
+ **kwargs: ta.Any,
4854
+ ) -> bytes:
4855
+ stdout, stderr = await _asyncio_subprocess_check_run(
4856
+ *args,
4857
+ stdout=asyncio.subprocess.PIPE,
4858
+ input=input,
4859
+ timeout=timeout,
4860
+ **kwargs,
4861
+ )
4862
+
4863
+ return check_not_none(stdout)
4864
+
4865
+
4866
+ async def asyncio_subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
4867
+ return (await asyncio_subprocess_check_output(*args, **kwargs)).decode().strip()
4868
+
4869
+
4870
+ ##
4871
+
4872
+
4873
+ async def _asyncio_subprocess_try_run(
4874
+ fn: ta.Callable[..., ta.Awaitable[T]],
4875
+ *args: ta.Any,
4876
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4877
+ **kwargs: ta.Any,
4878
+ ) -> ta.Union[T, Exception]:
4879
+ try:
4880
+ return await fn(*args, **kwargs)
4881
+ except try_exceptions as e: # noqa
4882
+ if log.isEnabledFor(logging.DEBUG):
4883
+ log.exception('command failed')
4884
+ return e
4885
+
4886
+
4887
+ async def asyncio_subprocess_try_call(
4888
+ *args: str,
4889
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4890
+ **kwargs: ta.Any,
4891
+ ) -> bool:
4892
+ if isinstance(await _asyncio_subprocess_try_run(
4893
+ asyncio_subprocess_check_call,
4894
+ *args,
4895
+ try_exceptions=try_exceptions,
4896
+ **kwargs,
4897
+ ), Exception):
4898
+ return False
4899
+ else:
4900
+ return True
4901
+
4902
+
4903
+ async def asyncio_subprocess_try_output(
4904
+ *args: str,
4905
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4906
+ **kwargs: ta.Any,
4907
+ ) -> ta.Optional[bytes]:
4908
+ if isinstance(ret := await _asyncio_subprocess_try_run(
4909
+ asyncio_subprocess_check_output,
4910
+ *args,
4911
+ try_exceptions=try_exceptions,
4912
+ **kwargs,
4913
+ ), Exception):
4914
+ return None
4915
+ else:
4916
+ return ret
4917
+
4918
+
4919
+ async def asyncio_subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
4920
+ out = await asyncio_subprocess_try_output(*args, **kwargs)
4921
+ return out.decode().strip() if out is not None else None
4922
+
4923
+
4924
+ ########################################
4925
+ # ../../../omdev/interp/inspect.py
4926
+
4927
+
4928
+ @dc.dataclass(frozen=True)
4929
+ class InterpInspection:
4930
+ exe: str
4931
+ version: Version
4932
+
4933
+ version_str: str
4934
+ config_vars: ta.Mapping[str, str]
4935
+ prefix: str
4936
+ base_prefix: str
4937
+
4938
+ @property
4939
+ def opts(self) -> InterpOpts:
4940
+ return InterpOpts(
4941
+ threaded=bool(self.config_vars.get('Py_GIL_DISABLED')),
4942
+ debug=bool(self.config_vars.get('Py_DEBUG')),
4943
+ )
4944
+
4945
+ @property
4946
+ def iv(self) -> InterpVersion:
4947
+ return InterpVersion(
4948
+ version=self.version,
4949
+ opts=self.opts,
4950
+ )
4951
+
4952
+ @property
4953
+ def is_venv(self) -> bool:
4954
+ return self.prefix != self.base_prefix
4955
+
4956
+
4957
+ class InterpInspector:
4958
+ def __init__(self) -> None:
4959
+ super().__init__()
4960
+
4961
+ self._cache: ta.Dict[str, ta.Optional[InterpInspection]] = {}
4962
+
4963
+ _RAW_INSPECTION_CODE = """
4964
+ __import__('json').dumps(dict(
4965
+ version_str=__import__('sys').version,
4966
+ prefix=__import__('sys').prefix,
4967
+ base_prefix=__import__('sys').base_prefix,
4968
+ config_vars=__import__('sysconfig').get_config_vars(),
4969
+ ))"""
4970
+
4971
+ _INSPECTION_CODE = ''.join(l.strip() for l in _RAW_INSPECTION_CODE.splitlines())
4972
+
4973
+ @staticmethod
4974
+ def _build_inspection(
4975
+ exe: str,
4976
+ output: str,
4977
+ ) -> InterpInspection:
4978
+ dct = json.loads(output)
4979
+
4980
+ version = Version(dct['version_str'].split()[0])
4981
+
4982
+ return InterpInspection(
4983
+ exe=exe,
4984
+ version=version,
4985
+ **{k: dct[k] for k in (
4986
+ 'version_str',
4987
+ 'prefix',
4988
+ 'base_prefix',
4989
+ 'config_vars',
4990
+ )},
4991
+ )
4992
+
4993
+ @classmethod
4994
+ def running(cls) -> 'InterpInspection':
4995
+ return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
4996
+
4997
+ async def _inspect(self, exe: str) -> InterpInspection:
4998
+ output = await asyncio_subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
4999
+ return self._build_inspection(exe, output.decode())
5000
+
5001
+ async def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
5002
+ try:
5003
+ return self._cache[exe]
5004
+ except KeyError:
5005
+ ret: ta.Optional[InterpInspection]
5006
+ try:
5007
+ ret = await self._inspect(exe)
5008
+ except Exception as e: # noqa
5009
+ if log.isEnabledFor(logging.DEBUG):
5010
+ log.exception('Failed to inspect interp: %s', exe)
5011
+ ret = None
5012
+ self._cache[exe] = ret
5013
+ return ret
5014
+
5015
+
5016
+ INTERP_INSPECTOR = InterpInspector()
5017
+
5018
+
5019
+ ########################################
5020
+ # ../commands/subprocess.py
5021
+
5022
+
5023
+ ##
5024
+
5025
+
5026
+ @dc.dataclass(frozen=True)
5027
+ class SubprocessCommand(Command['SubprocessCommand.Output']):
5028
+ cmd: ta.Sequence[str]
5029
+
5030
+ shell: bool = False
5031
+ cwd: ta.Optional[str] = None
5032
+ env: ta.Optional[ta.Mapping[str, str]] = None
5033
+
5034
+ stdout: str = 'pipe' # SubprocessChannelOption
5035
+ stderr: str = 'pipe' # SubprocessChannelOption
4205
5036
 
4206
5037
  input: ta.Optional[bytes] = None
4207
5038
  timeout: ta.Optional[float] = None
4208
5039
 
4209
- def __post_init__(self) -> None:
4210
- check_not_isinstance(self.cmd, str)
5040
+ def __post_init__(self) -> None:
5041
+ check_not_isinstance(self.cmd, str)
5042
+
5043
+ @dc.dataclass(frozen=True)
5044
+ class Output(Command.Output):
5045
+ rc: int
5046
+ pid: int
5047
+
5048
+ elapsed_s: float
5049
+
5050
+ stdout: ta.Optional[bytes] = None
5051
+ stderr: ta.Optional[bytes] = None
5052
+
5053
+
5054
+ ##
5055
+
5056
+
5057
+ class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
5058
+ async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
5059
+ proc: asyncio.subprocess.Process
5060
+ async with asyncio_subprocess_popen(
5061
+ *subprocess_maybe_shell_wrap_exec(*cmd.cmd),
5062
+
5063
+ shell=cmd.shell,
5064
+ cwd=cmd.cwd,
5065
+ env={**os.environ, **(cmd.env or {})},
5066
+
5067
+ stdin=subprocess.PIPE if cmd.input is not None else None,
5068
+ stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stdout)],
5069
+ stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, cmd.stderr)],
5070
+
5071
+ timeout=cmd.timeout,
5072
+ ) as proc:
5073
+ start_time = time.time()
5074
+ stdout, stderr = await asyncio_subprocess_communicate(
5075
+ proc,
5076
+ input=cmd.input,
5077
+ timeout=cmd.timeout,
5078
+ )
5079
+ end_time = time.time()
5080
+
5081
+ return SubprocessCommand.Output(
5082
+ rc=check_not_none(proc.returncode),
5083
+ pid=proc.pid,
5084
+
5085
+ elapsed_s=end_time - start_time,
5086
+
5087
+ stdout=stdout, # noqa
5088
+ stderr=stderr, # noqa
5089
+ )
5090
+
5091
+
5092
+ ########################################
5093
+ # ../remote/_main.py
5094
+
5095
+
5096
+ ##
5097
+
5098
+
5099
+ class _RemoteExecutionLogHandler(logging.Handler):
5100
+ def __init__(self, fn: ta.Callable[[str], None]) -> None:
5101
+ super().__init__()
5102
+ self._fn = fn
5103
+
5104
+ def emit(self, record):
5105
+ msg = self.format(record)
5106
+ self._fn(msg)
5107
+
5108
+
5109
+ ##
5110
+
5111
+
5112
+ class _RemoteExecutionMain:
5113
+ def __init__(
5114
+ self,
5115
+ chan: RemoteChannel,
5116
+ ) -> None:
5117
+ super().__init__()
5118
+
5119
+ self._chan = chan
5120
+
5121
+ self.__bootstrap: ta.Optional[MainBootstrap] = None
5122
+ self.__injector: ta.Optional[Injector] = None
5123
+
5124
+ @property
5125
+ def _bootstrap(self) -> MainBootstrap:
5126
+ return check_not_none(self.__bootstrap)
5127
+
5128
+ @property
5129
+ def _injector(self) -> Injector:
5130
+ return check_not_none(self.__injector)
5131
+
5132
+ #
5133
+
5134
+ def _timebomb_main(
5135
+ self,
5136
+ delay_s: float,
5137
+ *,
5138
+ sig: int = signal.SIGINT,
5139
+ code: int = 1,
5140
+ ) -> None:
5141
+ time.sleep(delay_s)
5142
+
5143
+ if (pgid := os.getpgid(0)) == os.getpid():
5144
+ os.killpg(pgid, sig)
5145
+
5146
+ os._exit(code) # noqa
5147
+
5148
+ @cached_nullary
5149
+ def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
5150
+ if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
5151
+ return None
5152
+
5153
+ thr = threading.Thread(
5154
+ target=functools.partial(self._timebomb_main, tbd),
5155
+ name=f'{self.__class__.__name__}.timebomb',
5156
+ daemon=True,
5157
+ )
5158
+
5159
+ thr.start()
4211
5160
 
4212
- @dc.dataclass(frozen=True)
4213
- class Output(Command.Output):
4214
- rc: int
4215
- pid: int
5161
+ log.debug('Started timebomb thread: %r', thr)
4216
5162
 
4217
- elapsed_s: float
5163
+ return thr
4218
5164
 
4219
- stdout: ta.Optional[bytes] = None
4220
- stderr: ta.Optional[bytes] = None
5165
+ #
4221
5166
 
5167
+ @cached_nullary
5168
+ def _log_handler(self) -> _RemoteLogHandler:
5169
+ return _RemoteLogHandler(self._chan)
4222
5170
 
4223
- ##
5171
+ #
4224
5172
 
5173
+ async def _setup(self) -> None:
5174
+ check_none(self.__bootstrap)
5175
+ check_none(self.__injector)
4225
5176
 
4226
- class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
4227
- def execute(self, inp: SubprocessCommand) -> SubprocessCommand.Output:
4228
- with subprocess.Popen(
4229
- subprocess_maybe_shell_wrap_exec(*inp.cmd),
5177
+ # Bootstrap
4230
5178
 
4231
- shell=inp.shell,
4232
- cwd=inp.cwd,
4233
- env={**os.environ, **(inp.env or {})},
5179
+ self.__bootstrap = check_not_none(await self._chan.recv_obj(MainBootstrap))
4234
5180
 
4235
- stdin=subprocess.PIPE if inp.input is not None else None,
4236
- stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stdout)],
4237
- stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stderr)],
4238
- ) as proc:
4239
- start_time = time.time()
4240
- stdout, stderr = proc.communicate(
4241
- input=inp.input,
4242
- timeout=inp.timeout,
4243
- )
4244
- end_time = time.time()
5181
+ if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
5182
+ pycharm_debug_connect(prd)
4245
5183
 
4246
- return SubprocessCommand.Output(
4247
- rc=proc.returncode,
4248
- pid=proc.pid,
5184
+ self.__injector = main_bootstrap(self._bootstrap)
4249
5185
 
4250
- elapsed_s=end_time - start_time,
5186
+ self._chan.set_marshaler(self._injector[ObjMarshalerManager])
4251
5187
 
4252
- stdout=stdout, # noqa
4253
- stderr=stderr, # noqa
5188
+ # Post-bootstrap
5189
+
5190
+ if self._bootstrap.remote_config.set_pgid:
5191
+ if os.getpgid(0) != os.getpid():
5192
+ log.debug('Setting pgid')
5193
+ os.setpgid(0, 0)
5194
+
5195
+ if (ds := self._bootstrap.remote_config.deathsig) is not None:
5196
+ log.debug('Setting deathsig: %s', ds)
5197
+ set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
5198
+
5199
+ self._timebomb_thread()
5200
+
5201
+ if self._bootstrap.remote_config.forward_logging:
5202
+ log.debug('Installing log forwarder')
5203
+ logging.root.addHandler(self._log_handler())
5204
+
5205
+ #
5206
+
5207
+ async def run(self) -> None:
5208
+ await self._setup()
5209
+
5210
+ executor = self._injector[LocalCommandExecutor]
5211
+
5212
+ handler = _RemoteCommandHandler(self._chan, executor)
5213
+
5214
+ await handler.run()
5215
+
5216
+
5217
+ def _remote_execution_main() -> None:
5218
+ rt = pyremote_bootstrap_finalize() # noqa
5219
+
5220
+ async def inner() -> None:
5221
+ input = await asyncio_open_stream_reader(rt.input) # noqa
5222
+ output = await asyncio_open_stream_writer(rt.output)
5223
+
5224
+ chan = RemoteChannelImpl(
5225
+ input,
5226
+ output,
4254
5227
  )
4255
5228
 
5229
+ await _RemoteExecutionMain(chan).run()
5230
+
5231
+ asyncio.run(inner())
5232
+
4256
5233
 
4257
5234
  ########################################
4258
5235
  # ../remote/spawning.py
4259
5236
 
4260
5237
 
4261
- class RemoteSpawning:
5238
+ ##
5239
+
5240
+
5241
+ class RemoteSpawning(abc.ABC):
4262
5242
  @dc.dataclass(frozen=True)
4263
5243
  class Target:
4264
5244
  shell: ta.Optional[str] = None
@@ -4269,15 +5249,35 @@ class RemoteSpawning:
4269
5249
 
4270
5250
  stderr: ta.Optional[str] = None # SubprocessChannelOption
4271
5251
 
4272
- #
5252
+ @dc.dataclass(frozen=True)
5253
+ class Spawned:
5254
+ stdin: asyncio.StreamWriter
5255
+ stdout: asyncio.StreamReader
5256
+ stderr: ta.Optional[asyncio.StreamReader]
5257
+
5258
+ @abc.abstractmethod
5259
+ def spawn(
5260
+ self,
5261
+ tgt: Target,
5262
+ src: str,
5263
+ *,
5264
+ timeout: ta.Optional[float] = None,
5265
+ debug: bool = False,
5266
+ ) -> ta.AsyncContextManager[Spawned]:
5267
+ raise NotImplementedError
4273
5268
 
4274
- class _PreparedCmd(ta.NamedTuple):
5269
+
5270
+ ##
5271
+
5272
+
5273
+ class SubprocessRemoteSpawning(RemoteSpawning):
5274
+ class _PreparedCmd(ta.NamedTuple): # noqa
4275
5275
  cmd: ta.Sequence[str]
4276
5276
  shell: bool
4277
5277
 
4278
5278
  def _prepare_cmd(
4279
5279
  self,
4280
- tgt: Target,
5280
+ tgt: RemoteSpawning.Target,
4281
5281
  src: str,
4282
5282
  ) -> _PreparedCmd:
4283
5283
  if tgt.shell is not None:
@@ -4285,44 +5285,38 @@ class RemoteSpawning:
4285
5285
  if tgt.shell_quote:
4286
5286
  sh_src = shlex.quote(sh_src)
4287
5287
  sh_cmd = f'{tgt.shell} {sh_src}'
4288
- return RemoteSpawning._PreparedCmd(
4289
- cmd=[sh_cmd],
4290
- shell=True,
4291
- )
5288
+ return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
4292
5289
 
4293
5290
  else:
4294
- return RemoteSpawning._PreparedCmd(
4295
- cmd=[tgt.python, '-c', src],
4296
- shell=False,
4297
- )
5291
+ return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
4298
5292
 
4299
5293
  #
4300
5294
 
4301
- @dc.dataclass(frozen=True)
4302
- class Spawned:
4303
- stdin: ta.IO
4304
- stdout: ta.IO
4305
- stderr: ta.Optional[ta.IO]
4306
-
4307
- @contextlib.contextmanager
4308
- def spawn(
5295
+ @contextlib.asynccontextmanager
5296
+ async def spawn(
4309
5297
  self,
4310
- tgt: Target,
5298
+ tgt: RemoteSpawning.Target,
4311
5299
  src: str,
4312
5300
  *,
4313
5301
  timeout: ta.Optional[float] = None,
4314
- ) -> ta.Generator[Spawned, None, None]:
5302
+ debug: bool = False,
5303
+ ) -> ta.AsyncGenerator[RemoteSpawning.Spawned, None]:
4315
5304
  pc = self._prepare_cmd(tgt, src)
4316
5305
 
4317
- with subprocess.Popen(
4318
- subprocess_maybe_shell_wrap_exec(*pc.cmd),
4319
- shell=pc.shell,
4320
- stdin=subprocess.PIPE,
4321
- stdout=subprocess.PIPE,
4322
- stderr=(
4323
- SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
4324
- if tgt.stderr is not None else None
4325
- ),
5306
+ cmd = pc.cmd
5307
+ if not debug:
5308
+ cmd = subprocess_maybe_shell_wrap_exec(*cmd)
5309
+
5310
+ async with asyncio_subprocess_popen(
5311
+ *cmd,
5312
+ shell=pc.shell,
5313
+ stdin=subprocess.PIPE,
5314
+ stdout=subprocess.PIPE,
5315
+ stderr=(
5316
+ SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
5317
+ if tgt.stderr is not None else None
5318
+ ),
5319
+ timeout=timeout,
4326
5320
  ) as proc:
4327
5321
  stdin = check_not_none(proc.stdin)
4328
5322
  stdout = check_not_none(proc.stdout)
@@ -4340,8 +5334,6 @@ class RemoteSpawning:
4340
5334
  except BrokenPipeError:
4341
5335
  pass
4342
5336
 
4343
- proc.wait(timeout)
4344
-
4345
5337
 
4346
5338
  ########################################
4347
5339
  # ../../../omdev/interp/providers.py
@@ -4370,17 +5362,17 @@ class InterpProvider(abc.ABC):
4370
5362
  setattr(cls, 'name', snake_case(cls.__name__[:-len(sfx)]))
4371
5363
 
4372
5364
  @abc.abstractmethod
4373
- def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5365
+ def get_installed_versions(self, spec: InterpSpecifier) -> ta.Awaitable[ta.Sequence[InterpVersion]]:
4374
5366
  raise NotImplementedError
4375
5367
 
4376
5368
  @abc.abstractmethod
4377
- def get_installed_version(self, version: InterpVersion) -> Interp:
5369
+ def get_installed_version(self, version: InterpVersion) -> ta.Awaitable[Interp]:
4378
5370
  raise NotImplementedError
4379
5371
 
4380
- def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5372
+ async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
4381
5373
  return []
4382
5374
 
4383
- def install_version(self, version: InterpVersion) -> Interp:
5375
+ async def install_version(self, version: InterpVersion) -> Interp:
4384
5376
  raise TypeError
4385
5377
 
4386
5378
 
@@ -4392,10 +5384,10 @@ class RunningInterpProvider(InterpProvider):
4392
5384
  def version(self) -> InterpVersion:
4393
5385
  return InterpInspector.running().iv
4394
5386
 
4395
- def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5387
+ async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
4396
5388
  return [self.version()]
4397
5389
 
4398
- def get_installed_version(self, version: InterpVersion) -> Interp:
5390
+ async def get_installed_version(self, version: InterpVersion) -> Interp:
4399
5391
  if version != self.version():
4400
5392
  raise KeyError(version)
4401
5393
  return Interp(
@@ -4405,159 +5397,26 @@ class RunningInterpProvider(InterpProvider):
4405
5397
 
4406
5398
 
4407
5399
  ########################################
4408
- # ../remote/execution.py
4409
-
4410
-
4411
- ##
4412
-
4413
-
4414
- class _RemoteExecutionLogHandler(logging.Handler):
4415
- def __init__(self, fn: ta.Callable[[str], None]) -> None:
4416
- super().__init__()
4417
- self._fn = fn
4418
-
4419
- def emit(self, record):
4420
- msg = self.format(record)
4421
- self._fn(msg)
4422
-
4423
-
4424
- @dc.dataclass(frozen=True)
4425
- class _RemoteExecutionRequest:
4426
- c: Command
4427
-
4428
-
4429
- @dc.dataclass(frozen=True)
4430
- class _RemoteExecutionLog:
4431
- s: str
4432
-
4433
-
4434
- @dc.dataclass(frozen=True)
4435
- class _RemoteExecutionResponse:
4436
- r: ta.Optional[CommandOutputOrExceptionData] = None
4437
- l: ta.Optional[_RemoteExecutionLog] = None
4438
-
4439
-
4440
- def _remote_execution_main() -> None:
4441
- rt = pyremote_bootstrap_finalize() # noqa
4442
-
4443
- chan = RemoteChannel(
4444
- rt.input,
4445
- rt.output,
4446
- )
4447
-
4448
- bs = check_not_none(chan.recv_obj(MainBootstrap))
4449
-
4450
- if (prd := bs.remote_config.pycharm_remote_debug) is not None:
4451
- pycharm_debug_connect(prd)
4452
-
4453
- injector = main_bootstrap(bs)
4454
-
4455
- chan.set_marshaler(injector[ObjMarshalerManager])
4456
-
4457
- #
4458
-
4459
- log_lock = threading.RLock()
4460
- send_logs = False
4461
-
4462
- def log_fn(s: str) -> None:
4463
- with log_lock:
4464
- if send_logs:
4465
- chan.send_obj(_RemoteExecutionResponse(l=_RemoteExecutionLog(s)))
4466
-
4467
- log_handler = _RemoteExecutionLogHandler(log_fn)
4468
- logging.root.addHandler(log_handler)
4469
-
4470
- #
4471
-
4472
- ce = injector[LocalCommandExecutor]
4473
-
4474
- while True:
4475
- req = chan.recv_obj(_RemoteExecutionRequest)
4476
- if req is None:
4477
- break
4478
-
4479
- with log_lock:
4480
- send_logs = True
4481
-
4482
- r = ce.try_execute(
4483
- req.c,
4484
- log=log,
4485
- omit_exc_object=True,
4486
- )
4487
-
4488
- with log_lock:
4489
- send_logs = False
4490
-
4491
- chan.send_obj(_RemoteExecutionResponse(r=CommandOutputOrExceptionData(
4492
- output=r.output,
4493
- exception=r.exception,
4494
- )))
5400
+ # ../remote/connection.py
4495
5401
 
4496
5402
 
4497
5403
  ##
4498
5404
 
4499
5405
 
4500
- @dc.dataclass()
4501
- class RemoteCommandError(Exception):
4502
- e: CommandException
4503
-
4504
-
4505
- class RemoteCommandExecutor(CommandExecutor):
4506
- def __init__(self, chan: RemoteChannel) -> None:
4507
- super().__init__()
4508
-
4509
- self._chan = chan
4510
-
4511
- def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
4512
- self._chan.send_obj(_RemoteExecutionRequest(cmd))
4513
-
4514
- while True:
4515
- if (r := self._chan.recv_obj(_RemoteExecutionResponse)) is None:
4516
- raise EOFError
4517
-
4518
- if r.l is not None:
4519
- log.info(r.l.s)
4520
-
4521
- if r.r is not None:
4522
- return r.r
4523
-
4524
- # @ta.override
4525
- def execute(self, cmd: Command) -> Command.Output:
4526
- r = self._remote_execute(cmd)
4527
- if (e := r.exception) is not None:
4528
- raise RemoteCommandError(e)
4529
- else:
4530
- return check_not_none(r.output)
4531
-
4532
- # @ta.override
4533
- def try_execute(
5406
+ class RemoteExecutionConnector(abc.ABC):
5407
+ @abc.abstractmethod
5408
+ def connect(
4534
5409
  self,
4535
- cmd: Command,
4536
- *,
4537
- log: ta.Optional[logging.Logger] = None,
4538
- omit_exc_object: bool = False,
4539
- ) -> CommandOutputOrException:
4540
- try:
4541
- r = self._remote_execute(cmd)
4542
-
4543
- except Exception as e: # noqa
4544
- if log is not None:
4545
- log.exception('Exception executing remote command: %r', type(cmd))
4546
-
4547
- return CommandOutputOrExceptionData(exception=CommandException.of(
4548
- e,
4549
- omit_exc_object=omit_exc_object,
4550
- cmd=cmd,
4551
- ))
4552
-
4553
- else:
4554
- return r
5410
+ tgt: RemoteSpawning.Target,
5411
+ bs: MainBootstrap,
5412
+ ) -> ta.AsyncContextManager[RemoteCommandExecutor]:
5413
+ raise NotImplementedError
4555
5414
 
4556
5415
 
4557
5416
  ##
4558
5417
 
4559
5418
 
4560
- class RemoteExecution:
5419
+ class PyremoteRemoteExecutionConnector(RemoteExecutionConnector):
4561
5420
  def __init__(
4562
5421
  self,
4563
5422
  *,
@@ -4590,38 +5449,43 @@ class RemoteExecution:
4590
5449
 
4591
5450
  #
4592
5451
 
4593
- @contextlib.contextmanager
4594
- def connect(
5452
+ @contextlib.asynccontextmanager
5453
+ async def connect(
4595
5454
  self,
4596
5455
  tgt: RemoteSpawning.Target,
4597
5456
  bs: MainBootstrap,
4598
- ) -> ta.Generator[RemoteCommandExecutor, None, None]:
5457
+ ) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
4599
5458
  spawn_src = self._spawn_src()
4600
5459
  remote_src = self._remote_src()
4601
5460
 
4602
- with self._spawning.spawn(
5461
+ async with self._spawning.spawn(
4603
5462
  tgt,
4604
5463
  spawn_src,
5464
+ debug=bs.main_config.debug,
4605
5465
  ) as proc:
4606
- res = PyremoteBootstrapDriver( # noqa
5466
+ res = await PyremoteBootstrapDriver( # noqa
4607
5467
  remote_src,
4608
5468
  PyremoteBootstrapOptions(
4609
5469
  debug=bs.main_config.debug,
4610
5470
  ),
4611
- ).run(
5471
+ ).async_run(
4612
5472
  proc.stdout,
4613
5473
  proc.stdin,
4614
5474
  )
4615
5475
 
4616
- chan = RemoteChannel(
5476
+ chan = RemoteChannelImpl(
4617
5477
  proc.stdout,
4618
5478
  proc.stdin,
4619
5479
  msh=self._msh,
4620
5480
  )
4621
5481
 
4622
- chan.send_obj(bs)
5482
+ await chan.send_obj(bs)
5483
+
5484
+ rce: RemoteCommandExecutor
5485
+ async with contextlib.aclosing(RemoteCommandExecutor(chan)) as rce:
5486
+ await rce.start()
4623
5487
 
4624
- yield RemoteCommandExecutor(chan)
5488
+ yield rce
4625
5489
 
4626
5490
 
4627
5491
  ########################################
@@ -4643,7 +5507,6 @@ TODO:
4643
5507
 
4644
5508
 
4645
5509
  class Pyenv:
4646
-
4647
5510
  def __init__(
4648
5511
  self,
4649
5512
  *,
@@ -4656,13 +5519,13 @@ class Pyenv:
4656
5519
 
4657
5520
  self._root_kw = root
4658
5521
 
4659
- @cached_nullary
4660
- def root(self) -> ta.Optional[str]:
5522
+ @async_cached_nullary
5523
+ async def root(self) -> ta.Optional[str]:
4661
5524
  if self._root_kw is not None:
4662
5525
  return self._root_kw
4663
5526
 
4664
5527
  if shutil.which('pyenv'):
4665
- return subprocess_check_output_str('pyenv', 'root')
5528
+ return await asyncio_subprocess_check_output_str('pyenv', 'root')
4666
5529
 
4667
5530
  d = os.path.expanduser('~/.pyenv')
4668
5531
  if os.path.isdir(d) and os.path.isfile(os.path.join(d, 'bin', 'pyenv')):
@@ -4670,12 +5533,12 @@ class Pyenv:
4670
5533
 
4671
5534
  return None
4672
5535
 
4673
- @cached_nullary
4674
- def exe(self) -> str:
4675
- return os.path.join(check_not_none(self.root()), 'bin', 'pyenv')
5536
+ @async_cached_nullary
5537
+ async def exe(self) -> str:
5538
+ return os.path.join(check_not_none(await self.root()), 'bin', 'pyenv')
4676
5539
 
4677
- def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
4678
- if (root := self.root()) is None:
5540
+ async def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
5541
+ if (root := await self.root()) is None:
4679
5542
  return []
4680
5543
  ret = []
4681
5544
  vp = os.path.join(root, 'versions')
@@ -4687,11 +5550,11 @@ class Pyenv:
4687
5550
  ret.append((dn, ep))
4688
5551
  return ret
4689
5552
 
4690
- def installable_versions(self) -> ta.List[str]:
4691
- if self.root() is None:
5553
+ async def installable_versions(self) -> ta.List[str]:
5554
+ if await self.root() is None:
4692
5555
  return []
4693
5556
  ret = []
4694
- s = subprocess_check_output_str(self.exe(), 'install', '--list')
5557
+ s = await asyncio_subprocess_check_output_str(await self.exe(), 'install', '--list')
4695
5558
  for l in s.splitlines():
4696
5559
  if not l.startswith(' '):
4697
5560
  continue
@@ -4701,12 +5564,12 @@ class Pyenv:
4701
5564
  ret.append(l)
4702
5565
  return ret
4703
5566
 
4704
- def update(self) -> bool:
4705
- if (root := self.root()) is None:
5567
+ async def update(self) -> bool:
5568
+ if (root := await self.root()) is None:
4706
5569
  return False
4707
5570
  if not os.path.isdir(os.path.join(root, '.git')):
4708
5571
  return False
4709
- subprocess_check_call('git', 'pull', cwd=root)
5572
+ await asyncio_subprocess_check_call('git', 'pull', cwd=root)
4710
5573
  return True
4711
5574
 
4712
5575
 
@@ -4767,17 +5630,16 @@ THREADED_PYENV_INSTALL_OPTS = PyenvInstallOpts(conf_opts=['--disable-gil'])
4767
5630
 
4768
5631
  class PyenvInstallOptsProvider(abc.ABC):
4769
5632
  @abc.abstractmethod
4770
- def opts(self) -> PyenvInstallOpts:
5633
+ def opts(self) -> ta.Awaitable[PyenvInstallOpts]:
4771
5634
  raise NotImplementedError
4772
5635
 
4773
5636
 
4774
5637
  class LinuxPyenvInstallOpts(PyenvInstallOptsProvider):
4775
- def opts(self) -> PyenvInstallOpts:
5638
+ async def opts(self) -> PyenvInstallOpts:
4776
5639
  return PyenvInstallOpts()
4777
5640
 
4778
5641
 
4779
5642
  class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
4780
-
4781
5643
  @cached_nullary
4782
5644
  def framework_opts(self) -> PyenvInstallOpts:
4783
5645
  return PyenvInstallOpts(conf_opts=['--enable-framework'])
@@ -4793,12 +5655,12 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
4793
5655
  'zlib',
4794
5656
  ]
4795
5657
 
4796
- @cached_nullary
4797
- def brew_deps_opts(self) -> PyenvInstallOpts:
5658
+ @async_cached_nullary
5659
+ async def brew_deps_opts(self) -> PyenvInstallOpts:
4798
5660
  cflags = []
4799
5661
  ldflags = []
4800
5662
  for dep in self.BREW_DEPS:
4801
- dep_prefix = subprocess_check_output_str('brew', '--prefix', dep)
5663
+ dep_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', dep)
4802
5664
  cflags.append(f'-I{dep_prefix}/include')
4803
5665
  ldflags.append(f'-L{dep_prefix}/lib')
4804
5666
  return PyenvInstallOpts(
@@ -4806,13 +5668,13 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
4806
5668
  ldflags=ldflags,
4807
5669
  )
4808
5670
 
4809
- @cached_nullary
4810
- def brew_tcl_opts(self) -> PyenvInstallOpts:
4811
- if subprocess_try_output('brew', '--prefix', 'tcl-tk') is None:
5671
+ @async_cached_nullary
5672
+ async def brew_tcl_opts(self) -> PyenvInstallOpts:
5673
+ if await asyncio_subprocess_try_output('brew', '--prefix', 'tcl-tk') is None:
4812
5674
  return PyenvInstallOpts()
4813
5675
 
4814
- tcl_tk_prefix = subprocess_check_output_str('brew', '--prefix', 'tcl-tk')
4815
- tcl_tk_ver_str = subprocess_check_output_str('brew', 'ls', '--versions', 'tcl-tk')
5676
+ tcl_tk_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', 'tcl-tk')
5677
+ tcl_tk_ver_str = await asyncio_subprocess_check_output_str('brew', 'ls', '--versions', 'tcl-tk')
4816
5678
  tcl_tk_ver = '.'.join(tcl_tk_ver_str.split()[1].split('.')[:2])
4817
5679
 
4818
5680
  return PyenvInstallOpts(conf_opts=[
@@ -4827,11 +5689,11 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
4827
5689
  # pkg_config_path += ':' + os.environ['PKG_CONFIG_PATH']
4828
5690
  # return PyenvInstallOpts(env={'PKG_CONFIG_PATH': pkg_config_path})
4829
5691
 
4830
- def opts(self) -> PyenvInstallOpts:
5692
+ async def opts(self) -> PyenvInstallOpts:
4831
5693
  return PyenvInstallOpts().merge(
4832
5694
  self.framework_opts(),
4833
- self.brew_deps_opts(),
4834
- self.brew_tcl_opts(),
5695
+ await self.brew_deps_opts(),
5696
+ await self.brew_tcl_opts(),
4835
5697
  # self.brew_ssl_opts(),
4836
5698
  )
4837
5699
 
@@ -4863,20 +5725,8 @@ class PyenvVersionInstaller:
4863
5725
  ) -> None:
4864
5726
  super().__init__()
4865
5727
 
4866
- if no_default_opts:
4867
- if opts is None:
4868
- opts = PyenvInstallOpts()
4869
- else:
4870
- lst = [opts if opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
4871
- if interp_opts.debug:
4872
- lst.append(DEBUG_PYENV_INSTALL_OPTS)
4873
- if interp_opts.threaded:
4874
- lst.append(THREADED_PYENV_INSTALL_OPTS)
4875
- lst.append(PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
4876
- opts = PyenvInstallOpts().merge(*lst)
4877
-
4878
5728
  self._version = version
4879
- self._opts = opts
5729
+ self._given_opts = opts
4880
5730
  self._interp_opts = interp_opts
4881
5731
  self._given_install_name = install_name
4882
5732
 
@@ -4887,9 +5737,21 @@ class PyenvVersionInstaller:
4887
5737
  def version(self) -> str:
4888
5738
  return self._version
4889
5739
 
4890
- @property
4891
- def opts(self) -> PyenvInstallOpts:
4892
- return self._opts
5740
+ @async_cached_nullary
5741
+ async def opts(self) -> PyenvInstallOpts:
5742
+ opts = self._given_opts
5743
+ if self._no_default_opts:
5744
+ if opts is None:
5745
+ opts = PyenvInstallOpts()
5746
+ else:
5747
+ lst = [self._given_opts if self._given_opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
5748
+ if self._interp_opts.debug:
5749
+ lst.append(DEBUG_PYENV_INSTALL_OPTS)
5750
+ if self._interp_opts.threaded:
5751
+ lst.append(THREADED_PYENV_INSTALL_OPTS)
5752
+ lst.append(await PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
5753
+ opts = PyenvInstallOpts().merge(*lst)
5754
+ return opts
4893
5755
 
4894
5756
  @cached_nullary
4895
5757
  def install_name(self) -> str:
@@ -4897,17 +5759,18 @@ class PyenvVersionInstaller:
4897
5759
  return self._given_install_name
4898
5760
  return self._version + ('-debug' if self._interp_opts.debug else '')
4899
5761
 
4900
- @cached_nullary
4901
- def install_dir(self) -> str:
4902
- return str(os.path.join(check_not_none(self._pyenv.root()), 'versions', self.install_name()))
5762
+ @async_cached_nullary
5763
+ async def install_dir(self) -> str:
5764
+ return str(os.path.join(check_not_none(await self._pyenv.root()), 'versions', self.install_name()))
4903
5765
 
4904
- @cached_nullary
4905
- def install(self) -> str:
4906
- env = {**os.environ, **self._opts.env}
5766
+ @async_cached_nullary
5767
+ async def install(self) -> str:
5768
+ opts = await self.opts()
5769
+ env = {**os.environ, **opts.env}
4907
5770
  for k, l in [
4908
- ('CFLAGS', self._opts.cflags),
4909
- ('LDFLAGS', self._opts.ldflags),
4910
- ('PYTHON_CONFIGURE_OPTS', self._opts.conf_opts),
5771
+ ('CFLAGS', opts.cflags),
5772
+ ('LDFLAGS', opts.ldflags),
5773
+ ('PYTHON_CONFIGURE_OPTS', opts.conf_opts),
4911
5774
  ]:
4912
5775
  v = ' '.join(l)
4913
5776
  if k in os.environ:
@@ -4915,13 +5778,13 @@ class PyenvVersionInstaller:
4915
5778
  env[k] = v
4916
5779
 
4917
5780
  conf_args = [
4918
- *self._opts.opts,
5781
+ *opts.opts,
4919
5782
  self._version,
4920
5783
  ]
4921
5784
 
4922
5785
  if self._given_install_name is not None:
4923
5786
  full_args = [
4924
- os.path.join(check_not_none(self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'),
5787
+ os.path.join(check_not_none(await self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'), # noqa
4925
5788
  *conf_args,
4926
5789
  self.install_dir(),
4927
5790
  ]
@@ -4932,12 +5795,12 @@ class PyenvVersionInstaller:
4932
5795
  *conf_args,
4933
5796
  ]
4934
5797
 
4935
- subprocess_check_call(
5798
+ await asyncio_subprocess_check_call(
4936
5799
  *full_args,
4937
5800
  env=env,
4938
5801
  )
4939
5802
 
4940
- exe = os.path.join(self.install_dir(), 'bin', 'python')
5803
+ exe = os.path.join(await self.install_dir(), 'bin', 'python')
4941
5804
  if not os.path.isfile(exe):
4942
5805
  raise RuntimeError(f'Interpreter not found: {exe}')
4943
5806
  return exe
@@ -4947,7 +5810,6 @@ class PyenvVersionInstaller:
4947
5810
 
4948
5811
 
4949
5812
  class PyenvInterpProvider(InterpProvider):
4950
-
4951
5813
  def __init__(
4952
5814
  self,
4953
5815
  pyenv: Pyenv = Pyenv(),
@@ -4990,11 +5852,11 @@ class PyenvInterpProvider(InterpProvider):
4990
5852
  exe: str
4991
5853
  version: InterpVersion
4992
5854
 
4993
- def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
5855
+ async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
4994
5856
  iv: ta.Optional[InterpVersion]
4995
5857
  if self._inspect:
4996
5858
  try:
4997
- iv = check_not_none(self._inspector.inspect(ep)).iv
5859
+ iv = check_not_none(await self._inspector.inspect(ep)).iv
4998
5860
  except Exception as e: # noqa
4999
5861
  return None
5000
5862
  else:
@@ -5007,10 +5869,10 @@ class PyenvInterpProvider(InterpProvider):
5007
5869
  version=iv,
5008
5870
  )
5009
5871
 
5010
- def installed(self) -> ta.Sequence[Installed]:
5872
+ async def installed(self) -> ta.Sequence[Installed]:
5011
5873
  ret: ta.List[PyenvInterpProvider.Installed] = []
5012
- for vn, ep in self._pyenv.version_exes():
5013
- if (i := self._make_installed(vn, ep)) is None:
5874
+ for vn, ep in await self._pyenv.version_exes():
5875
+ if (i := await self._make_installed(vn, ep)) is None:
5014
5876
  log.debug('Invalid pyenv version: %s', vn)
5015
5877
  continue
5016
5878
  ret.append(i)
@@ -5018,11 +5880,11 @@ class PyenvInterpProvider(InterpProvider):
5018
5880
 
5019
5881
  #
5020
5882
 
5021
- def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5022
- return [i.version for i in self.installed()]
5883
+ async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5884
+ return [i.version for i in await self.installed()]
5023
5885
 
5024
- def get_installed_version(self, version: InterpVersion) -> Interp:
5025
- for i in self.installed():
5886
+ async def get_installed_version(self, version: InterpVersion) -> Interp:
5887
+ for i in await self.installed():
5026
5888
  if i.version == version:
5027
5889
  return Interp(
5028
5890
  exe=i.exe,
@@ -5032,10 +5894,10 @@ class PyenvInterpProvider(InterpProvider):
5032
5894
 
5033
5895
  #
5034
5896
 
5035
- def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5897
+ async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5036
5898
  lst = []
5037
5899
 
5038
- for vs in self._pyenv.installable_versions():
5900
+ for vs in await self._pyenv.installable_versions():
5039
5901
  if (iv := self.guess_version(vs)) is None:
5040
5902
  continue
5041
5903
  if iv.opts.debug:
@@ -5045,16 +5907,16 @@ class PyenvInterpProvider(InterpProvider):
5045
5907
 
5046
5908
  return lst
5047
5909
 
5048
- def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5049
- lst = self._get_installable_versions(spec)
5910
+ async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5911
+ lst = await self._get_installable_versions(spec)
5050
5912
 
5051
5913
  if self._try_update and not any(v in spec for v in lst):
5052
5914
  if self._pyenv.update():
5053
- lst = self._get_installable_versions(spec)
5915
+ lst = await self._get_installable_versions(spec)
5054
5916
 
5055
5917
  return lst
5056
5918
 
5057
- def install_version(self, version: InterpVersion) -> Interp:
5919
+ async def install_version(self, version: InterpVersion) -> Interp:
5058
5920
  inst_version = str(version.version)
5059
5921
  inst_opts = version.opts
5060
5922
  if inst_opts.threaded:
@@ -5066,7 +5928,7 @@ class PyenvInterpProvider(InterpProvider):
5066
5928
  interp_opts=inst_opts,
5067
5929
  )
5068
5930
 
5069
- exe = installer.install()
5931
+ exe = await installer.install()
5070
5932
  return Interp(exe, version)
5071
5933
 
5072
5934
 
@@ -5145,7 +6007,7 @@ class SystemInterpProvider(InterpProvider):
5145
6007
 
5146
6008
  #
5147
6009
 
5148
- def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
6010
+ async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
5149
6011
  if not self.inspect:
5150
6012
  s = os.path.basename(exe)
5151
6013
  if s.startswith('python'):
@@ -5155,13 +6017,13 @@ class SystemInterpProvider(InterpProvider):
5155
6017
  return InterpVersion.parse(s)
5156
6018
  except InvalidVersion:
5157
6019
  pass
5158
- ii = self.inspector.inspect(exe)
6020
+ ii = await self.inspector.inspect(exe)
5159
6021
  return ii.iv if ii is not None else None
5160
6022
 
5161
- def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
6023
+ async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
5162
6024
  lst = []
5163
6025
  for e in self.exes():
5164
- if (ev := self.get_exe_version(e)) is None:
6026
+ if (ev := await self.get_exe_version(e)) is None:
5165
6027
  log.debug('Invalid system version: %s', e)
5166
6028
  continue
5167
6029
  lst.append((e, ev))
@@ -5169,11 +6031,11 @@ class SystemInterpProvider(InterpProvider):
5169
6031
 
5170
6032
  #
5171
6033
 
5172
- def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
5173
- return [ev for e, ev in self.exe_versions()]
6034
+ async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
6035
+ return [ev for e, ev in await self.exe_versions()]
5174
6036
 
5175
- def get_installed_version(self, version: InterpVersion) -> Interp:
5176
- for e, ev in self.exe_versions():
6037
+ async def get_installed_version(self, version: InterpVersion) -> Interp:
6038
+ for e, ev in await self.exe_versions():
5177
6039
  if ev != version:
5178
6040
  continue
5179
6041
  return Interp(
@@ -5194,9 +6056,11 @@ def bind_remote(
5194
6056
  lst: ta.List[InjectorBindingOrBindings] = [
5195
6057
  inj.bind(remote_config),
5196
6058
 
5197
- inj.bind(RemoteSpawning, singleton=True),
6059
+ inj.bind(SubprocessRemoteSpawning, singleton=True),
6060
+ inj.bind(RemoteSpawning, to_key=SubprocessRemoteSpawning),
5198
6061
 
5199
- inj.bind(RemoteExecution, singleton=True),
6062
+ inj.bind(PyremoteRemoteExecutionConnector, singleton=True),
6063
+ inj.bind(RemoteExecutionConnector, to_key=PyremoteRemoteExecutionConnector),
5200
6064
  ]
5201
6065
 
5202
6066
  if (pf := remote_config.payload_file) is not None:
@@ -5220,13 +6084,14 @@ class InterpResolver:
5220
6084
  providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
5221
6085
  ) -> None:
5222
6086
  super().__init__()
6087
+
5223
6088
  self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
5224
6089
 
5225
- def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
6090
+ async def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
5226
6091
  lst = [
5227
6092
  (i, si)
5228
6093
  for i, p in enumerate(self._providers.values())
5229
- for si in p.get_installed_versions(spec)
6094
+ for si in await p.get_installed_versions(spec)
5230
6095
  if spec.contains(si)
5231
6096
  ]
5232
6097
 
@@ -5238,16 +6103,16 @@ class InterpResolver:
5238
6103
  bp = list(self._providers.values())[bi]
5239
6104
  return (bp, bv)
5240
6105
 
5241
- def resolve(
6106
+ async def resolve(
5242
6107
  self,
5243
6108
  spec: InterpSpecifier,
5244
6109
  *,
5245
6110
  install: bool = False,
5246
6111
  ) -> ta.Optional[Interp]:
5247
- tup = self._resolve_installed(spec)
6112
+ tup = await self._resolve_installed(spec)
5248
6113
  if tup is not None:
5249
6114
  bp, bv = tup
5250
- return bp.get_installed_version(bv)
6115
+ return await bp.get_installed_version(bv)
5251
6116
 
5252
6117
  if not install:
5253
6118
  return None
@@ -5255,21 +6120,21 @@ class InterpResolver:
5255
6120
  tp = list(self._providers.values())[0] # noqa
5256
6121
 
5257
6122
  sv = sorted(
5258
- [s for s in tp.get_installable_versions(spec) if s in spec],
6123
+ [s for s in await tp.get_installable_versions(spec) if s in spec],
5259
6124
  key=lambda s: s.version,
5260
6125
  )
5261
6126
  if not sv:
5262
6127
  return None
5263
6128
 
5264
6129
  bv = sv[-1]
5265
- return tp.install_version(bv)
6130
+ return await tp.install_version(bv)
5266
6131
 
5267
- def list(self, spec: InterpSpecifier) -> None:
6132
+ async def list(self, spec: InterpSpecifier) -> None:
5268
6133
  print('installed:')
5269
6134
  for n, p in self._providers.items():
5270
6135
  lst = [
5271
6136
  si
5272
- for si in p.get_installed_versions(spec)
6137
+ for si in await p.get_installed_versions(spec)
5273
6138
  if spec.contains(si)
5274
6139
  ]
5275
6140
  if lst:
@@ -5283,7 +6148,7 @@ class InterpResolver:
5283
6148
  for n, p in self._providers.items():
5284
6149
  lst = [
5285
6150
  si
5286
- for si in p.get_installable_versions(spec)
6151
+ for si in await p.get_installable_versions(spec)
5287
6152
  if spec.contains(si)
5288
6153
  ]
5289
6154
  if lst:
@@ -5325,9 +6190,9 @@ class InterpCommand(Command['InterpCommand.Output']):
5325
6190
 
5326
6191
 
5327
6192
  class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
5328
- def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
6193
+ async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
5329
6194
  i = InterpSpecifier.parse(check_not_none(cmd.spec))
5330
- o = check_not_none(DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
6195
+ o = check_not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
5331
6196
  return InterpCommand.Output(
5332
6197
  exe=o.exe,
5333
6198
  version=str(o.version.version),
@@ -5366,7 +6231,7 @@ def bind_command(
5366
6231
  class _FactoryCommandExecutor(CommandExecutor):
5367
6232
  factory: ta.Callable[[], CommandExecutor]
5368
6233
 
5369
- def execute(self, i: Command) -> Command.Output:
6234
+ def execute(self, i: Command) -> ta.Awaitable[Command.Output]:
5370
6235
  return self.factory().execute(i)
5371
6236
 
5372
6237
 
@@ -5521,31 +6386,7 @@ def main_bootstrap(bs: MainBootstrap) -> Injector:
5521
6386
  ##
5522
6387
 
5523
6388
 
5524
- def _main() -> None:
5525
- import argparse
5526
-
5527
- parser = argparse.ArgumentParser()
5528
-
5529
- parser.add_argument('--_payload-file')
5530
-
5531
- parser.add_argument('-s', '--shell')
5532
- parser.add_argument('-q', '--shell-quote', action='store_true')
5533
- parser.add_argument('--python', default='python3')
5534
-
5535
- parser.add_argument('--pycharm-debug-port', type=int)
5536
- parser.add_argument('--pycharm-debug-host')
5537
- parser.add_argument('--pycharm-debug-version')
5538
-
5539
- parser.add_argument('--debug', action='store_true')
5540
-
5541
- parser.add_argument('--local', action='store_true')
5542
-
5543
- parser.add_argument('command', nargs='+')
5544
-
5545
- args = parser.parse_args()
5546
-
5547
- #
5548
-
6389
+ async def _async_main(args: ta.Any) -> None:
5549
6390
  bs = MainBootstrap(
5550
6391
  main_config=MainConfig(
5551
6392
  log_level='DEBUG' if args.debug else 'INFO',
@@ -5558,12 +6399,16 @@ def _main() -> None:
5558
6399
 
5559
6400
  pycharm_remote_debug=PycharmRemoteDebug(
5560
6401
  port=args.pycharm_debug_port,
5561
- host=args.pycharm_debug_host,
6402
+ **(dict(host=args.pycharm_debug_host) if args.pycharm_debug_host is not None else {}),
5562
6403
  install_version=args.pycharm_debug_version,
5563
6404
  ) if args.pycharm_debug_port is not None else None,
6405
+
6406
+ timebomb_delay_s=args.remote_timebomb_delay_s,
5564
6407
  ),
5565
6408
  )
5566
6409
 
6410
+ #
6411
+
5567
6412
  injector = main_bootstrap(
5568
6413
  bs,
5569
6414
  )
@@ -5582,7 +6427,7 @@ def _main() -> None:
5582
6427
 
5583
6428
  #
5584
6429
 
5585
- with contextlib.ExitStack() as es:
6430
+ async with contextlib.AsyncExitStack() as es:
5586
6431
  ce: CommandExecutor
5587
6432
 
5588
6433
  if args.local:
@@ -5595,16 +6440,51 @@ def _main() -> None:
5595
6440
  python=args.python,
5596
6441
  )
5597
6442
 
5598
- ce = es.enter_context(injector[RemoteExecution].connect(tgt, bs)) # noqa
6443
+ ce = await es.enter_async_context(injector[RemoteExecutionConnector].connect(tgt, bs)) # noqa
5599
6444
 
5600
- for cmd in cmds:
5601
- r = ce.try_execute(
6445
+ async def run_command(cmd: Command) -> None:
6446
+ res = await ce.try_execute(
5602
6447
  cmd,
5603
6448
  log=log,
5604
6449
  omit_exc_object=True,
5605
6450
  )
5606
6451
 
5607
- print(msh.marshal_obj(r, opts=ObjMarshalOptions(raw_bytes=True)))
6452
+ print(msh.marshal_obj(res, opts=ObjMarshalOptions(raw_bytes=True)))
6453
+
6454
+ await asyncio.gather(*[
6455
+ run_command(cmd)
6456
+ for cmd in cmds
6457
+ ])
6458
+
6459
+
6460
+ def _main() -> None:
6461
+ import argparse
6462
+
6463
+ parser = argparse.ArgumentParser()
6464
+
6465
+ parser.add_argument('--_payload-file')
6466
+
6467
+ parser.add_argument('-s', '--shell')
6468
+ parser.add_argument('-q', '--shell-quote', action='store_true')
6469
+ parser.add_argument('--python', default='python3')
6470
+
6471
+ parser.add_argument('--pycharm-debug-port', type=int)
6472
+ parser.add_argument('--pycharm-debug-host')
6473
+ parser.add_argument('--pycharm-debug-version')
6474
+
6475
+ parser.add_argument('--remote-timebomb-delay-s', type=float)
6476
+
6477
+ parser.add_argument('--debug', action='store_true')
6478
+
6479
+ parser.add_argument('--local', action='store_true')
6480
+
6481
+ parser.add_argument('command', nargs='+')
6482
+
6483
+ args = parser.parse_args()
6484
+
6485
+ #
6486
+
6487
+ asyncio.run(_async_main(args))
5608
6488
 
5609
6489
 
5610
6490
  if __name__ == '__main__':