ominfra 0.0.0.dev155__py3-none-any.whl → 0.0.0.dev156__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/manage/commands/subprocess.py +3 -4
- ominfra/manage/deploy/git.py +3 -3
- ominfra/manage/deploy/venvs.py +4 -4
- ominfra/manage/remote/spawning.py +2 -2
- ominfra/manage/system/packages.py +13 -15
- ominfra/scripts/journald2aws.py +181 -127
- ominfra/scripts/manage.py +330 -314
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/RECORD +13 -13
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -5788,168 +5788,222 @@ SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
|
|
5788
5788
|
_SUBPROCESS_SHELL_WRAP_EXECS = False
|
5789
5789
|
|
5790
5790
|
|
5791
|
-
def subprocess_shell_wrap_exec(*
|
5792
|
-
return ('sh', '-c', ' '.join(map(shlex.quote,
|
5791
|
+
def subprocess_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
|
5792
|
+
return ('sh', '-c', ' '.join(map(shlex.quote, cmd)))
|
5793
5793
|
|
5794
5794
|
|
5795
|
-
def subprocess_maybe_shell_wrap_exec(*
|
5795
|
+
def subprocess_maybe_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
|
5796
5796
|
if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
|
5797
|
-
return subprocess_shell_wrap_exec(*
|
5797
|
+
return subprocess_shell_wrap_exec(*cmd)
|
5798
5798
|
else:
|
5799
|
-
return
|
5800
|
-
|
5801
|
-
|
5802
|
-
def prepare_subprocess_invocation(
|
5803
|
-
*args: str,
|
5804
|
-
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
5805
|
-
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
5806
|
-
quiet: bool = False,
|
5807
|
-
shell: bool = False,
|
5808
|
-
**kwargs: ta.Any,
|
5809
|
-
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
5810
|
-
log.debug('prepare_subprocess_invocation: args=%r', args)
|
5811
|
-
if extra_env:
|
5812
|
-
log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
|
5813
|
-
|
5814
|
-
if extra_env:
|
5815
|
-
env = {**(env if env is not None else os.environ), **extra_env}
|
5816
|
-
|
5817
|
-
if quiet and 'stderr' not in kwargs:
|
5818
|
-
if not log.isEnabledFor(logging.DEBUG):
|
5819
|
-
kwargs['stderr'] = subprocess.DEVNULL
|
5820
|
-
|
5821
|
-
if not shell:
|
5822
|
-
args = subprocess_maybe_shell_wrap_exec(*args)
|
5823
|
-
|
5824
|
-
return args, dict(
|
5825
|
-
env=env,
|
5826
|
-
shell=shell,
|
5827
|
-
**kwargs,
|
5828
|
-
)
|
5799
|
+
return cmd
|
5829
5800
|
|
5830
5801
|
|
5831
5802
|
##
|
5832
5803
|
|
5833
5804
|
|
5834
|
-
|
5835
|
-
|
5836
|
-
|
5837
|
-
|
5838
|
-
|
5839
|
-
|
5840
|
-
|
5841
|
-
|
5842
|
-
|
5843
|
-
|
5805
|
+
def subprocess_close(
|
5806
|
+
proc: subprocess.Popen,
|
5807
|
+
timeout: ta.Optional[float] = None,
|
5808
|
+
) -> None:
|
5809
|
+
# TODO: terminate, sleep, kill
|
5810
|
+
if proc.stdout:
|
5811
|
+
proc.stdout.close()
|
5812
|
+
if proc.stderr:
|
5813
|
+
proc.stderr.close()
|
5814
|
+
if proc.stdin:
|
5815
|
+
proc.stdin.close()
|
5844
5816
|
|
5845
|
-
|
5846
|
-
end_time = time.time()
|
5847
|
-
elapsed_s = end_time - start_time
|
5848
|
-
log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
|
5817
|
+
proc.wait(timeout)
|
5849
5818
|
|
5850
5819
|
|
5851
5820
|
##
|
5852
5821
|
|
5853
5822
|
|
5854
|
-
|
5855
|
-
|
5856
|
-
stdout: ta.Any = sys.stderr,
|
5857
|
-
**kwargs: ta.Any,
|
5858
|
-
) -> None:
|
5859
|
-
args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
|
5860
|
-
with subprocess_common_context(*args, **kwargs):
|
5861
|
-
return subprocess.check_call(args, **kwargs) # type: ignore
|
5823
|
+
class AbstractSubprocesses(abc.ABC): # noqa
|
5824
|
+
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = log
|
5862
5825
|
|
5826
|
+
def __init__(
|
5827
|
+
self,
|
5828
|
+
*,
|
5829
|
+
log: ta.Optional[logging.Logger] = None,
|
5830
|
+
try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
|
5831
|
+
) -> None:
|
5832
|
+
super().__init__()
|
5863
5833
|
|
5864
|
-
|
5865
|
-
|
5866
|
-
**kwargs: ta.Any,
|
5867
|
-
) -> bytes:
|
5868
|
-
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
5869
|
-
with subprocess_common_context(*args, **kwargs):
|
5870
|
-
return subprocess.check_output(args, **kwargs)
|
5834
|
+
self._log = log if log is not None else self.DEFAULT_LOGGER
|
5835
|
+
self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
|
5871
5836
|
|
5837
|
+
#
|
5872
5838
|
|
5873
|
-
def
|
5874
|
-
|
5839
|
+
def prepare_args(
|
5840
|
+
self,
|
5841
|
+
*cmd: str,
|
5842
|
+
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
5843
|
+
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
5844
|
+
quiet: bool = False,
|
5845
|
+
shell: bool = False,
|
5846
|
+
**kwargs: ta.Any,
|
5847
|
+
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
5848
|
+
if self._log:
|
5849
|
+
self._log.debug('Subprocesses.prepare_args: cmd=%r', cmd)
|
5850
|
+
if extra_env:
|
5851
|
+
self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
|
5875
5852
|
|
5853
|
+
if extra_env:
|
5854
|
+
env = {**(env if env is not None else os.environ), **extra_env}
|
5876
5855
|
|
5877
|
-
|
5856
|
+
if quiet and 'stderr' not in kwargs:
|
5857
|
+
if self._log and not self._log.isEnabledFor(logging.DEBUG):
|
5858
|
+
kwargs['stderr'] = subprocess.DEVNULL
|
5878
5859
|
|
5860
|
+
if not shell:
|
5861
|
+
cmd = subprocess_maybe_shell_wrap_exec(*cmd)
|
5879
5862
|
|
5880
|
-
|
5881
|
-
|
5882
|
-
|
5883
|
-
|
5863
|
+
return cmd, dict(
|
5864
|
+
env=env,
|
5865
|
+
shell=shell,
|
5866
|
+
**kwargs,
|
5867
|
+
)
|
5884
5868
|
|
5869
|
+
@contextlib.contextmanager
|
5870
|
+
def wrap_call(self, *cmd: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
|
5871
|
+
start_time = time.time()
|
5872
|
+
try:
|
5873
|
+
if self._log:
|
5874
|
+
self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
|
5875
|
+
yield
|
5885
5876
|
|
5886
|
-
|
5887
|
-
|
5888
|
-
|
5889
|
-
|
5890
|
-
**kwargs: ta.Any,
|
5891
|
-
) -> ta.Union[T, Exception]:
|
5892
|
-
try:
|
5893
|
-
return fn(*args, **kwargs)
|
5894
|
-
except try_exceptions as e: # noqa
|
5895
|
-
if log.isEnabledFor(logging.DEBUG):
|
5896
|
-
log.exception('command failed')
|
5897
|
-
return e
|
5898
|
-
|
5899
|
-
|
5900
|
-
def subprocess_try_call(
|
5901
|
-
*args: str,
|
5902
|
-
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
5903
|
-
**kwargs: ta.Any,
|
5904
|
-
) -> bool:
|
5905
|
-
if isinstance(_subprocess_try_run(
|
5906
|
-
subprocess_check_call,
|
5907
|
-
*args,
|
5908
|
-
try_exceptions=try_exceptions,
|
5909
|
-
**kwargs,
|
5910
|
-
), Exception):
|
5911
|
-
return False
|
5912
|
-
else:
|
5913
|
-
return True
|
5877
|
+
except Exception as exc: # noqa
|
5878
|
+
if self._log:
|
5879
|
+
self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
|
5880
|
+
raise
|
5914
5881
|
|
5882
|
+
finally:
|
5883
|
+
end_time = time.time()
|
5884
|
+
elapsed_s = end_time - start_time
|
5885
|
+
if self._log:
|
5886
|
+
self._log.debug('sSubprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
|
5915
5887
|
|
5916
|
-
|
5917
|
-
|
5918
|
-
|
5919
|
-
|
5920
|
-
|
5921
|
-
|
5922
|
-
|
5923
|
-
|
5924
|
-
|
5925
|
-
|
5926
|
-
|
5927
|
-
|
5928
|
-
else:
|
5929
|
-
return ret
|
5888
|
+
@contextlib.contextmanager
|
5889
|
+
def prepare_and_wrap(
|
5890
|
+
self,
|
5891
|
+
*cmd: ta.Any,
|
5892
|
+
**kwargs: ta.Any,
|
5893
|
+
) -> ta.Iterator[ta.Tuple[
|
5894
|
+
ta.Tuple[ta.Any, ...],
|
5895
|
+
ta.Dict[str, ta.Any],
|
5896
|
+
]]:
|
5897
|
+
cmd, kwargs = self.prepare_args(*cmd, **kwargs)
|
5898
|
+
with self.wrap_call(*cmd, **kwargs):
|
5899
|
+
yield cmd, kwargs
|
5930
5900
|
|
5901
|
+
#
|
5902
|
+
|
5903
|
+
DEFAULT_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
|
5904
|
+
FileNotFoundError,
|
5905
|
+
subprocess.CalledProcessError,
|
5906
|
+
)
|
5907
|
+
|
5908
|
+
def try_fn(
|
5909
|
+
self,
|
5910
|
+
fn: ta.Callable[..., T],
|
5911
|
+
*cmd: str,
|
5912
|
+
try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
|
5913
|
+
**kwargs: ta.Any,
|
5914
|
+
) -> ta.Union[T, Exception]:
|
5915
|
+
if try_exceptions is None:
|
5916
|
+
try_exceptions = self._try_exceptions
|
5917
|
+
|
5918
|
+
try:
|
5919
|
+
return fn(*cmd, **kwargs)
|
5920
|
+
|
5921
|
+
except try_exceptions as e: # noqa
|
5922
|
+
if self._log and self._log.isEnabledFor(logging.DEBUG):
|
5923
|
+
self._log.exception('command failed')
|
5924
|
+
return e
|
5925
|
+
|
5926
|
+
async def async_try_fn(
|
5927
|
+
self,
|
5928
|
+
fn: ta.Callable[..., ta.Awaitable[T]],
|
5929
|
+
*cmd: ta.Any,
|
5930
|
+
try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
|
5931
|
+
**kwargs: ta.Any,
|
5932
|
+
) -> ta.Union[T, Exception]:
|
5933
|
+
if try_exceptions is None:
|
5934
|
+
try_exceptions = self._try_exceptions
|
5931
5935
|
|
5932
|
-
|
5933
|
-
|
5934
|
-
|
5936
|
+
try:
|
5937
|
+
return await fn(*cmd, **kwargs)
|
5938
|
+
|
5939
|
+
except try_exceptions as e: # noqa
|
5940
|
+
if self._log and self._log.isEnabledFor(logging.DEBUG):
|
5941
|
+
self._log.exception('command failed')
|
5942
|
+
return e
|
5935
5943
|
|
5936
5944
|
|
5937
5945
|
##
|
5938
5946
|
|
5939
5947
|
|
5940
|
-
|
5941
|
-
|
5942
|
-
|
5943
|
-
|
5944
|
-
|
5945
|
-
|
5946
|
-
|
5947
|
-
|
5948
|
-
|
5949
|
-
if proc.stdin:
|
5950
|
-
proc.stdin.close()
|
5948
|
+
class Subprocesses(AbstractSubprocesses):
|
5949
|
+
def check_call(
|
5950
|
+
self,
|
5951
|
+
*cmd: str,
|
5952
|
+
stdout: ta.Any = sys.stderr,
|
5953
|
+
**kwargs: ta.Any,
|
5954
|
+
) -> None:
|
5955
|
+
with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
|
5956
|
+
subprocess.check_call(cmd, **kwargs)
|
5951
5957
|
|
5952
|
-
|
5958
|
+
def check_output(
|
5959
|
+
self,
|
5960
|
+
*cmd: str,
|
5961
|
+
**kwargs: ta.Any,
|
5962
|
+
) -> bytes:
|
5963
|
+
with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
|
5964
|
+
return subprocess.check_output(cmd, **kwargs)
|
5965
|
+
|
5966
|
+
def check_output_str(
|
5967
|
+
self,
|
5968
|
+
*cmd: str,
|
5969
|
+
**kwargs: ta.Any,
|
5970
|
+
) -> str:
|
5971
|
+
return self.check_output(*cmd, **kwargs).decode().strip()
|
5972
|
+
|
5973
|
+
#
|
5974
|
+
|
5975
|
+
def try_call(
|
5976
|
+
self,
|
5977
|
+
*cmd: str,
|
5978
|
+
**kwargs: ta.Any,
|
5979
|
+
) -> bool:
|
5980
|
+
if isinstance(self.try_fn(self.check_call, *cmd, **kwargs), Exception):
|
5981
|
+
return False
|
5982
|
+
else:
|
5983
|
+
return True
|
5984
|
+
|
5985
|
+
def try_output(
|
5986
|
+
self,
|
5987
|
+
*cmd: str,
|
5988
|
+
**kwargs: ta.Any,
|
5989
|
+
) -> ta.Optional[bytes]:
|
5990
|
+
if isinstance(ret := self.try_fn(self.check_output, *cmd, **kwargs), Exception):
|
5991
|
+
return None
|
5992
|
+
else:
|
5993
|
+
return ret
|
5994
|
+
|
5995
|
+
def try_output_str(
|
5996
|
+
self,
|
5997
|
+
*cmd: str,
|
5998
|
+
**kwargs: ta.Any,
|
5999
|
+
) -> ta.Optional[str]:
|
6000
|
+
if (ret := self.try_output(*cmd, **kwargs)) is None:
|
6001
|
+
return None
|
6002
|
+
else:
|
6003
|
+
return ret.decode().strip()
|
6004
|
+
|
6005
|
+
|
6006
|
+
subprocesses = Subprocesses()
|
5953
6007
|
|
5954
6008
|
|
5955
6009
|
########################################
|
@@ -6384,43 +6438,6 @@ class SystemConfig:
|
|
6384
6438
|
##
|
6385
6439
|
|
6386
6440
|
|
6387
|
-
@contextlib.asynccontextmanager
|
6388
|
-
async def asyncio_subprocess_popen(
|
6389
|
-
*cmd: str,
|
6390
|
-
shell: bool = False,
|
6391
|
-
timeout: ta.Optional[float] = None,
|
6392
|
-
**kwargs: ta.Any,
|
6393
|
-
) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
|
6394
|
-
fac: ta.Any
|
6395
|
-
if shell:
|
6396
|
-
fac = functools.partial(
|
6397
|
-
asyncio.create_subprocess_shell,
|
6398
|
-
check.single(cmd),
|
6399
|
-
)
|
6400
|
-
else:
|
6401
|
-
fac = functools.partial(
|
6402
|
-
asyncio.create_subprocess_exec,
|
6403
|
-
*cmd,
|
6404
|
-
)
|
6405
|
-
|
6406
|
-
with subprocess_common_context(
|
6407
|
-
*cmd,
|
6408
|
-
shell=shell,
|
6409
|
-
timeout=timeout,
|
6410
|
-
**kwargs,
|
6411
|
-
):
|
6412
|
-
proc: asyncio.subprocess.Process
|
6413
|
-
proc = await fac(**kwargs)
|
6414
|
-
try:
|
6415
|
-
yield proc
|
6416
|
-
|
6417
|
-
finally:
|
6418
|
-
await asyncio_maybe_timeout(proc.wait(), timeout)
|
6419
|
-
|
6420
|
-
|
6421
|
-
##
|
6422
|
-
|
6423
|
-
|
6424
6441
|
class AsyncioProcessCommunicator:
|
6425
6442
|
def __init__(
|
6426
6443
|
self,
|
@@ -6531,148 +6548,147 @@ class AsyncioProcessCommunicator:
|
|
6531
6548
|
return await asyncio_maybe_timeout(self._communicate(input), timeout)
|
6532
6549
|
|
6533
6550
|
|
6534
|
-
|
6535
|
-
proc: asyncio.subprocess.Process,
|
6536
|
-
input: ta.Any = None, # noqa
|
6537
|
-
timeout: ta.Optional[float] = None,
|
6538
|
-
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
6539
|
-
return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
|
6540
|
-
|
6551
|
+
##
|
6541
6552
|
|
6542
|
-
@dc.dataclass(frozen=True)
|
6543
|
-
class AsyncioSubprocessOutput:
|
6544
|
-
proc: asyncio.subprocess.Process
|
6545
|
-
stdout: ta.Optional[bytes]
|
6546
|
-
stderr: ta.Optional[bytes]
|
6547
6553
|
|
6554
|
+
class AsyncioSubprocesses(AbstractSubprocesses):
|
6555
|
+
async def communicate(
|
6556
|
+
self,
|
6557
|
+
proc: asyncio.subprocess.Process,
|
6558
|
+
input: ta.Any = None, # noqa
|
6559
|
+
timeout: ta.Optional[float] = None,
|
6560
|
+
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
6561
|
+
return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
|
6548
6562
|
|
6549
|
-
|
6550
|
-
*args: str,
|
6551
|
-
input: ta.Any = None, # noqa
|
6552
|
-
timeout: ta.Optional[float] = None,
|
6553
|
-
check: bool = False, # noqa
|
6554
|
-
capture_output: ta.Optional[bool] = None,
|
6555
|
-
**kwargs: ta.Any,
|
6556
|
-
) -> AsyncioSubprocessOutput:
|
6557
|
-
if capture_output:
|
6558
|
-
kwargs.setdefault('stdout', subprocess.PIPE)
|
6559
|
-
kwargs.setdefault('stderr', subprocess.PIPE)
|
6560
|
-
|
6561
|
-
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
6562
|
-
|
6563
|
-
proc: asyncio.subprocess.Process
|
6564
|
-
async with asyncio_subprocess_popen(*args, **kwargs) as proc:
|
6565
|
-
stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
|
6566
|
-
|
6567
|
-
if check and proc.returncode:
|
6568
|
-
raise subprocess.CalledProcessError(
|
6569
|
-
proc.returncode,
|
6570
|
-
args,
|
6571
|
-
output=stdout,
|
6572
|
-
stderr=stderr,
|
6573
|
-
)
|
6563
|
+
#
|
6574
6564
|
|
6575
|
-
|
6576
|
-
|
6577
|
-
|
6578
|
-
|
6579
|
-
|
6565
|
+
@contextlib.asynccontextmanager
|
6566
|
+
async def popen(
|
6567
|
+
self,
|
6568
|
+
*cmd: str,
|
6569
|
+
shell: bool = False,
|
6570
|
+
timeout: ta.Optional[float] = None,
|
6571
|
+
**kwargs: ta.Any,
|
6572
|
+
) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
|
6573
|
+
fac: ta.Any
|
6574
|
+
if shell:
|
6575
|
+
fac = functools.partial(
|
6576
|
+
asyncio.create_subprocess_shell,
|
6577
|
+
check.single(cmd),
|
6578
|
+
)
|
6579
|
+
else:
|
6580
|
+
fac = functools.partial(
|
6581
|
+
asyncio.create_subprocess_exec,
|
6582
|
+
*cmd,
|
6583
|
+
)
|
6580
6584
|
|
6585
|
+
with self.prepare_and_wrap( *cmd, shell=shell, **kwargs) as (cmd, kwargs): # noqa
|
6586
|
+
proc: asyncio.subprocess.Process = await fac(**kwargs)
|
6587
|
+
try:
|
6588
|
+
yield proc
|
6581
6589
|
|
6582
|
-
|
6590
|
+
finally:
|
6591
|
+
await asyncio_maybe_timeout(proc.wait(), timeout)
|
6583
6592
|
|
6593
|
+
#
|
6584
6594
|
|
6585
|
-
|
6586
|
-
|
6587
|
-
|
6588
|
-
|
6589
|
-
|
6590
|
-
**kwargs: ta.Any,
|
6591
|
-
) -> None:
|
6592
|
-
await asyncio_subprocess_run(
|
6593
|
-
*args,
|
6594
|
-
stdout=stdout,
|
6595
|
-
input=input,
|
6596
|
-
timeout=timeout,
|
6597
|
-
check=True,
|
6598
|
-
**kwargs,
|
6599
|
-
)
|
6595
|
+
@dc.dataclass(frozen=True)
|
6596
|
+
class RunOutput:
|
6597
|
+
proc: asyncio.subprocess.Process
|
6598
|
+
stdout: ta.Optional[bytes]
|
6599
|
+
stderr: ta.Optional[bytes]
|
6600
6600
|
|
6601
|
+
async def run(
|
6602
|
+
self,
|
6603
|
+
*cmd: str,
|
6604
|
+
input: ta.Any = None, # noqa
|
6605
|
+
timeout: ta.Optional[float] = None,
|
6606
|
+
check: bool = False, # noqa
|
6607
|
+
capture_output: ta.Optional[bool] = None,
|
6608
|
+
**kwargs: ta.Any,
|
6609
|
+
) -> RunOutput:
|
6610
|
+
if capture_output:
|
6611
|
+
kwargs.setdefault('stdout', subprocess.PIPE)
|
6612
|
+
kwargs.setdefault('stderr', subprocess.PIPE)
|
6601
6613
|
|
6602
|
-
|
6603
|
-
*
|
6604
|
-
|
6605
|
-
|
6606
|
-
|
6607
|
-
|
6608
|
-
|
6609
|
-
|
6610
|
-
|
6611
|
-
|
6612
|
-
|
6613
|
-
check=True,
|
6614
|
-
**kwargs,
|
6615
|
-
)
|
6614
|
+
proc: asyncio.subprocess.Process
|
6615
|
+
async with self.popen(*cmd, **kwargs) as proc:
|
6616
|
+
stdout, stderr = await self.communicate(proc, input, timeout)
|
6617
|
+
|
6618
|
+
if check and proc.returncode:
|
6619
|
+
raise subprocess.CalledProcessError(
|
6620
|
+
proc.returncode,
|
6621
|
+
cmd,
|
6622
|
+
output=stdout,
|
6623
|
+
stderr=stderr,
|
6624
|
+
)
|
6616
6625
|
|
6617
|
-
|
6626
|
+
return self.RunOutput(
|
6627
|
+
proc,
|
6628
|
+
stdout,
|
6629
|
+
stderr,
|
6630
|
+
)
|
6618
6631
|
|
6632
|
+
#
|
6619
6633
|
|
6620
|
-
async def
|
6621
|
-
|
6634
|
+
async def check_call(
|
6635
|
+
self,
|
6636
|
+
*cmd: str,
|
6637
|
+
stdout: ta.Any = sys.stderr,
|
6638
|
+
**kwargs: ta.Any,
|
6639
|
+
) -> None:
|
6640
|
+
with self.prepare_and_wrap(*cmd, stdout=stdout, check=True, **kwargs) as (cmd, kwargs): # noqa
|
6641
|
+
await self.run(*cmd, **kwargs)
|
6622
6642
|
|
6643
|
+
async def check_output(
|
6644
|
+
self,
|
6645
|
+
*cmd: str,
|
6646
|
+
**kwargs: ta.Any,
|
6647
|
+
) -> bytes:
|
6648
|
+
with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs): # noqa
|
6649
|
+
return check.not_none((await self.run(*cmd, **kwargs)).stdout)
|
6623
6650
|
|
6624
|
-
|
6651
|
+
async def check_output_str(
|
6652
|
+
self,
|
6653
|
+
*cmd: str,
|
6654
|
+
**kwargs: ta.Any,
|
6655
|
+
) -> str:
|
6656
|
+
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
6625
6657
|
|
6658
|
+
#
|
6626
6659
|
|
6627
|
-
async def
|
6628
|
-
|
6629
|
-
|
6630
|
-
|
6631
|
-
|
6632
|
-
|
6633
|
-
|
6634
|
-
|
6635
|
-
|
6636
|
-
if log.isEnabledFor(logging.DEBUG):
|
6637
|
-
log.exception('command failed')
|
6638
|
-
return e
|
6639
|
-
|
6640
|
-
|
6641
|
-
async def asyncio_subprocess_try_call(
|
6642
|
-
*args: str,
|
6643
|
-
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
6644
|
-
**kwargs: ta.Any,
|
6645
|
-
) -> bool:
|
6646
|
-
if isinstance(await _asyncio_subprocess_try_run(
|
6647
|
-
asyncio_subprocess_check_call,
|
6648
|
-
*args,
|
6649
|
-
try_exceptions=try_exceptions,
|
6650
|
-
**kwargs,
|
6651
|
-
), Exception):
|
6652
|
-
return False
|
6653
|
-
else:
|
6654
|
-
return True
|
6660
|
+
async def try_call(
|
6661
|
+
self,
|
6662
|
+
*cmd: str,
|
6663
|
+
**kwargs: ta.Any,
|
6664
|
+
) -> bool:
|
6665
|
+
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
6666
|
+
return False
|
6667
|
+
else:
|
6668
|
+
return True
|
6655
6669
|
|
6670
|
+
async def try_output(
|
6671
|
+
self,
|
6672
|
+
*cmd: str,
|
6673
|
+
**kwargs: ta.Any,
|
6674
|
+
) -> ta.Optional[bytes]:
|
6675
|
+
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
6676
|
+
return None
|
6677
|
+
else:
|
6678
|
+
return ret
|
6656
6679
|
|
6657
|
-
async def
|
6658
|
-
|
6659
|
-
|
6660
|
-
|
6661
|
-
) -> ta.Optional[
|
6662
|
-
|
6663
|
-
|
6664
|
-
|
6665
|
-
|
6666
|
-
**kwargs,
|
6667
|
-
), Exception):
|
6668
|
-
return None
|
6669
|
-
else:
|
6670
|
-
return ret
|
6680
|
+
async def try_output_str(
|
6681
|
+
self,
|
6682
|
+
*cmd: str,
|
6683
|
+
**kwargs: ta.Any,
|
6684
|
+
) -> ta.Optional[str]:
|
6685
|
+
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
6686
|
+
return None
|
6687
|
+
else:
|
6688
|
+
return ret.decode().strip()
|
6671
6689
|
|
6672
6690
|
|
6673
|
-
|
6674
|
-
out = await asyncio_subprocess_try_output(*args, **kwargs)
|
6675
|
-
return out.decode().strip() if out is not None else None
|
6691
|
+
asyncio_subprocesses = AsyncioSubprocesses()
|
6676
6692
|
|
6677
6693
|
|
6678
6694
|
########################################
|
@@ -6749,7 +6765,7 @@ class InterpInspector:
|
|
6749
6765
|
return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
|
6750
6766
|
|
6751
6767
|
async def _inspect(self, exe: str) -> InterpInspection:
|
6752
|
-
output = await
|
6768
|
+
output = await asyncio_subprocesses.check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
|
6753
6769
|
return self._build_inspection(exe, output.decode())
|
6754
6770
|
|
6755
6771
|
async def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
|
@@ -6823,7 +6839,7 @@ class SubprocessCommand(Command['SubprocessCommand.Output']):
|
|
6823
6839
|
class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
|
6824
6840
|
async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
|
6825
6841
|
proc: asyncio.subprocess.Process
|
6826
|
-
async with
|
6842
|
+
async with asyncio_subprocesses.popen(
|
6827
6843
|
*subprocess_maybe_shell_wrap_exec(*cmd.cmd),
|
6828
6844
|
|
6829
6845
|
shell=cmd.shell,
|
@@ -6837,7 +6853,7 @@ class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCom
|
|
6837
6853
|
timeout=cmd.timeout,
|
6838
6854
|
) as proc:
|
6839
6855
|
start_time = time.time()
|
6840
|
-
stdout, stderr = await
|
6856
|
+
stdout, stderr = await asyncio_subprocesses.communicate(
|
6841
6857
|
proc,
|
6842
6858
|
input=cmd.input,
|
6843
6859
|
timeout=cmd.timeout,
|
@@ -6937,7 +6953,7 @@ class DeployGitManager(DeployPathOwner):
|
|
6937
6953
|
return f'https://{self._repo.host}/{self._repo.path}'
|
6938
6954
|
|
6939
6955
|
async def _call(self, *cmd: str) -> None:
|
6940
|
-
await
|
6956
|
+
await asyncio_subprocesses.check_call(
|
6941
6957
|
*cmd,
|
6942
6958
|
cwd=self._dir,
|
6943
6959
|
)
|
@@ -6963,7 +6979,7 @@ class DeployGitManager(DeployPathOwner):
|
|
6963
6979
|
# FIXME: temp dir swap
|
6964
6980
|
os.makedirs(dst_dir)
|
6965
6981
|
|
6966
|
-
dst_call = functools.partial(
|
6982
|
+
dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_dir)
|
6967
6983
|
await dst_call('git', 'init')
|
6968
6984
|
|
6969
6985
|
await dst_call('git', 'remote', 'add', 'local', self._dir)
|
@@ -7015,7 +7031,7 @@ class DeployVenvManager(DeployPathOwner):
|
|
7015
7031
|
) -> None:
|
7016
7032
|
sys_exe = 'python3'
|
7017
7033
|
|
7018
|
-
await
|
7034
|
+
await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
|
7019
7035
|
|
7020
7036
|
#
|
7021
7037
|
|
@@ -7027,12 +7043,12 @@ class DeployVenvManager(DeployPathOwner):
|
|
7027
7043
|
|
7028
7044
|
if os.path.isfile(reqs_txt):
|
7029
7045
|
if use_uv:
|
7030
|
-
await
|
7046
|
+
await asyncio_subprocesses.check_call(venv_exe, '-m', 'pip', 'install', 'uv')
|
7031
7047
|
pip_cmd = ['-m', 'uv', 'pip']
|
7032
7048
|
else:
|
7033
7049
|
pip_cmd = ['-m', 'pip']
|
7034
7050
|
|
7035
|
-
await
|
7051
|
+
await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
|
7036
7052
|
|
7037
7053
|
async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
|
7038
7054
|
await self.setup_venv(
|
@@ -7117,7 +7133,7 @@ class SubprocessRemoteSpawning(RemoteSpawning):
|
|
7117
7133
|
if not debug:
|
7118
7134
|
cmd = subprocess_maybe_shell_wrap_exec(*cmd)
|
7119
7135
|
|
7120
|
-
async with
|
7136
|
+
async with asyncio_subprocesses.popen(
|
7121
7137
|
*cmd,
|
7122
7138
|
shell=pc.shell,
|
7123
7139
|
stdin=subprocess.PIPE,
|
@@ -7179,10 +7195,10 @@ class SystemPackageManager(abc.ABC):
|
|
7179
7195
|
|
7180
7196
|
class BrewSystemPackageManager(SystemPackageManager):
|
7181
7197
|
async def update(self) -> None:
|
7182
|
-
await
|
7198
|
+
await asyncio_subprocesses.check_call('brew', 'update')
|
7183
7199
|
|
7184
7200
|
async def upgrade(self) -> None:
|
7185
|
-
await
|
7201
|
+
await asyncio_subprocesses.check_call('brew', 'upgrade')
|
7186
7202
|
|
7187
7203
|
async def install(self, *packages: SystemPackageOrStr) -> None:
|
7188
7204
|
es: ta.List[str] = []
|
@@ -7191,11 +7207,11 @@ class BrewSystemPackageManager(SystemPackageManager):
|
|
7191
7207
|
es.append(p.name + (f'@{p.version}' if p.version is not None else ''))
|
7192
7208
|
else:
|
7193
7209
|
es.append(p)
|
7194
|
-
await
|
7210
|
+
await asyncio_subprocesses.check_call('brew', 'install', *es)
|
7195
7211
|
|
7196
7212
|
async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
|
7197
7213
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
|
7198
|
-
o = await
|
7214
|
+
o = await asyncio_subprocesses.check_output('brew', 'info', '--json', *pns)
|
7199
7215
|
j = json.loads(o.decode())
|
7200
7216
|
d: ta.Dict[str, SystemPackage] = {}
|
7201
7217
|
for e in j:
|
@@ -7214,18 +7230,18 @@ class AptSystemPackageManager(SystemPackageManager):
|
|
7214
7230
|
}
|
7215
7231
|
|
7216
7232
|
async def update(self) -> None:
|
7217
|
-
await
|
7233
|
+
await asyncio_subprocesses.check_call('sudo', 'apt', 'update', env={**os.environ, **self._APT_ENV})
|
7218
7234
|
|
7219
7235
|
async def upgrade(self) -> None:
|
7220
|
-
await
|
7236
|
+
await asyncio_subprocesses.check_call('sudo', 'apt', 'upgrade', '-y', env={**os.environ, **self._APT_ENV})
|
7221
7237
|
|
7222
7238
|
async def install(self, *packages: SystemPackageOrStr) -> None:
|
7223
7239
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages] # FIXME: versions
|
7224
|
-
await
|
7240
|
+
await asyncio_subprocesses.check_call('sudo', 'apt', 'install', '-y', *pns, env={**os.environ, **self._APT_ENV})
|
7225
7241
|
|
7226
7242
|
async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
|
7227
7243
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
|
7228
|
-
out = await
|
7244
|
+
out = await asyncio_subprocesses.run(
|
7229
7245
|
'dpkg-query', '-W', '-f=${Package}=${Version}\n', *pns,
|
7230
7246
|
capture_output=True,
|
7231
7247
|
check=False,
|
@@ -7242,20 +7258,20 @@ class AptSystemPackageManager(SystemPackageManager):
|
|
7242
7258
|
|
7243
7259
|
class YumSystemPackageManager(SystemPackageManager):
|
7244
7260
|
async def update(self) -> None:
|
7245
|
-
await
|
7261
|
+
await asyncio_subprocesses.check_call('sudo', 'yum', 'check-update')
|
7246
7262
|
|
7247
7263
|
async def upgrade(self) -> None:
|
7248
|
-
await
|
7264
|
+
await asyncio_subprocesses.check_call('sudo', 'yum', 'update')
|
7249
7265
|
|
7250
7266
|
async def install(self, *packages: SystemPackageOrStr) -> None:
|
7251
7267
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages] # FIXME: versions
|
7252
|
-
await
|
7268
|
+
await asyncio_subprocesses.check_call('sudo', 'yum', 'install', *pns)
|
7253
7269
|
|
7254
7270
|
async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
|
7255
7271
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
|
7256
7272
|
d: ta.Dict[str, SystemPackage] = {}
|
7257
7273
|
for pn in pns:
|
7258
|
-
out = await
|
7274
|
+
out = await asyncio_subprocesses.run(
|
7259
7275
|
'rpm', '-q', pn,
|
7260
7276
|
capture_output=True,
|
7261
7277
|
)
|
@@ -7704,7 +7720,7 @@ class Pyenv:
|
|
7704
7720
|
return self._root_kw
|
7705
7721
|
|
7706
7722
|
if shutil.which('pyenv'):
|
7707
|
-
return await
|
7723
|
+
return await asyncio_subprocesses.check_output_str('pyenv', 'root')
|
7708
7724
|
|
7709
7725
|
d = os.path.expanduser('~/.pyenv')
|
7710
7726
|
if os.path.isdir(d) and os.path.isfile(os.path.join(d, 'bin', 'pyenv')):
|
@@ -7733,7 +7749,7 @@ class Pyenv:
|
|
7733
7749
|
if await self.root() is None:
|
7734
7750
|
return []
|
7735
7751
|
ret = []
|
7736
|
-
s = await
|
7752
|
+
s = await asyncio_subprocesses.check_output_str(await self.exe(), 'install', '--list')
|
7737
7753
|
for l in s.splitlines():
|
7738
7754
|
if not l.startswith(' '):
|
7739
7755
|
continue
|
@@ -7748,7 +7764,7 @@ class Pyenv:
|
|
7748
7764
|
return False
|
7749
7765
|
if not os.path.isdir(os.path.join(root, '.git')):
|
7750
7766
|
return False
|
7751
|
-
await
|
7767
|
+
await asyncio_subprocesses.check_call('git', 'pull', cwd=root)
|
7752
7768
|
return True
|
7753
7769
|
|
7754
7770
|
|
@@ -7839,7 +7855,7 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
7839
7855
|
cflags = []
|
7840
7856
|
ldflags = []
|
7841
7857
|
for dep in self.BREW_DEPS:
|
7842
|
-
dep_prefix = await
|
7858
|
+
dep_prefix = await asyncio_subprocesses.check_output_str('brew', '--prefix', dep)
|
7843
7859
|
cflags.append(f'-I{dep_prefix}/include')
|
7844
7860
|
ldflags.append(f'-L{dep_prefix}/lib')
|
7845
7861
|
return PyenvInstallOpts(
|
@@ -7849,11 +7865,11 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
7849
7865
|
|
7850
7866
|
@async_cached_nullary
|
7851
7867
|
async def brew_tcl_opts(self) -> PyenvInstallOpts:
|
7852
|
-
if await
|
7868
|
+
if await asyncio_subprocesses.try_output('brew', '--prefix', 'tcl-tk') is None:
|
7853
7869
|
return PyenvInstallOpts()
|
7854
7870
|
|
7855
|
-
tcl_tk_prefix = await
|
7856
|
-
tcl_tk_ver_str = await
|
7871
|
+
tcl_tk_prefix = await asyncio_subprocesses.check_output_str('brew', '--prefix', 'tcl-tk')
|
7872
|
+
tcl_tk_ver_str = await asyncio_subprocesses.check_output_str('brew', 'ls', '--versions', 'tcl-tk')
|
7857
7873
|
tcl_tk_ver = '.'.join(tcl_tk_ver_str.split()[1].split('.')[:2])
|
7858
7874
|
|
7859
7875
|
return PyenvInstallOpts(conf_opts=[
|
@@ -7974,7 +7990,7 @@ class PyenvVersionInstaller:
|
|
7974
7990
|
*conf_args,
|
7975
7991
|
]
|
7976
7992
|
|
7977
|
-
await
|
7993
|
+
await asyncio_subprocesses.check_call(
|
7978
7994
|
*full_args,
|
7979
7995
|
env=env,
|
7980
7996
|
)
|