ominfra 0.0.0.dev155__py3-none-any.whl → 0.0.0.dev156__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/manage/commands/subprocess.py +3 -4
- ominfra/manage/deploy/git.py +3 -3
- ominfra/manage/deploy/venvs.py +4 -4
- ominfra/manage/remote/spawning.py +2 -2
- ominfra/manage/system/packages.py +13 -15
- ominfra/scripts/journald2aws.py +181 -127
- ominfra/scripts/manage.py +330 -314
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/RECORD +13 -13
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev155.dist-info → ominfra-0.0.0.dev156.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -5788,168 +5788,222 @@ SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
|
|
5788
5788
|
_SUBPROCESS_SHELL_WRAP_EXECS = False
|
5789
5789
|
|
5790
5790
|
|
5791
|
-
def subprocess_shell_wrap_exec(*
|
5792
|
-
return ('sh', '-c', ' '.join(map(shlex.quote,
|
5791
|
+
def subprocess_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
|
5792
|
+
return ('sh', '-c', ' '.join(map(shlex.quote, cmd)))
|
5793
5793
|
|
5794
5794
|
|
5795
|
-
def subprocess_maybe_shell_wrap_exec(*
|
5795
|
+
def subprocess_maybe_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
|
5796
5796
|
if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
|
5797
|
-
return subprocess_shell_wrap_exec(*
|
5797
|
+
return subprocess_shell_wrap_exec(*cmd)
|
5798
5798
|
else:
|
5799
|
-
return
|
5800
|
-
|
5801
|
-
|
5802
|
-
def prepare_subprocess_invocation(
|
5803
|
-
*args: str,
|
5804
|
-
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
5805
|
-
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
5806
|
-
quiet: bool = False,
|
5807
|
-
shell: bool = False,
|
5808
|
-
**kwargs: ta.Any,
|
5809
|
-
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
5810
|
-
log.debug('prepare_subprocess_invocation: args=%r', args)
|
5811
|
-
if extra_env:
|
5812
|
-
log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
|
5813
|
-
|
5814
|
-
if extra_env:
|
5815
|
-
env = {**(env if env is not None else os.environ), **extra_env}
|
5816
|
-
|
5817
|
-
if quiet and 'stderr' not in kwargs:
|
5818
|
-
if not log.isEnabledFor(logging.DEBUG):
|
5819
|
-
kwargs['stderr'] = subprocess.DEVNULL
|
5820
|
-
|
5821
|
-
if not shell:
|
5822
|
-
args = subprocess_maybe_shell_wrap_exec(*args)
|
5823
|
-
|
5824
|
-
return args, dict(
|
5825
|
-
env=env,
|
5826
|
-
shell=shell,
|
5827
|
-
**kwargs,
|
5828
|
-
)
|
5799
|
+
return cmd
|
5829
5800
|
|
5830
5801
|
|
5831
5802
|
##
|
5832
5803
|
|
5833
5804
|
|
5834
|
-
|
5835
|
-
|
5836
|
-
|
5837
|
-
|
5838
|
-
|
5839
|
-
|
5840
|
-
|
5841
|
-
|
5842
|
-
|
5843
|
-
|
5805
|
+
def subprocess_close(
|
5806
|
+
proc: subprocess.Popen,
|
5807
|
+
timeout: ta.Optional[float] = None,
|
5808
|
+
) -> None:
|
5809
|
+
# TODO: terminate, sleep, kill
|
5810
|
+
if proc.stdout:
|
5811
|
+
proc.stdout.close()
|
5812
|
+
if proc.stderr:
|
5813
|
+
proc.stderr.close()
|
5814
|
+
if proc.stdin:
|
5815
|
+
proc.stdin.close()
|
5844
5816
|
|
5845
|
-
|
5846
|
-
end_time = time.time()
|
5847
|
-
elapsed_s = end_time - start_time
|
5848
|
-
log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
|
5817
|
+
proc.wait(timeout)
|
5849
5818
|
|
5850
5819
|
|
5851
5820
|
##
|
5852
5821
|
|
5853
5822
|
|
5854
|
-
|
5855
|
-
|
5856
|
-
stdout: ta.Any = sys.stderr,
|
5857
|
-
**kwargs: ta.Any,
|
5858
|
-
) -> None:
|
5859
|
-
args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
|
5860
|
-
with subprocess_common_context(*args, **kwargs):
|
5861
|
-
return subprocess.check_call(args, **kwargs) # type: ignore
|
5823
|
+
class AbstractSubprocesses(abc.ABC): # noqa
|
5824
|
+
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = log
|
5862
5825
|
|
5826
|
+
def __init__(
|
5827
|
+
self,
|
5828
|
+
*,
|
5829
|
+
log: ta.Optional[logging.Logger] = None,
|
5830
|
+
try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
|
5831
|
+
) -> None:
|
5832
|
+
super().__init__()
|
5863
5833
|
|
5864
|
-
|
5865
|
-
|
5866
|
-
**kwargs: ta.Any,
|
5867
|
-
) -> bytes:
|
5868
|
-
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
5869
|
-
with subprocess_common_context(*args, **kwargs):
|
5870
|
-
return subprocess.check_output(args, **kwargs)
|
5834
|
+
self._log = log if log is not None else self.DEFAULT_LOGGER
|
5835
|
+
self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
|
5871
5836
|
|
5837
|
+
#
|
5872
5838
|
|
5873
|
-
def
|
5874
|
-
|
5839
|
+
def prepare_args(
|
5840
|
+
self,
|
5841
|
+
*cmd: str,
|
5842
|
+
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
5843
|
+
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
5844
|
+
quiet: bool = False,
|
5845
|
+
shell: bool = False,
|
5846
|
+
**kwargs: ta.Any,
|
5847
|
+
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
5848
|
+
if self._log:
|
5849
|
+
self._log.debug('Subprocesses.prepare_args: cmd=%r', cmd)
|
5850
|
+
if extra_env:
|
5851
|
+
self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
|
5875
5852
|
|
5853
|
+
if extra_env:
|
5854
|
+
env = {**(env if env is not None else os.environ), **extra_env}
|
5876
5855
|
|
5877
|
-
|
5856
|
+
if quiet and 'stderr' not in kwargs:
|
5857
|
+
if self._log and not self._log.isEnabledFor(logging.DEBUG):
|
5858
|
+
kwargs['stderr'] = subprocess.DEVNULL
|
5878
5859
|
|
5860
|
+
if not shell:
|
5861
|
+
cmd = subprocess_maybe_shell_wrap_exec(*cmd)
|
5879
5862
|
|
5880
|
-
|
5881
|
-
|
5882
|
-
|
5883
|
-
|
5863
|
+
return cmd, dict(
|
5864
|
+
env=env,
|
5865
|
+
shell=shell,
|
5866
|
+
**kwargs,
|
5867
|
+
)
|
5884
5868
|
|
5869
|
+
@contextlib.contextmanager
|
5870
|
+
def wrap_call(self, *cmd: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
|
5871
|
+
start_time = time.time()
|
5872
|
+
try:
|
5873
|
+
if self._log:
|
5874
|
+
self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
|
5875
|
+
yield
|
5885
5876
|
|
5886
|
-
|
5887
|
-
|
5888
|
-
|
5889
|
-
|
5890
|
-
**kwargs: ta.Any,
|
5891
|
-
) -> ta.Union[T, Exception]:
|
5892
|
-
try:
|
5893
|
-
return fn(*args, **kwargs)
|
5894
|
-
except try_exceptions as e: # noqa
|
5895
|
-
if log.isEnabledFor(logging.DEBUG):
|
5896
|
-
log.exception('command failed')
|
5897
|
-
return e
|
5898
|
-
|
5899
|
-
|
5900
|
-
def subprocess_try_call(
|
5901
|
-
*args: str,
|
5902
|
-
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
5903
|
-
**kwargs: ta.Any,
|
5904
|
-
) -> bool:
|
5905
|
-
if isinstance(_subprocess_try_run(
|
5906
|
-
subprocess_check_call,
|
5907
|
-
*args,
|
5908
|
-
try_exceptions=try_exceptions,
|
5909
|
-
**kwargs,
|
5910
|
-
), Exception):
|
5911
|
-
return False
|
5912
|
-
else:
|
5913
|
-
return True
|
5877
|
+
except Exception as exc: # noqa
|
5878
|
+
if self._log:
|
5879
|
+
self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
|
5880
|
+
raise
|
5914
5881
|
|
5882
|
+
finally:
|
5883
|
+
end_time = time.time()
|
5884
|
+
elapsed_s = end_time - start_time
|
5885
|
+
if self._log:
|
5886
|
+
self._log.debug('sSubprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
|
5915
5887
|
|
5916
|
-
|
5917
|
-
|
5918
|
-
|
5919
|
-
|
5920
|
-
|
5921
|
-
|
5922
|
-
|
5923
|
-
|
5924
|
-
|
5925
|
-
|
5926
|
-
|
5927
|
-
|
5928
|
-
else:
|
5929
|
-
return ret
|
5888
|
+
@contextlib.contextmanager
|
5889
|
+
def prepare_and_wrap(
|
5890
|
+
self,
|
5891
|
+
*cmd: ta.Any,
|
5892
|
+
**kwargs: ta.Any,
|
5893
|
+
) -> ta.Iterator[ta.Tuple[
|
5894
|
+
ta.Tuple[ta.Any, ...],
|
5895
|
+
ta.Dict[str, ta.Any],
|
5896
|
+
]]:
|
5897
|
+
cmd, kwargs = self.prepare_args(*cmd, **kwargs)
|
5898
|
+
with self.wrap_call(*cmd, **kwargs):
|
5899
|
+
yield cmd, kwargs
|
5930
5900
|
|
5901
|
+
#
|
5902
|
+
|
5903
|
+
DEFAULT_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
|
5904
|
+
FileNotFoundError,
|
5905
|
+
subprocess.CalledProcessError,
|
5906
|
+
)
|
5907
|
+
|
5908
|
+
def try_fn(
|
5909
|
+
self,
|
5910
|
+
fn: ta.Callable[..., T],
|
5911
|
+
*cmd: str,
|
5912
|
+
try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
|
5913
|
+
**kwargs: ta.Any,
|
5914
|
+
) -> ta.Union[T, Exception]:
|
5915
|
+
if try_exceptions is None:
|
5916
|
+
try_exceptions = self._try_exceptions
|
5917
|
+
|
5918
|
+
try:
|
5919
|
+
return fn(*cmd, **kwargs)
|
5920
|
+
|
5921
|
+
except try_exceptions as e: # noqa
|
5922
|
+
if self._log and self._log.isEnabledFor(logging.DEBUG):
|
5923
|
+
self._log.exception('command failed')
|
5924
|
+
return e
|
5925
|
+
|
5926
|
+
async def async_try_fn(
|
5927
|
+
self,
|
5928
|
+
fn: ta.Callable[..., ta.Awaitable[T]],
|
5929
|
+
*cmd: ta.Any,
|
5930
|
+
try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
|
5931
|
+
**kwargs: ta.Any,
|
5932
|
+
) -> ta.Union[T, Exception]:
|
5933
|
+
if try_exceptions is None:
|
5934
|
+
try_exceptions = self._try_exceptions
|
5931
5935
|
|
5932
|
-
|
5933
|
-
|
5934
|
-
|
5936
|
+
try:
|
5937
|
+
return await fn(*cmd, **kwargs)
|
5938
|
+
|
5939
|
+
except try_exceptions as e: # noqa
|
5940
|
+
if self._log and self._log.isEnabledFor(logging.DEBUG):
|
5941
|
+
self._log.exception('command failed')
|
5942
|
+
return e
|
5935
5943
|
|
5936
5944
|
|
5937
5945
|
##
|
5938
5946
|
|
5939
5947
|
|
5940
|
-
|
5941
|
-
|
5942
|
-
|
5943
|
-
|
5944
|
-
|
5945
|
-
|
5946
|
-
|
5947
|
-
|
5948
|
-
|
5949
|
-
if proc.stdin:
|
5950
|
-
proc.stdin.close()
|
5948
|
+
class Subprocesses(AbstractSubprocesses):
|
5949
|
+
def check_call(
|
5950
|
+
self,
|
5951
|
+
*cmd: str,
|
5952
|
+
stdout: ta.Any = sys.stderr,
|
5953
|
+
**kwargs: ta.Any,
|
5954
|
+
) -> None:
|
5955
|
+
with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
|
5956
|
+
subprocess.check_call(cmd, **kwargs)
|
5951
5957
|
|
5952
|
-
|
5958
|
+
def check_output(
|
5959
|
+
self,
|
5960
|
+
*cmd: str,
|
5961
|
+
**kwargs: ta.Any,
|
5962
|
+
) -> bytes:
|
5963
|
+
with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
|
5964
|
+
return subprocess.check_output(cmd, **kwargs)
|
5965
|
+
|
5966
|
+
def check_output_str(
|
5967
|
+
self,
|
5968
|
+
*cmd: str,
|
5969
|
+
**kwargs: ta.Any,
|
5970
|
+
) -> str:
|
5971
|
+
return self.check_output(*cmd, **kwargs).decode().strip()
|
5972
|
+
|
5973
|
+
#
|
5974
|
+
|
5975
|
+
def try_call(
|
5976
|
+
self,
|
5977
|
+
*cmd: str,
|
5978
|
+
**kwargs: ta.Any,
|
5979
|
+
) -> bool:
|
5980
|
+
if isinstance(self.try_fn(self.check_call, *cmd, **kwargs), Exception):
|
5981
|
+
return False
|
5982
|
+
else:
|
5983
|
+
return True
|
5984
|
+
|
5985
|
+
def try_output(
|
5986
|
+
self,
|
5987
|
+
*cmd: str,
|
5988
|
+
**kwargs: ta.Any,
|
5989
|
+
) -> ta.Optional[bytes]:
|
5990
|
+
if isinstance(ret := self.try_fn(self.check_output, *cmd, **kwargs), Exception):
|
5991
|
+
return None
|
5992
|
+
else:
|
5993
|
+
return ret
|
5994
|
+
|
5995
|
+
def try_output_str(
|
5996
|
+
self,
|
5997
|
+
*cmd: str,
|
5998
|
+
**kwargs: ta.Any,
|
5999
|
+
) -> ta.Optional[str]:
|
6000
|
+
if (ret := self.try_output(*cmd, **kwargs)) is None:
|
6001
|
+
return None
|
6002
|
+
else:
|
6003
|
+
return ret.decode().strip()
|
6004
|
+
|
6005
|
+
|
6006
|
+
subprocesses = Subprocesses()
|
5953
6007
|
|
5954
6008
|
|
5955
6009
|
########################################
|
@@ -6384,43 +6438,6 @@ class SystemConfig:
|
|
6384
6438
|
##
|
6385
6439
|
|
6386
6440
|
|
6387
|
-
@contextlib.asynccontextmanager
|
6388
|
-
async def asyncio_subprocess_popen(
|
6389
|
-
*cmd: str,
|
6390
|
-
shell: bool = False,
|
6391
|
-
timeout: ta.Optional[float] = None,
|
6392
|
-
**kwargs: ta.Any,
|
6393
|
-
) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
|
6394
|
-
fac: ta.Any
|
6395
|
-
if shell:
|
6396
|
-
fac = functools.partial(
|
6397
|
-
asyncio.create_subprocess_shell,
|
6398
|
-
check.single(cmd),
|
6399
|
-
)
|
6400
|
-
else:
|
6401
|
-
fac = functools.partial(
|
6402
|
-
asyncio.create_subprocess_exec,
|
6403
|
-
*cmd,
|
6404
|
-
)
|
6405
|
-
|
6406
|
-
with subprocess_common_context(
|
6407
|
-
*cmd,
|
6408
|
-
shell=shell,
|
6409
|
-
timeout=timeout,
|
6410
|
-
**kwargs,
|
6411
|
-
):
|
6412
|
-
proc: asyncio.subprocess.Process
|
6413
|
-
proc = await fac(**kwargs)
|
6414
|
-
try:
|
6415
|
-
yield proc
|
6416
|
-
|
6417
|
-
finally:
|
6418
|
-
await asyncio_maybe_timeout(proc.wait(), timeout)
|
6419
|
-
|
6420
|
-
|
6421
|
-
##
|
6422
|
-
|
6423
|
-
|
6424
6441
|
class AsyncioProcessCommunicator:
|
6425
6442
|
def __init__(
|
6426
6443
|
self,
|
@@ -6531,148 +6548,147 @@ class AsyncioProcessCommunicator:
|
|
6531
6548
|
return await asyncio_maybe_timeout(self._communicate(input), timeout)
|
6532
6549
|
|
6533
6550
|
|
6534
|
-
|
6535
|
-
proc: asyncio.subprocess.Process,
|
6536
|
-
input: ta.Any = None, # noqa
|
6537
|
-
timeout: ta.Optional[float] = None,
|
6538
|
-
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
6539
|
-
return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
|
6540
|
-
|
6551
|
+
##
|
6541
6552
|
|
6542
|
-
@dc.dataclass(frozen=True)
|
6543
|
-
class AsyncioSubprocessOutput:
|
6544
|
-
proc: asyncio.subprocess.Process
|
6545
|
-
stdout: ta.Optional[bytes]
|
6546
|
-
stderr: ta.Optional[bytes]
|
6547
6553
|
|
6554
|
+
class AsyncioSubprocesses(AbstractSubprocesses):
|
6555
|
+
async def communicate(
|
6556
|
+
self,
|
6557
|
+
proc: asyncio.subprocess.Process,
|
6558
|
+
input: ta.Any = None, # noqa
|
6559
|
+
timeout: ta.Optional[float] = None,
|
6560
|
+
) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
|
6561
|
+
return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
|
6548
6562
|
|
6549
|
-
|
6550
|
-
*args: str,
|
6551
|
-
input: ta.Any = None, # noqa
|
6552
|
-
timeout: ta.Optional[float] = None,
|
6553
|
-
check: bool = False, # noqa
|
6554
|
-
capture_output: ta.Optional[bool] = None,
|
6555
|
-
**kwargs: ta.Any,
|
6556
|
-
) -> AsyncioSubprocessOutput:
|
6557
|
-
if capture_output:
|
6558
|
-
kwargs.setdefault('stdout', subprocess.PIPE)
|
6559
|
-
kwargs.setdefault('stderr', subprocess.PIPE)
|
6560
|
-
|
6561
|
-
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
6562
|
-
|
6563
|
-
proc: asyncio.subprocess.Process
|
6564
|
-
async with asyncio_subprocess_popen(*args, **kwargs) as proc:
|
6565
|
-
stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
|
6566
|
-
|
6567
|
-
if check and proc.returncode:
|
6568
|
-
raise subprocess.CalledProcessError(
|
6569
|
-
proc.returncode,
|
6570
|
-
args,
|
6571
|
-
output=stdout,
|
6572
|
-
stderr=stderr,
|
6573
|
-
)
|
6563
|
+
#
|
6574
6564
|
|
6575
|
-
|
6576
|
-
|
6577
|
-
|
6578
|
-
|
6579
|
-
|
6565
|
+
@contextlib.asynccontextmanager
|
6566
|
+
async def popen(
|
6567
|
+
self,
|
6568
|
+
*cmd: str,
|
6569
|
+
shell: bool = False,
|
6570
|
+
timeout: ta.Optional[float] = None,
|
6571
|
+
**kwargs: ta.Any,
|
6572
|
+
) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
|
6573
|
+
fac: ta.Any
|
6574
|
+
if shell:
|
6575
|
+
fac = functools.partial(
|
6576
|
+
asyncio.create_subprocess_shell,
|
6577
|
+
check.single(cmd),
|
6578
|
+
)
|
6579
|
+
else:
|
6580
|
+
fac = functools.partial(
|
6581
|
+
asyncio.create_subprocess_exec,
|
6582
|
+
*cmd,
|
6583
|
+
)
|
6580
6584
|
|
6585
|
+
with self.prepare_and_wrap( *cmd, shell=shell, **kwargs) as (cmd, kwargs): # noqa
|
6586
|
+
proc: asyncio.subprocess.Process = await fac(**kwargs)
|
6587
|
+
try:
|
6588
|
+
yield proc
|
6581
6589
|
|
6582
|
-
|
6590
|
+
finally:
|
6591
|
+
await asyncio_maybe_timeout(proc.wait(), timeout)
|
6583
6592
|
|
6593
|
+
#
|
6584
6594
|
|
6585
|
-
|
6586
|
-
|
6587
|
-
|
6588
|
-
|
6589
|
-
|
6590
|
-
**kwargs: ta.Any,
|
6591
|
-
) -> None:
|
6592
|
-
await asyncio_subprocess_run(
|
6593
|
-
*args,
|
6594
|
-
stdout=stdout,
|
6595
|
-
input=input,
|
6596
|
-
timeout=timeout,
|
6597
|
-
check=True,
|
6598
|
-
**kwargs,
|
6599
|
-
)
|
6595
|
+
@dc.dataclass(frozen=True)
|
6596
|
+
class RunOutput:
|
6597
|
+
proc: asyncio.subprocess.Process
|
6598
|
+
stdout: ta.Optional[bytes]
|
6599
|
+
stderr: ta.Optional[bytes]
|
6600
6600
|
|
6601
|
+
async def run(
|
6602
|
+
self,
|
6603
|
+
*cmd: str,
|
6604
|
+
input: ta.Any = None, # noqa
|
6605
|
+
timeout: ta.Optional[float] = None,
|
6606
|
+
check: bool = False, # noqa
|
6607
|
+
capture_output: ta.Optional[bool] = None,
|
6608
|
+
**kwargs: ta.Any,
|
6609
|
+
) -> RunOutput:
|
6610
|
+
if capture_output:
|
6611
|
+
kwargs.setdefault('stdout', subprocess.PIPE)
|
6612
|
+
kwargs.setdefault('stderr', subprocess.PIPE)
|
6601
6613
|
|
6602
|
-
|
6603
|
-
*
|
6604
|
-
|
6605
|
-
|
6606
|
-
|
6607
|
-
|
6608
|
-
|
6609
|
-
|
6610
|
-
|
6611
|
-
|
6612
|
-
|
6613
|
-
check=True,
|
6614
|
-
**kwargs,
|
6615
|
-
)
|
6614
|
+
proc: asyncio.subprocess.Process
|
6615
|
+
async with self.popen(*cmd, **kwargs) as proc:
|
6616
|
+
stdout, stderr = await self.communicate(proc, input, timeout)
|
6617
|
+
|
6618
|
+
if check and proc.returncode:
|
6619
|
+
raise subprocess.CalledProcessError(
|
6620
|
+
proc.returncode,
|
6621
|
+
cmd,
|
6622
|
+
output=stdout,
|
6623
|
+
stderr=stderr,
|
6624
|
+
)
|
6616
6625
|
|
6617
|
-
|
6626
|
+
return self.RunOutput(
|
6627
|
+
proc,
|
6628
|
+
stdout,
|
6629
|
+
stderr,
|
6630
|
+
)
|
6618
6631
|
|
6632
|
+
#
|
6619
6633
|
|
6620
|
-
async def
|
6621
|
-
|
6634
|
+
async def check_call(
|
6635
|
+
self,
|
6636
|
+
*cmd: str,
|
6637
|
+
stdout: ta.Any = sys.stderr,
|
6638
|
+
**kwargs: ta.Any,
|
6639
|
+
) -> None:
|
6640
|
+
with self.prepare_and_wrap(*cmd, stdout=stdout, check=True, **kwargs) as (cmd, kwargs): # noqa
|
6641
|
+
await self.run(*cmd, **kwargs)
|
6622
6642
|
|
6643
|
+
async def check_output(
|
6644
|
+
self,
|
6645
|
+
*cmd: str,
|
6646
|
+
**kwargs: ta.Any,
|
6647
|
+
) -> bytes:
|
6648
|
+
with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs): # noqa
|
6649
|
+
return check.not_none((await self.run(*cmd, **kwargs)).stdout)
|
6623
6650
|
|
6624
|
-
|
6651
|
+
async def check_output_str(
|
6652
|
+
self,
|
6653
|
+
*cmd: str,
|
6654
|
+
**kwargs: ta.Any,
|
6655
|
+
) -> str:
|
6656
|
+
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
6625
6657
|
|
6658
|
+
#
|
6626
6659
|
|
6627
|
-
async def
|
6628
|
-
|
6629
|
-
|
6630
|
-
|
6631
|
-
|
6632
|
-
|
6633
|
-
|
6634
|
-
|
6635
|
-
|
6636
|
-
if log.isEnabledFor(logging.DEBUG):
|
6637
|
-
log.exception('command failed')
|
6638
|
-
return e
|
6639
|
-
|
6640
|
-
|
6641
|
-
async def asyncio_subprocess_try_call(
|
6642
|
-
*args: str,
|
6643
|
-
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
6644
|
-
**kwargs: ta.Any,
|
6645
|
-
) -> bool:
|
6646
|
-
if isinstance(await _asyncio_subprocess_try_run(
|
6647
|
-
asyncio_subprocess_check_call,
|
6648
|
-
*args,
|
6649
|
-
try_exceptions=try_exceptions,
|
6650
|
-
**kwargs,
|
6651
|
-
), Exception):
|
6652
|
-
return False
|
6653
|
-
else:
|
6654
|
-
return True
|
6660
|
+
async def try_call(
|
6661
|
+
self,
|
6662
|
+
*cmd: str,
|
6663
|
+
**kwargs: ta.Any,
|
6664
|
+
) -> bool:
|
6665
|
+
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
6666
|
+
return False
|
6667
|
+
else:
|
6668
|
+
return True
|
6655
6669
|
|
6670
|
+
async def try_output(
|
6671
|
+
self,
|
6672
|
+
*cmd: str,
|
6673
|
+
**kwargs: ta.Any,
|
6674
|
+
) -> ta.Optional[bytes]:
|
6675
|
+
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
6676
|
+
return None
|
6677
|
+
else:
|
6678
|
+
return ret
|
6656
6679
|
|
6657
|
-
async def
|
6658
|
-
|
6659
|
-
|
6660
|
-
|
6661
|
-
) -> ta.Optional[
|
6662
|
-
|
6663
|
-
|
6664
|
-
|
6665
|
-
|
6666
|
-
**kwargs,
|
6667
|
-
), Exception):
|
6668
|
-
return None
|
6669
|
-
else:
|
6670
|
-
return ret
|
6680
|
+
async def try_output_str(
|
6681
|
+
self,
|
6682
|
+
*cmd: str,
|
6683
|
+
**kwargs: ta.Any,
|
6684
|
+
) -> ta.Optional[str]:
|
6685
|
+
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
6686
|
+
return None
|
6687
|
+
else:
|
6688
|
+
return ret.decode().strip()
|
6671
6689
|
|
6672
6690
|
|
6673
|
-
|
6674
|
-
out = await asyncio_subprocess_try_output(*args, **kwargs)
|
6675
|
-
return out.decode().strip() if out is not None else None
|
6691
|
+
asyncio_subprocesses = AsyncioSubprocesses()
|
6676
6692
|
|
6677
6693
|
|
6678
6694
|
########################################
|
@@ -6749,7 +6765,7 @@ class InterpInspector:
|
|
6749
6765
|
return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
|
6750
6766
|
|
6751
6767
|
async def _inspect(self, exe: str) -> InterpInspection:
|
6752
|
-
output = await
|
6768
|
+
output = await asyncio_subprocesses.check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
|
6753
6769
|
return self._build_inspection(exe, output.decode())
|
6754
6770
|
|
6755
6771
|
async def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
|
@@ -6823,7 +6839,7 @@ class SubprocessCommand(Command['SubprocessCommand.Output']):
|
|
6823
6839
|
class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
|
6824
6840
|
async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
|
6825
6841
|
proc: asyncio.subprocess.Process
|
6826
|
-
async with
|
6842
|
+
async with asyncio_subprocesses.popen(
|
6827
6843
|
*subprocess_maybe_shell_wrap_exec(*cmd.cmd),
|
6828
6844
|
|
6829
6845
|
shell=cmd.shell,
|
@@ -6837,7 +6853,7 @@ class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCom
|
|
6837
6853
|
timeout=cmd.timeout,
|
6838
6854
|
) as proc:
|
6839
6855
|
start_time = time.time()
|
6840
|
-
stdout, stderr = await
|
6856
|
+
stdout, stderr = await asyncio_subprocesses.communicate(
|
6841
6857
|
proc,
|
6842
6858
|
input=cmd.input,
|
6843
6859
|
timeout=cmd.timeout,
|
@@ -6937,7 +6953,7 @@ class DeployGitManager(DeployPathOwner):
|
|
6937
6953
|
return f'https://{self._repo.host}/{self._repo.path}'
|
6938
6954
|
|
6939
6955
|
async def _call(self, *cmd: str) -> None:
|
6940
|
-
await
|
6956
|
+
await asyncio_subprocesses.check_call(
|
6941
6957
|
*cmd,
|
6942
6958
|
cwd=self._dir,
|
6943
6959
|
)
|
@@ -6963,7 +6979,7 @@ class DeployGitManager(DeployPathOwner):
|
|
6963
6979
|
# FIXME: temp dir swap
|
6964
6980
|
os.makedirs(dst_dir)
|
6965
6981
|
|
6966
|
-
dst_call = functools.partial(
|
6982
|
+
dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_dir)
|
6967
6983
|
await dst_call('git', 'init')
|
6968
6984
|
|
6969
6985
|
await dst_call('git', 'remote', 'add', 'local', self._dir)
|
@@ -7015,7 +7031,7 @@ class DeployVenvManager(DeployPathOwner):
|
|
7015
7031
|
) -> None:
|
7016
7032
|
sys_exe = 'python3'
|
7017
7033
|
|
7018
|
-
await
|
7034
|
+
await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
|
7019
7035
|
|
7020
7036
|
#
|
7021
7037
|
|
@@ -7027,12 +7043,12 @@ class DeployVenvManager(DeployPathOwner):
|
|
7027
7043
|
|
7028
7044
|
if os.path.isfile(reqs_txt):
|
7029
7045
|
if use_uv:
|
7030
|
-
await
|
7046
|
+
await asyncio_subprocesses.check_call(venv_exe, '-m', 'pip', 'install', 'uv')
|
7031
7047
|
pip_cmd = ['-m', 'uv', 'pip']
|
7032
7048
|
else:
|
7033
7049
|
pip_cmd = ['-m', 'pip']
|
7034
7050
|
|
7035
|
-
await
|
7051
|
+
await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
|
7036
7052
|
|
7037
7053
|
async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
|
7038
7054
|
await self.setup_venv(
|
@@ -7117,7 +7133,7 @@ class SubprocessRemoteSpawning(RemoteSpawning):
|
|
7117
7133
|
if not debug:
|
7118
7134
|
cmd = subprocess_maybe_shell_wrap_exec(*cmd)
|
7119
7135
|
|
7120
|
-
async with
|
7136
|
+
async with asyncio_subprocesses.popen(
|
7121
7137
|
*cmd,
|
7122
7138
|
shell=pc.shell,
|
7123
7139
|
stdin=subprocess.PIPE,
|
@@ -7179,10 +7195,10 @@ class SystemPackageManager(abc.ABC):
|
|
7179
7195
|
|
7180
7196
|
class BrewSystemPackageManager(SystemPackageManager):
|
7181
7197
|
async def update(self) -> None:
|
7182
|
-
await
|
7198
|
+
await asyncio_subprocesses.check_call('brew', 'update')
|
7183
7199
|
|
7184
7200
|
async def upgrade(self) -> None:
|
7185
|
-
await
|
7201
|
+
await asyncio_subprocesses.check_call('brew', 'upgrade')
|
7186
7202
|
|
7187
7203
|
async def install(self, *packages: SystemPackageOrStr) -> None:
|
7188
7204
|
es: ta.List[str] = []
|
@@ -7191,11 +7207,11 @@ class BrewSystemPackageManager(SystemPackageManager):
|
|
7191
7207
|
es.append(p.name + (f'@{p.version}' if p.version is not None else ''))
|
7192
7208
|
else:
|
7193
7209
|
es.append(p)
|
7194
|
-
await
|
7210
|
+
await asyncio_subprocesses.check_call('brew', 'install', *es)
|
7195
7211
|
|
7196
7212
|
async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
|
7197
7213
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
|
7198
|
-
o = await
|
7214
|
+
o = await asyncio_subprocesses.check_output('brew', 'info', '--json', *pns)
|
7199
7215
|
j = json.loads(o.decode())
|
7200
7216
|
d: ta.Dict[str, SystemPackage] = {}
|
7201
7217
|
for e in j:
|
@@ -7214,18 +7230,18 @@ class AptSystemPackageManager(SystemPackageManager):
|
|
7214
7230
|
}
|
7215
7231
|
|
7216
7232
|
async def update(self) -> None:
|
7217
|
-
await
|
7233
|
+
await asyncio_subprocesses.check_call('sudo', 'apt', 'update', env={**os.environ, **self._APT_ENV})
|
7218
7234
|
|
7219
7235
|
async def upgrade(self) -> None:
|
7220
|
-
await
|
7236
|
+
await asyncio_subprocesses.check_call('sudo', 'apt', 'upgrade', '-y', env={**os.environ, **self._APT_ENV})
|
7221
7237
|
|
7222
7238
|
async def install(self, *packages: SystemPackageOrStr) -> None:
|
7223
7239
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages] # FIXME: versions
|
7224
|
-
await
|
7240
|
+
await asyncio_subprocesses.check_call('sudo', 'apt', 'install', '-y', *pns, env={**os.environ, **self._APT_ENV})
|
7225
7241
|
|
7226
7242
|
async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
|
7227
7243
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
|
7228
|
-
out = await
|
7244
|
+
out = await asyncio_subprocesses.run(
|
7229
7245
|
'dpkg-query', '-W', '-f=${Package}=${Version}\n', *pns,
|
7230
7246
|
capture_output=True,
|
7231
7247
|
check=False,
|
@@ -7242,20 +7258,20 @@ class AptSystemPackageManager(SystemPackageManager):
|
|
7242
7258
|
|
7243
7259
|
class YumSystemPackageManager(SystemPackageManager):
|
7244
7260
|
async def update(self) -> None:
|
7245
|
-
await
|
7261
|
+
await asyncio_subprocesses.check_call('sudo', 'yum', 'check-update')
|
7246
7262
|
|
7247
7263
|
async def upgrade(self) -> None:
|
7248
|
-
await
|
7264
|
+
await asyncio_subprocesses.check_call('sudo', 'yum', 'update')
|
7249
7265
|
|
7250
7266
|
async def install(self, *packages: SystemPackageOrStr) -> None:
|
7251
7267
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages] # FIXME: versions
|
7252
|
-
await
|
7268
|
+
await asyncio_subprocesses.check_call('sudo', 'yum', 'install', *pns)
|
7253
7269
|
|
7254
7270
|
async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
|
7255
7271
|
pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
|
7256
7272
|
d: ta.Dict[str, SystemPackage] = {}
|
7257
7273
|
for pn in pns:
|
7258
|
-
out = await
|
7274
|
+
out = await asyncio_subprocesses.run(
|
7259
7275
|
'rpm', '-q', pn,
|
7260
7276
|
capture_output=True,
|
7261
7277
|
)
|
@@ -7704,7 +7720,7 @@ class Pyenv:
|
|
7704
7720
|
return self._root_kw
|
7705
7721
|
|
7706
7722
|
if shutil.which('pyenv'):
|
7707
|
-
return await
|
7723
|
+
return await asyncio_subprocesses.check_output_str('pyenv', 'root')
|
7708
7724
|
|
7709
7725
|
d = os.path.expanduser('~/.pyenv')
|
7710
7726
|
if os.path.isdir(d) and os.path.isfile(os.path.join(d, 'bin', 'pyenv')):
|
@@ -7733,7 +7749,7 @@ class Pyenv:
|
|
7733
7749
|
if await self.root() is None:
|
7734
7750
|
return []
|
7735
7751
|
ret = []
|
7736
|
-
s = await
|
7752
|
+
s = await asyncio_subprocesses.check_output_str(await self.exe(), 'install', '--list')
|
7737
7753
|
for l in s.splitlines():
|
7738
7754
|
if not l.startswith(' '):
|
7739
7755
|
continue
|
@@ -7748,7 +7764,7 @@ class Pyenv:
|
|
7748
7764
|
return False
|
7749
7765
|
if not os.path.isdir(os.path.join(root, '.git')):
|
7750
7766
|
return False
|
7751
|
-
await
|
7767
|
+
await asyncio_subprocesses.check_call('git', 'pull', cwd=root)
|
7752
7768
|
return True
|
7753
7769
|
|
7754
7770
|
|
@@ -7839,7 +7855,7 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
7839
7855
|
cflags = []
|
7840
7856
|
ldflags = []
|
7841
7857
|
for dep in self.BREW_DEPS:
|
7842
|
-
dep_prefix = await
|
7858
|
+
dep_prefix = await asyncio_subprocesses.check_output_str('brew', '--prefix', dep)
|
7843
7859
|
cflags.append(f'-I{dep_prefix}/include')
|
7844
7860
|
ldflags.append(f'-L{dep_prefix}/lib')
|
7845
7861
|
return PyenvInstallOpts(
|
@@ -7849,11 +7865,11 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
|
|
7849
7865
|
|
7850
7866
|
@async_cached_nullary
|
7851
7867
|
async def brew_tcl_opts(self) -> PyenvInstallOpts:
|
7852
|
-
if await
|
7868
|
+
if await asyncio_subprocesses.try_output('brew', '--prefix', 'tcl-tk') is None:
|
7853
7869
|
return PyenvInstallOpts()
|
7854
7870
|
|
7855
|
-
tcl_tk_prefix = await
|
7856
|
-
tcl_tk_ver_str = await
|
7871
|
+
tcl_tk_prefix = await asyncio_subprocesses.check_output_str('brew', '--prefix', 'tcl-tk')
|
7872
|
+
tcl_tk_ver_str = await asyncio_subprocesses.check_output_str('brew', 'ls', '--versions', 'tcl-tk')
|
7857
7873
|
tcl_tk_ver = '.'.join(tcl_tk_ver_str.split()[1].split('.')[:2])
|
7858
7874
|
|
7859
7875
|
return PyenvInstallOpts(conf_opts=[
|
@@ -7974,7 +7990,7 @@ class PyenvVersionInstaller:
|
|
7974
7990
|
*conf_args,
|
7975
7991
|
]
|
7976
7992
|
|
7977
|
-
await
|
7993
|
+
await asyncio_subprocesses.check_call(
|
7978
7994
|
*full_args,
|
7979
7995
|
env=env,
|
7980
7996
|
)
|