ominfra 0.0.0.dev147__py3-none-any.whl → 0.0.0.dev148__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/configs.py +30 -5
- ominfra/manage/commands/base.py +5 -5
- ominfra/manage/commands/execution.py +2 -2
- ominfra/manage/commands/inject.py +1 -1
- ominfra/manage/commands/interp.py +2 -2
- ominfra/manage/commands/subprocess.py +22 -14
- ominfra/manage/deploy/command.py +1 -1
- ominfra/manage/main.py +48 -32
- ominfra/manage/remote/_main.py +172 -0
- ominfra/manage/remote/channel.py +41 -16
- ominfra/manage/remote/config.py +10 -0
- ominfra/manage/remote/connection.py +106 -0
- ominfra/manage/remote/execution.py +244 -155
- ominfra/manage/remote/inject.py +7 -3
- ominfra/manage/remote/spawning.py +51 -33
- ominfra/pyremote.py +28 -3
- ominfra/scripts/journald2aws.py +127 -28
- ominfra/scripts/manage.py +1360 -485
- ominfra/scripts/supervisor.py +59 -10
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/RECORD +25 -23
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev147.dist-info → ominfra-0.0.0.dev148.dist-info}/top_level.txt +0 -0
@@ -1,17 +1,23 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
+
import abc
|
3
|
+
import asyncio
|
2
4
|
import contextlib
|
3
5
|
import dataclasses as dc
|
4
6
|
import shlex
|
5
7
|
import subprocess
|
6
8
|
import typing as ta
|
7
9
|
|
10
|
+
from omlish.lite.asyncio.subprocesses import asyncio_subprocess_popen
|
8
11
|
from omlish.lite.check import check_not_none
|
9
12
|
from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
|
10
13
|
from omlish.lite.subprocesses import SubprocessChannelOption
|
11
14
|
from omlish.lite.subprocesses import subprocess_maybe_shell_wrap_exec
|
12
15
|
|
13
16
|
|
14
|
-
|
17
|
+
##
|
18
|
+
|
19
|
+
|
20
|
+
class RemoteSpawning(abc.ABC):
|
15
21
|
@dc.dataclass(frozen=True)
|
16
22
|
class Target:
|
17
23
|
shell: ta.Optional[str] = None
|
@@ -22,15 +28,35 @@ class RemoteSpawning:
|
|
22
28
|
|
23
29
|
stderr: ta.Optional[str] = None # SubprocessChannelOption
|
24
30
|
|
25
|
-
|
31
|
+
@dc.dataclass(frozen=True)
|
32
|
+
class Spawned:
|
33
|
+
stdin: asyncio.StreamWriter
|
34
|
+
stdout: asyncio.StreamReader
|
35
|
+
stderr: ta.Optional[asyncio.StreamReader]
|
36
|
+
|
37
|
+
@abc.abstractmethod
|
38
|
+
def spawn(
|
39
|
+
self,
|
40
|
+
tgt: Target,
|
41
|
+
src: str,
|
42
|
+
*,
|
43
|
+
timeout: ta.Optional[float] = None,
|
44
|
+
debug: bool = False,
|
45
|
+
) -> ta.AsyncContextManager[Spawned]:
|
46
|
+
raise NotImplementedError
|
47
|
+
|
48
|
+
|
49
|
+
##
|
26
50
|
|
27
|
-
|
51
|
+
|
52
|
+
class SubprocessRemoteSpawning(RemoteSpawning):
|
53
|
+
class _PreparedCmd(ta.NamedTuple): # noqa
|
28
54
|
cmd: ta.Sequence[str]
|
29
55
|
shell: bool
|
30
56
|
|
31
57
|
def _prepare_cmd(
|
32
58
|
self,
|
33
|
-
tgt: Target,
|
59
|
+
tgt: RemoteSpawning.Target,
|
34
60
|
src: str,
|
35
61
|
) -> _PreparedCmd:
|
36
62
|
if tgt.shell is not None:
|
@@ -38,44 +64,38 @@ class RemoteSpawning:
|
|
38
64
|
if tgt.shell_quote:
|
39
65
|
sh_src = shlex.quote(sh_src)
|
40
66
|
sh_cmd = f'{tgt.shell} {sh_src}'
|
41
|
-
return
|
42
|
-
cmd=[sh_cmd],
|
43
|
-
shell=True,
|
44
|
-
)
|
67
|
+
return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
|
45
68
|
|
46
69
|
else:
|
47
|
-
return
|
48
|
-
cmd=[tgt.python, '-c', src],
|
49
|
-
shell=False,
|
50
|
-
)
|
70
|
+
return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
|
51
71
|
|
52
72
|
#
|
53
73
|
|
54
|
-
@
|
55
|
-
|
56
|
-
stdin: ta.IO
|
57
|
-
stdout: ta.IO
|
58
|
-
stderr: ta.Optional[ta.IO]
|
59
|
-
|
60
|
-
@contextlib.contextmanager
|
61
|
-
def spawn(
|
74
|
+
@contextlib.asynccontextmanager
|
75
|
+
async def spawn(
|
62
76
|
self,
|
63
|
-
tgt: Target,
|
77
|
+
tgt: RemoteSpawning.Target,
|
64
78
|
src: str,
|
65
79
|
*,
|
66
80
|
timeout: ta.Optional[float] = None,
|
67
|
-
|
81
|
+
debug: bool = False,
|
82
|
+
) -> ta.AsyncGenerator[RemoteSpawning.Spawned, None]:
|
68
83
|
pc = self._prepare_cmd(tgt, src)
|
69
84
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
85
|
+
cmd = pc.cmd
|
86
|
+
if not debug:
|
87
|
+
cmd = subprocess_maybe_shell_wrap_exec(*cmd)
|
88
|
+
|
89
|
+
async with asyncio_subprocess_popen(
|
90
|
+
*cmd,
|
91
|
+
shell=pc.shell,
|
92
|
+
stdin=subprocess.PIPE,
|
93
|
+
stdout=subprocess.PIPE,
|
94
|
+
stderr=(
|
95
|
+
SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
|
96
|
+
if tgt.stderr is not None else None
|
97
|
+
),
|
98
|
+
timeout=timeout,
|
79
99
|
) as proc:
|
80
100
|
stdin = check_not_none(proc.stdin)
|
81
101
|
stdout = check_not_none(proc.stdout)
|
@@ -92,5 +112,3 @@ class RemoteSpawning:
|
|
92
112
|
stdin.close()
|
93
113
|
except BrokenPipeError:
|
94
114
|
pass
|
95
|
-
|
96
|
-
proc.wait(timeout)
|
ominfra/pyremote.py
CHANGED
@@ -324,10 +324,8 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
324
324
|
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
325
325
|
main_src = sf.read()
|
326
326
|
|
327
|
-
# Restore
|
327
|
+
# Restore vars
|
328
328
|
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
329
|
-
|
330
|
-
# Grab context name
|
331
329
|
context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
|
332
330
|
|
333
331
|
# Write third ack
|
@@ -501,3 +499,30 @@ class PyremoteBootstrapDriver:
|
|
501
499
|
output.flush()
|
502
500
|
else:
|
503
501
|
raise TypeError(go)
|
502
|
+
|
503
|
+
async def async_run(
|
504
|
+
self,
|
505
|
+
input: ta.Any, # asyncio.StreamWriter # noqa
|
506
|
+
output: ta.Any, # asyncio.StreamReader
|
507
|
+
) -> Result:
|
508
|
+
gen = self.gen()
|
509
|
+
|
510
|
+
gi: ta.Optional[bytes] = None
|
511
|
+
while True:
|
512
|
+
try:
|
513
|
+
if gi is not None:
|
514
|
+
go = gen.send(gi)
|
515
|
+
else:
|
516
|
+
go = next(gen)
|
517
|
+
except StopIteration as e:
|
518
|
+
return e.value
|
519
|
+
|
520
|
+
if isinstance(go, self.Read):
|
521
|
+
if len(gi := await input.read(go.sz)) != go.sz:
|
522
|
+
raise EOFError
|
523
|
+
elif isinstance(go, self.Write):
|
524
|
+
gi = None
|
525
|
+
output.write(go.d)
|
526
|
+
await output.drain()
|
527
|
+
else:
|
528
|
+
raise TypeError(go)
|
ominfra/scripts/journald2aws.py
CHANGED
@@ -898,7 +898,10 @@ def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
|
|
898
898
|
# ../../../../../omlish/lite/cached.py
|
899
899
|
|
900
900
|
|
901
|
-
|
901
|
+
##
|
902
|
+
|
903
|
+
|
904
|
+
class _AbstractCachedNullary:
|
902
905
|
def __init__(self, fn):
|
903
906
|
super().__init__()
|
904
907
|
self._fn = fn
|
@@ -906,17 +909,25 @@ class _cached_nullary: # noqa
|
|
906
909
|
functools.update_wrapper(self, fn)
|
907
910
|
|
908
911
|
def __call__(self, *args, **kwargs): # noqa
|
909
|
-
|
910
|
-
self._value = self._fn()
|
911
|
-
return self._value
|
912
|
+
raise TypeError
|
912
913
|
|
913
914
|
def __get__(self, instance, owner): # noqa
|
914
915
|
bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
|
915
916
|
return bound
|
916
917
|
|
917
918
|
|
919
|
+
##
|
920
|
+
|
921
|
+
|
922
|
+
class _CachedNullary(_AbstractCachedNullary):
|
923
|
+
def __call__(self, *args, **kwargs): # noqa
|
924
|
+
if self._value is self._missing:
|
925
|
+
self._value = self._fn()
|
926
|
+
return self._value
|
927
|
+
|
928
|
+
|
918
929
|
def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
919
|
-
return
|
930
|
+
return _CachedNullary(fn)
|
920
931
|
|
921
932
|
|
922
933
|
def static_init(fn: CallableT) -> CallableT:
|
@@ -925,6 +936,20 @@ def static_init(fn: CallableT) -> CallableT:
|
|
925
936
|
return fn
|
926
937
|
|
927
938
|
|
939
|
+
##
|
940
|
+
|
941
|
+
|
942
|
+
class _AsyncCachedNullary(_AbstractCachedNullary):
|
943
|
+
async def __call__(self, *args, **kwargs):
|
944
|
+
if self._value is self._missing:
|
945
|
+
self._value = await self._fn()
|
946
|
+
return self._value
|
947
|
+
|
948
|
+
|
949
|
+
def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
950
|
+
return _AsyncCachedNullary(fn)
|
951
|
+
|
952
|
+
|
928
953
|
########################################
|
929
954
|
# ../../../../../omlish/lite/check.py
|
930
955
|
|
@@ -2831,6 +2856,33 @@ class AwsLogMessageBuilder:
|
|
2831
2856
|
# ../../../../configs.py
|
2832
2857
|
|
2833
2858
|
|
2859
|
+
def parse_config_file(
|
2860
|
+
name: str,
|
2861
|
+
f: ta.TextIO,
|
2862
|
+
) -> ConfigMapping:
|
2863
|
+
if name.endswith('.toml'):
|
2864
|
+
return toml_loads(f.read())
|
2865
|
+
|
2866
|
+
elif any(name.endswith(e) for e in ('.yml', '.yaml')):
|
2867
|
+
yaml = __import__('yaml')
|
2868
|
+
return yaml.safe_load(f)
|
2869
|
+
|
2870
|
+
elif name.endswith('.ini'):
|
2871
|
+
import configparser
|
2872
|
+
cp = configparser.ConfigParser()
|
2873
|
+
cp.read_file(f)
|
2874
|
+
config_dct: ta.Dict[str, ta.Any] = {}
|
2875
|
+
for sec in cp.sections():
|
2876
|
+
cd = config_dct
|
2877
|
+
for k in sec.split('.'):
|
2878
|
+
cd = cd.setdefault(k, {})
|
2879
|
+
cd.update(cp.items(sec))
|
2880
|
+
return config_dct
|
2881
|
+
|
2882
|
+
else:
|
2883
|
+
return json.loads(f.read())
|
2884
|
+
|
2885
|
+
|
2834
2886
|
def read_config_file(
|
2835
2887
|
path: str,
|
2836
2888
|
cls: ta.Type[T],
|
@@ -2838,13 +2890,10 @@ def read_config_file(
|
|
2838
2890
|
prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
|
2839
2891
|
) -> T:
|
2840
2892
|
with open(path) as cf:
|
2841
|
-
|
2842
|
-
config_dct = toml_loads(cf.read())
|
2843
|
-
else:
|
2844
|
-
config_dct = json.loads(cf.read())
|
2893
|
+
config_dct = parse_config_file(os.path.basename(path), cf)
|
2845
2894
|
|
2846
2895
|
if prepare is not None:
|
2847
|
-
config_dct = prepare(config_dct)
|
2896
|
+
config_dct = prepare(config_dct)
|
2848
2897
|
|
2849
2898
|
return unmarshal_obj(config_dct, cls)
|
2850
2899
|
|
@@ -3177,7 +3226,7 @@ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
|
|
3177
3226
|
return args
|
3178
3227
|
|
3179
3228
|
|
3180
|
-
def
|
3229
|
+
def prepare_subprocess_invocation(
|
3181
3230
|
*args: str,
|
3182
3231
|
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
3183
3232
|
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
@@ -3185,9 +3234,9 @@ def _prepare_subprocess_invocation(
|
|
3185
3234
|
shell: bool = False,
|
3186
3235
|
**kwargs: ta.Any,
|
3187
3236
|
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
3188
|
-
log.debug(args)
|
3237
|
+
log.debug('prepare_subprocess_invocation: args=%r', args)
|
3189
3238
|
if extra_env:
|
3190
|
-
log.debug(extra_env)
|
3239
|
+
log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
|
3191
3240
|
|
3192
3241
|
if extra_env:
|
3193
3242
|
env = {**(env if env is not None else os.environ), **extra_env}
|
@@ -3206,14 +3255,46 @@ def _prepare_subprocess_invocation(
|
|
3206
3255
|
)
|
3207
3256
|
|
3208
3257
|
|
3209
|
-
|
3210
|
-
args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
|
3211
|
-
return subprocess.check_call(args, **kwargs) # type: ignore
|
3258
|
+
##
|
3212
3259
|
|
3213
3260
|
|
3214
|
-
|
3215
|
-
|
3216
|
-
|
3261
|
+
@contextlib.contextmanager
|
3262
|
+
def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
|
3263
|
+
start_time = time.time()
|
3264
|
+
try:
|
3265
|
+
log.debug('subprocess_common_context.try: args=%r', args)
|
3266
|
+
yield
|
3267
|
+
|
3268
|
+
except Exception as exc: # noqa
|
3269
|
+
log.debug('subprocess_common_context.except: exc=%r', exc)
|
3270
|
+
raise
|
3271
|
+
|
3272
|
+
finally:
|
3273
|
+
end_time = time.time()
|
3274
|
+
elapsed_s = end_time - start_time
|
3275
|
+
log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
|
3276
|
+
|
3277
|
+
|
3278
|
+
##
|
3279
|
+
|
3280
|
+
|
3281
|
+
def subprocess_check_call(
|
3282
|
+
*args: str,
|
3283
|
+
stdout: ta.Any = sys.stderr,
|
3284
|
+
**kwargs: ta.Any,
|
3285
|
+
) -> None:
|
3286
|
+
args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
|
3287
|
+
with subprocess_common_context(*args, **kwargs):
|
3288
|
+
return subprocess.check_call(args, **kwargs) # type: ignore
|
3289
|
+
|
3290
|
+
|
3291
|
+
def subprocess_check_output(
|
3292
|
+
*args: str,
|
3293
|
+
**kwargs: ta.Any,
|
3294
|
+
) -> bytes:
|
3295
|
+
args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
|
3296
|
+
with subprocess_common_context(*args, **kwargs):
|
3297
|
+
return subprocess.check_output(args, **kwargs)
|
3217
3298
|
|
3218
3299
|
|
3219
3300
|
def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
|
@@ -3229,16 +3310,31 @@ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
|
|
3229
3310
|
)
|
3230
3311
|
|
3231
3312
|
|
3232
|
-
def
|
3233
|
-
|
3313
|
+
def _subprocess_try_run(
|
3314
|
+
fn: ta.Callable[..., T],
|
3315
|
+
*args: ta.Any,
|
3234
3316
|
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
3235
3317
|
**kwargs: ta.Any,
|
3236
|
-
) ->
|
3318
|
+
) -> ta.Union[T, Exception]:
|
3237
3319
|
try:
|
3238
|
-
|
3320
|
+
return fn(*args, **kwargs)
|
3239
3321
|
except try_exceptions as e: # noqa
|
3240
3322
|
if log.isEnabledFor(logging.DEBUG):
|
3241
3323
|
log.exception('command failed')
|
3324
|
+
return e
|
3325
|
+
|
3326
|
+
|
3327
|
+
def subprocess_try_call(
|
3328
|
+
*args: str,
|
3329
|
+
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
3330
|
+
**kwargs: ta.Any,
|
3331
|
+
) -> bool:
|
3332
|
+
if isinstance(_subprocess_try_run(
|
3333
|
+
subprocess_check_call,
|
3334
|
+
*args,
|
3335
|
+
try_exceptions=try_exceptions,
|
3336
|
+
**kwargs,
|
3337
|
+
), Exception):
|
3242
3338
|
return False
|
3243
3339
|
else:
|
3244
3340
|
return True
|
@@ -3249,12 +3345,15 @@ def subprocess_try_output(
|
|
3249
3345
|
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
3250
3346
|
**kwargs: ta.Any,
|
3251
3347
|
) -> ta.Optional[bytes]:
|
3252
|
-
|
3253
|
-
|
3254
|
-
|
3255
|
-
|
3256
|
-
|
3348
|
+
if isinstance(ret := _subprocess_try_run(
|
3349
|
+
subprocess_check_output,
|
3350
|
+
*args,
|
3351
|
+
try_exceptions=try_exceptions,
|
3352
|
+
**kwargs,
|
3353
|
+
), Exception):
|
3257
3354
|
return None
|
3355
|
+
else:
|
3356
|
+
return ret
|
3258
3357
|
|
3259
3358
|
|
3260
3359
|
def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|