ominfra 0.0.0.dev147__py3-none-any.whl → 0.0.0.dev148__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,17 +1,23 @@
1
1
  # ruff: noqa: UP006 UP007
2
+ import abc
3
+ import asyncio
2
4
  import contextlib
3
5
  import dataclasses as dc
4
6
  import shlex
5
7
  import subprocess
6
8
  import typing as ta
7
9
 
10
+ from omlish.lite.asyncio.subprocesses import asyncio_subprocess_popen
8
11
  from omlish.lite.check import check_not_none
9
12
  from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
10
13
  from omlish.lite.subprocesses import SubprocessChannelOption
11
14
  from omlish.lite.subprocesses import subprocess_maybe_shell_wrap_exec
12
15
 
13
16
 
14
- class RemoteSpawning:
17
+ ##
18
+
19
+
20
+ class RemoteSpawning(abc.ABC):
15
21
  @dc.dataclass(frozen=True)
16
22
  class Target:
17
23
  shell: ta.Optional[str] = None
@@ -22,15 +28,35 @@ class RemoteSpawning:
22
28
 
23
29
  stderr: ta.Optional[str] = None # SubprocessChannelOption
24
30
 
25
- #
31
+ @dc.dataclass(frozen=True)
32
+ class Spawned:
33
+ stdin: asyncio.StreamWriter
34
+ stdout: asyncio.StreamReader
35
+ stderr: ta.Optional[asyncio.StreamReader]
36
+
37
+ @abc.abstractmethod
38
+ def spawn(
39
+ self,
40
+ tgt: Target,
41
+ src: str,
42
+ *,
43
+ timeout: ta.Optional[float] = None,
44
+ debug: bool = False,
45
+ ) -> ta.AsyncContextManager[Spawned]:
46
+ raise NotImplementedError
47
+
48
+
49
+ ##
26
50
 
27
- class _PreparedCmd(ta.NamedTuple):
51
+
52
+ class SubprocessRemoteSpawning(RemoteSpawning):
53
+ class _PreparedCmd(ta.NamedTuple): # noqa
28
54
  cmd: ta.Sequence[str]
29
55
  shell: bool
30
56
 
31
57
  def _prepare_cmd(
32
58
  self,
33
- tgt: Target,
59
+ tgt: RemoteSpawning.Target,
34
60
  src: str,
35
61
  ) -> _PreparedCmd:
36
62
  if tgt.shell is not None:
@@ -38,44 +64,38 @@ class RemoteSpawning:
38
64
  if tgt.shell_quote:
39
65
  sh_src = shlex.quote(sh_src)
40
66
  sh_cmd = f'{tgt.shell} {sh_src}'
41
- return RemoteSpawning._PreparedCmd(
42
- cmd=[sh_cmd],
43
- shell=True,
44
- )
67
+ return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
45
68
 
46
69
  else:
47
- return RemoteSpawning._PreparedCmd(
48
- cmd=[tgt.python, '-c', src],
49
- shell=False,
50
- )
70
+ return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
51
71
 
52
72
  #
53
73
 
54
- @dc.dataclass(frozen=True)
55
- class Spawned:
56
- stdin: ta.IO
57
- stdout: ta.IO
58
- stderr: ta.Optional[ta.IO]
59
-
60
- @contextlib.contextmanager
61
- def spawn(
74
+ @contextlib.asynccontextmanager
75
+ async def spawn(
62
76
  self,
63
- tgt: Target,
77
+ tgt: RemoteSpawning.Target,
64
78
  src: str,
65
79
  *,
66
80
  timeout: ta.Optional[float] = None,
67
- ) -> ta.Generator[Spawned, None, None]:
81
+ debug: bool = False,
82
+ ) -> ta.AsyncGenerator[RemoteSpawning.Spawned, None]:
68
83
  pc = self._prepare_cmd(tgt, src)
69
84
 
70
- with subprocess.Popen(
71
- subprocess_maybe_shell_wrap_exec(*pc.cmd),
72
- shell=pc.shell,
73
- stdin=subprocess.PIPE,
74
- stdout=subprocess.PIPE,
75
- stderr=(
76
- SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
77
- if tgt.stderr is not None else None
78
- ),
85
+ cmd = pc.cmd
86
+ if not debug:
87
+ cmd = subprocess_maybe_shell_wrap_exec(*cmd)
88
+
89
+ async with asyncio_subprocess_popen(
90
+ *cmd,
91
+ shell=pc.shell,
92
+ stdin=subprocess.PIPE,
93
+ stdout=subprocess.PIPE,
94
+ stderr=(
95
+ SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
96
+ if tgt.stderr is not None else None
97
+ ),
98
+ timeout=timeout,
79
99
  ) as proc:
80
100
  stdin = check_not_none(proc.stdin)
81
101
  stdout = check_not_none(proc.stdout)
@@ -92,5 +112,3 @@ class RemoteSpawning:
92
112
  stdin.close()
93
113
  except BrokenPipeError:
94
114
  pass
95
-
96
- proc.wait(timeout)
ominfra/pyremote.py CHANGED
@@ -324,10 +324,8 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
324
324
  with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
325
325
  main_src = sf.read()
326
326
 
327
- # Restore original argv0
327
+ # Restore vars
328
328
  sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
329
-
330
- # Grab context name
331
329
  context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
332
330
 
333
331
  # Write third ack
@@ -501,3 +499,30 @@ class PyremoteBootstrapDriver:
501
499
  output.flush()
502
500
  else:
503
501
  raise TypeError(go)
502
+
503
+ async def async_run(
504
+ self,
505
+ input: ta.Any, # asyncio.StreamWriter # noqa
506
+ output: ta.Any, # asyncio.StreamReader
507
+ ) -> Result:
508
+ gen = self.gen()
509
+
510
+ gi: ta.Optional[bytes] = None
511
+ while True:
512
+ try:
513
+ if gi is not None:
514
+ go = gen.send(gi)
515
+ else:
516
+ go = next(gen)
517
+ except StopIteration as e:
518
+ return e.value
519
+
520
+ if isinstance(go, self.Read):
521
+ if len(gi := await input.read(go.sz)) != go.sz:
522
+ raise EOFError
523
+ elif isinstance(go, self.Write):
524
+ gi = None
525
+ output.write(go.d)
526
+ await output.drain()
527
+ else:
528
+ raise TypeError(go)
@@ -898,7 +898,10 @@ def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
898
898
  # ../../../../../omlish/lite/cached.py
899
899
 
900
900
 
901
- class _cached_nullary: # noqa
901
+ ##
902
+
903
+
904
+ class _AbstractCachedNullary:
902
905
  def __init__(self, fn):
903
906
  super().__init__()
904
907
  self._fn = fn
@@ -906,17 +909,25 @@ class _cached_nullary: # noqa
906
909
  functools.update_wrapper(self, fn)
907
910
 
908
911
  def __call__(self, *args, **kwargs): # noqa
909
- if self._value is self._missing:
910
- self._value = self._fn()
911
- return self._value
912
+ raise TypeError
912
913
 
913
914
  def __get__(self, instance, owner): # noqa
914
915
  bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
915
916
  return bound
916
917
 
917
918
 
919
+ ##
920
+
921
+
922
+ class _CachedNullary(_AbstractCachedNullary):
923
+ def __call__(self, *args, **kwargs): # noqa
924
+ if self._value is self._missing:
925
+ self._value = self._fn()
926
+ return self._value
927
+
928
+
918
929
  def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
919
- return _cached_nullary(fn)
930
+ return _CachedNullary(fn)
920
931
 
921
932
 
922
933
  def static_init(fn: CallableT) -> CallableT:
@@ -925,6 +936,20 @@ def static_init(fn: CallableT) -> CallableT:
925
936
  return fn
926
937
 
927
938
 
939
+ ##
940
+
941
+
942
+ class _AsyncCachedNullary(_AbstractCachedNullary):
943
+ async def __call__(self, *args, **kwargs):
944
+ if self._value is self._missing:
945
+ self._value = await self._fn()
946
+ return self._value
947
+
948
+
949
+ def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
950
+ return _AsyncCachedNullary(fn)
951
+
952
+
928
953
  ########################################
929
954
  # ../../../../../omlish/lite/check.py
930
955
 
@@ -2831,6 +2856,33 @@ class AwsLogMessageBuilder:
2831
2856
  # ../../../../configs.py
2832
2857
 
2833
2858
 
2859
+ def parse_config_file(
2860
+ name: str,
2861
+ f: ta.TextIO,
2862
+ ) -> ConfigMapping:
2863
+ if name.endswith('.toml'):
2864
+ return toml_loads(f.read())
2865
+
2866
+ elif any(name.endswith(e) for e in ('.yml', '.yaml')):
2867
+ yaml = __import__('yaml')
2868
+ return yaml.safe_load(f)
2869
+
2870
+ elif name.endswith('.ini'):
2871
+ import configparser
2872
+ cp = configparser.ConfigParser()
2873
+ cp.read_file(f)
2874
+ config_dct: ta.Dict[str, ta.Any] = {}
2875
+ for sec in cp.sections():
2876
+ cd = config_dct
2877
+ for k in sec.split('.'):
2878
+ cd = cd.setdefault(k, {})
2879
+ cd.update(cp.items(sec))
2880
+ return config_dct
2881
+
2882
+ else:
2883
+ return json.loads(f.read())
2884
+
2885
+
2834
2886
  def read_config_file(
2835
2887
  path: str,
2836
2888
  cls: ta.Type[T],
@@ -2838,13 +2890,10 @@ def read_config_file(
2838
2890
  prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
2839
2891
  ) -> T:
2840
2892
  with open(path) as cf:
2841
- if path.endswith('.toml'):
2842
- config_dct = toml_loads(cf.read())
2843
- else:
2844
- config_dct = json.loads(cf.read())
2893
+ config_dct = parse_config_file(os.path.basename(path), cf)
2845
2894
 
2846
2895
  if prepare is not None:
2847
- config_dct = prepare(config_dct) # type: ignore
2896
+ config_dct = prepare(config_dct)
2848
2897
 
2849
2898
  return unmarshal_obj(config_dct, cls)
2850
2899
 
@@ -3177,7 +3226,7 @@ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
3177
3226
  return args
3178
3227
 
3179
3228
 
3180
- def _prepare_subprocess_invocation(
3229
+ def prepare_subprocess_invocation(
3181
3230
  *args: str,
3182
3231
  env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
3183
3232
  extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
@@ -3185,9 +3234,9 @@ def _prepare_subprocess_invocation(
3185
3234
  shell: bool = False,
3186
3235
  **kwargs: ta.Any,
3187
3236
  ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
3188
- log.debug(args)
3237
+ log.debug('prepare_subprocess_invocation: args=%r', args)
3189
3238
  if extra_env:
3190
- log.debug(extra_env)
3239
+ log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
3191
3240
 
3192
3241
  if extra_env:
3193
3242
  env = {**(env if env is not None else os.environ), **extra_env}
@@ -3206,14 +3255,46 @@ def _prepare_subprocess_invocation(
3206
3255
  )
3207
3256
 
3208
3257
 
3209
- def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
3210
- args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
3211
- return subprocess.check_call(args, **kwargs) # type: ignore
3258
+ ##
3212
3259
 
3213
3260
 
3214
- def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
3215
- args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
3216
- return subprocess.check_output(args, **kwargs)
3261
+ @contextlib.contextmanager
3262
+ def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
3263
+ start_time = time.time()
3264
+ try:
3265
+ log.debug('subprocess_common_context.try: args=%r', args)
3266
+ yield
3267
+
3268
+ except Exception as exc: # noqa
3269
+ log.debug('subprocess_common_context.except: exc=%r', exc)
3270
+ raise
3271
+
3272
+ finally:
3273
+ end_time = time.time()
3274
+ elapsed_s = end_time - start_time
3275
+ log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
3276
+
3277
+
3278
+ ##
3279
+
3280
+
3281
+ def subprocess_check_call(
3282
+ *args: str,
3283
+ stdout: ta.Any = sys.stderr,
3284
+ **kwargs: ta.Any,
3285
+ ) -> None:
3286
+ args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
3287
+ with subprocess_common_context(*args, **kwargs):
3288
+ return subprocess.check_call(args, **kwargs) # type: ignore
3289
+
3290
+
3291
+ def subprocess_check_output(
3292
+ *args: str,
3293
+ **kwargs: ta.Any,
3294
+ ) -> bytes:
3295
+ args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
3296
+ with subprocess_common_context(*args, **kwargs):
3297
+ return subprocess.check_output(args, **kwargs)
3217
3298
 
3218
3299
 
3219
3300
  def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
@@ -3229,16 +3310,31 @@ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
3229
3310
  )
3230
3311
 
3231
3312
 
3232
- def subprocess_try_call(
3233
- *args: str,
3313
+ def _subprocess_try_run(
3314
+ fn: ta.Callable[..., T],
3315
+ *args: ta.Any,
3234
3316
  try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
3235
3317
  **kwargs: ta.Any,
3236
- ) -> bool:
3318
+ ) -> ta.Union[T, Exception]:
3237
3319
  try:
3238
- subprocess_check_call(*args, **kwargs)
3320
+ return fn(*args, **kwargs)
3239
3321
  except try_exceptions as e: # noqa
3240
3322
  if log.isEnabledFor(logging.DEBUG):
3241
3323
  log.exception('command failed')
3324
+ return e
3325
+
3326
+
3327
+ def subprocess_try_call(
3328
+ *args: str,
3329
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
3330
+ **kwargs: ta.Any,
3331
+ ) -> bool:
3332
+ if isinstance(_subprocess_try_run(
3333
+ subprocess_check_call,
3334
+ *args,
3335
+ try_exceptions=try_exceptions,
3336
+ **kwargs,
3337
+ ), Exception):
3242
3338
  return False
3243
3339
  else:
3244
3340
  return True
@@ -3249,12 +3345,15 @@ def subprocess_try_output(
3249
3345
  try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
3250
3346
  **kwargs: ta.Any,
3251
3347
  ) -> ta.Optional[bytes]:
3252
- try:
3253
- return subprocess_check_output(*args, **kwargs)
3254
- except try_exceptions as e: # noqa
3255
- if log.isEnabledFor(logging.DEBUG):
3256
- log.exception('command failed')
3348
+ if isinstance(ret := _subprocess_try_run(
3349
+ subprocess_check_output,
3350
+ *args,
3351
+ try_exceptions=try_exceptions,
3352
+ **kwargs,
3353
+ ), Exception):
3257
3354
  return None
3355
+ else:
3356
+ return ret
3258
3357
 
3259
3358
 
3260
3359
  def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]: