ominfra 0.0.0.dev132__py3-none-any.whl → 0.0.0.dev134__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/{deploy → manage/deploy}/_executor.py +10 -10
- ominfra/{deploy → manage/deploy}/poly/_main.py +6 -6
- ominfra/{deploy → manage/deploy}/remote.py +1 -1
- ominfra/pyremote.py +389 -0
- ominfra/scripts/supervisor.py +1 -1
- ominfra/supervisor/http.py +1 -1
- {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/RECORD +36 -39
- ominfra/pyremote/__init__.py +0 -0
- ominfra/pyremote/_runcommands.py +0 -1201
- ominfra/pyremote/bootstrap.py +0 -149
- ominfra/pyremote/runcommands.py +0 -56
- /ominfra/{deploy → manage/deploy}/__init__.py +0 -0
- /ominfra/{deploy → manage/deploy}/configs.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/__init__.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/base.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/concerns/__init__.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/concerns/dirs.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/concerns/nginx.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/concerns/repo.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/concerns/supervisor.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/concerns/systemd.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/concerns/user.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/concerns/venv.py +0 -0
- /ominfra/{deploy → manage/deploy}/executor/main.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/__init__.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/base.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/configs.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/deploy.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/main.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/nginx.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/repo.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/runtime.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/site.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/supervisor.py +0 -0
- /ominfra/{deploy → manage/deploy}/poly/venv.py +0 -0
- {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/top_level.txt +0 -0
ominfra/pyremote/_runcommands.py
DELETED
@@ -1,1201 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python3
|
2
|
-
# noinspection DuplicatedCode
|
3
|
-
# @omlish-lite
|
4
|
-
# @omlish-script
|
5
|
-
# @omlish-amalg-output runcommands.py
|
6
|
-
# ruff: noqa: N802 UP006 UP007 UP036
|
7
|
-
import abc
|
8
|
-
import base64
|
9
|
-
import collections.abc
|
10
|
-
import contextlib
|
11
|
-
import dataclasses as dc
|
12
|
-
import datetime
|
13
|
-
import decimal
|
14
|
-
import enum
|
15
|
-
import fractions
|
16
|
-
import functools
|
17
|
-
import inspect
|
18
|
-
import io
|
19
|
-
import json
|
20
|
-
import logging
|
21
|
-
import os
|
22
|
-
import shlex
|
23
|
-
import subprocess
|
24
|
-
import sys
|
25
|
-
import textwrap
|
26
|
-
import threading
|
27
|
-
import types
|
28
|
-
import typing as ta
|
29
|
-
import uuid
|
30
|
-
import weakref # noqa
|
31
|
-
import zlib
|
32
|
-
|
33
|
-
|
34
|
-
########################################
|
35
|
-
|
36
|
-
|
37
|
-
if sys.version_info < (3, 8):
|
38
|
-
raise OSError(f'Requires python (3, 8), got {sys.version_info} from {sys.executable}') # noqa
|
39
|
-
|
40
|
-
|
41
|
-
########################################
|
42
|
-
|
43
|
-
|
44
|
-
# ../../omlish/lite/cached.py
|
45
|
-
T = ta.TypeVar('T')
|
46
|
-
|
47
|
-
# ../../omlish/lite/check.py
|
48
|
-
SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
|
49
|
-
|
50
|
-
|
51
|
-
########################################
|
52
|
-
# ../bootstrap.py
|
53
|
-
"""
|
54
|
-
Basically this: https://mitogen.networkgenomics.com/howitworks.html
|
55
|
-
"""
|
56
|
-
|
57
|
-
|
58
|
-
##
|
59
|
-
|
60
|
-
|
61
|
-
_BOOTSTRAP_COMM_FD = 100
|
62
|
-
_BOOTSTRAP_SRC_FD = 101
|
63
|
-
|
64
|
-
_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CPID'
|
65
|
-
_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
|
66
|
-
|
67
|
-
BOOTSTRAP_ACK0 = b'OPYR000\n'
|
68
|
-
BOOTSTRAP_ACK1 = b'OPYR001\n'
|
69
|
-
|
70
|
-
_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
|
71
|
-
|
72
|
-
_BOOTSTRAP_IMPORTS = [
|
73
|
-
'base64',
|
74
|
-
'os',
|
75
|
-
'sys',
|
76
|
-
'zlib',
|
77
|
-
]
|
78
|
-
|
79
|
-
|
80
|
-
def _bootstrap_main(context_name: str, main_z_len: int) -> None:
|
81
|
-
# Two copies of main src to be sent to parent
|
82
|
-
r0, w0 = os.pipe()
|
83
|
-
r1, w1 = os.pipe()
|
84
|
-
|
85
|
-
if (cp := os.fork()):
|
86
|
-
# Parent process
|
87
|
-
|
88
|
-
# Dup original stdin to comm_fd for use as comm channel
|
89
|
-
os.dup2(0, _BOOTSTRAP_COMM_FD)
|
90
|
-
|
91
|
-
# Overwrite stdin (fed to python repl) with first copy of src
|
92
|
-
os.dup2(r0, 0)
|
93
|
-
|
94
|
-
# Dup second copy of src to src_fd to recover after launch
|
95
|
-
os.dup2(r1, _BOOTSTRAP_SRC_FD)
|
96
|
-
|
97
|
-
# Close remaining fd's
|
98
|
-
for f in [r0, w0, r1, w1]:
|
99
|
-
os.close(f)
|
100
|
-
|
101
|
-
# Save child pid to close after relaunch
|
102
|
-
os.environ[_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
|
103
|
-
|
104
|
-
# Save original argv0
|
105
|
-
os.environ[_BOOTSTRAP_ARGV0_VAR] = sys.executable
|
106
|
-
|
107
|
-
# Start repl reading stdin from r0
|
108
|
-
os.execl(sys.executable, sys.executable + (_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
|
109
|
-
|
110
|
-
else:
|
111
|
-
# Child process
|
112
|
-
|
113
|
-
# Write first ack
|
114
|
-
os.write(1, BOOTSTRAP_ACK0)
|
115
|
-
|
116
|
-
# Read main src from stdin
|
117
|
-
main_src = zlib.decompress(os.fdopen(0, 'rb').read(main_z_len))
|
118
|
-
|
119
|
-
# Write both copies of main src
|
120
|
-
for w in [w0, w1]:
|
121
|
-
fp = os.fdopen(w, 'wb', 0)
|
122
|
-
fp.write(main_src)
|
123
|
-
fp.close()
|
124
|
-
|
125
|
-
# Write second ack
|
126
|
-
os.write(1, BOOTSTRAP_ACK1)
|
127
|
-
|
128
|
-
sys.exit(0)
|
129
|
-
|
130
|
-
|
131
|
-
#
|
132
|
-
|
133
|
-
|
134
|
-
def bootstrap_payload(context_name: str, main_z_len: int) -> str:
|
135
|
-
bs_src = textwrap.dedent(inspect.getsource(_bootstrap_main))
|
136
|
-
|
137
|
-
for gl in [
|
138
|
-
'_BOOTSTRAP_COMM_FD',
|
139
|
-
'_BOOTSTRAP_SRC_FD',
|
140
|
-
|
141
|
-
'_BOOTSTRAP_CHILD_PID_VAR',
|
142
|
-
'_BOOTSTRAP_ARGV0_VAR',
|
143
|
-
|
144
|
-
'BOOTSTRAP_ACK0',
|
145
|
-
'BOOTSTRAP_ACK1',
|
146
|
-
|
147
|
-
'_BOOTSTRAP_PROC_TITLE_FMT',
|
148
|
-
]:
|
149
|
-
bs_src = bs_src.replace(gl, repr(globals()[gl]))
|
150
|
-
|
151
|
-
bs_src = '\n'.join(
|
152
|
-
cl
|
153
|
-
for l in bs_src.splitlines()
|
154
|
-
if (cl := (l.split('#')[0]).rstrip())
|
155
|
-
if cl.strip()
|
156
|
-
)
|
157
|
-
|
158
|
-
bs_z = zlib.compress(bs_src.encode('utf-8'))
|
159
|
-
bs_z64 = base64.encodebytes(bs_z).replace(b'\n', b'')
|
160
|
-
|
161
|
-
stmts = [
|
162
|
-
f'import {", ".join(_BOOTSTRAP_IMPORTS)}',
|
163
|
-
f'exec(zlib.decompress(base64.decodebytes({bs_z64!r})))',
|
164
|
-
f'_bootstrap_main({context_name!r}, {main_z_len})',
|
165
|
-
]
|
166
|
-
|
167
|
-
cmd = '; '.join(stmts)
|
168
|
-
return cmd
|
169
|
-
|
170
|
-
|
171
|
-
#
|
172
|
-
|
173
|
-
|
174
|
-
class PostBoostrap(ta.NamedTuple):
|
175
|
-
input: ta.BinaryIO
|
176
|
-
main_src: str
|
177
|
-
|
178
|
-
|
179
|
-
def post_boostrap() -> PostBoostrap:
|
180
|
-
# Restore original argv0
|
181
|
-
sys.executable = os.environ.pop(_BOOTSTRAP_ARGV0_VAR)
|
182
|
-
|
183
|
-
# Reap boostrap child
|
184
|
-
os.waitpid(int(os.environ.pop(_BOOTSTRAP_CHILD_PID_VAR)), 0)
|
185
|
-
|
186
|
-
# Read second copy of main src
|
187
|
-
r1 = os.fdopen(_BOOTSTRAP_SRC_FD, 'rb', 0)
|
188
|
-
main_src = r1.read().decode('utf-8')
|
189
|
-
r1.close()
|
190
|
-
|
191
|
-
return PostBoostrap(
|
192
|
-
input=os.fdopen(_BOOTSTRAP_COMM_FD, 'rb', 0),
|
193
|
-
main_src=main_src,
|
194
|
-
)
|
195
|
-
|
196
|
-
|
197
|
-
########################################
|
198
|
-
# ../../../omlish/lite/cached.py
|
199
|
-
|
200
|
-
|
201
|
-
class _cached_nullary: # noqa
|
202
|
-
def __init__(self, fn):
|
203
|
-
super().__init__()
|
204
|
-
self._fn = fn
|
205
|
-
self._value = self._missing = object()
|
206
|
-
functools.update_wrapper(self, fn)
|
207
|
-
|
208
|
-
def __call__(self, *args, **kwargs): # noqa
|
209
|
-
if self._value is self._missing:
|
210
|
-
self._value = self._fn()
|
211
|
-
return self._value
|
212
|
-
|
213
|
-
def __get__(self, instance, owner): # noqa
|
214
|
-
bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
|
215
|
-
return bound
|
216
|
-
|
217
|
-
|
218
|
-
def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
219
|
-
return _cached_nullary(fn)
|
220
|
-
|
221
|
-
|
222
|
-
########################################
|
223
|
-
# ../../../omlish/lite/check.py
|
224
|
-
|
225
|
-
|
226
|
-
def check_isinstance(v: ta.Any, spec: ta.Union[ta.Type[T], tuple]) -> T:
|
227
|
-
if not isinstance(v, spec):
|
228
|
-
raise TypeError(v)
|
229
|
-
return v
|
230
|
-
|
231
|
-
|
232
|
-
def check_not_isinstance(v: T, spec: ta.Union[type, tuple]) -> T:
|
233
|
-
if isinstance(v, spec):
|
234
|
-
raise TypeError(v)
|
235
|
-
return v
|
236
|
-
|
237
|
-
|
238
|
-
def check_none(v: T) -> None:
|
239
|
-
if v is not None:
|
240
|
-
raise ValueError(v)
|
241
|
-
|
242
|
-
|
243
|
-
def check_not_none(v: ta.Optional[T]) -> T:
|
244
|
-
if v is None:
|
245
|
-
raise ValueError
|
246
|
-
return v
|
247
|
-
|
248
|
-
|
249
|
-
def check_not(v: ta.Any) -> None:
|
250
|
-
if v:
|
251
|
-
raise ValueError(v)
|
252
|
-
return v
|
253
|
-
|
254
|
-
|
255
|
-
def check_non_empty_str(v: ta.Optional[str]) -> str:
|
256
|
-
if not v:
|
257
|
-
raise ValueError
|
258
|
-
return v
|
259
|
-
|
260
|
-
|
261
|
-
def check_state(v: bool, msg: str = 'Illegal state') -> None:
|
262
|
-
if not v:
|
263
|
-
raise ValueError(msg)
|
264
|
-
|
265
|
-
|
266
|
-
def check_equal(l: T, r: T) -> T:
|
267
|
-
if l != r:
|
268
|
-
raise ValueError(l, r)
|
269
|
-
return l
|
270
|
-
|
271
|
-
|
272
|
-
def check_not_equal(l: T, r: T) -> T:
|
273
|
-
if l == r:
|
274
|
-
raise ValueError(l, r)
|
275
|
-
return l
|
276
|
-
|
277
|
-
|
278
|
-
def check_is(l: T, r: T) -> T:
|
279
|
-
if l is not r:
|
280
|
-
raise ValueError(l, r)
|
281
|
-
return l
|
282
|
-
|
283
|
-
|
284
|
-
def check_is_not(l: T, r: ta.Any) -> T:
|
285
|
-
if l is r:
|
286
|
-
raise ValueError(l, r)
|
287
|
-
return l
|
288
|
-
|
289
|
-
|
290
|
-
def check_in(v: T, c: ta.Container[T]) -> T:
|
291
|
-
if v not in c:
|
292
|
-
raise ValueError(v, c)
|
293
|
-
return v
|
294
|
-
|
295
|
-
|
296
|
-
def check_not_in(v: T, c: ta.Container[T]) -> T:
|
297
|
-
if v in c:
|
298
|
-
raise ValueError(v, c)
|
299
|
-
return v
|
300
|
-
|
301
|
-
|
302
|
-
def check_single(vs: ta.Iterable[T]) -> T:
|
303
|
-
[v] = vs
|
304
|
-
return v
|
305
|
-
|
306
|
-
|
307
|
-
def check_empty(v: SizedT) -> SizedT:
|
308
|
-
if len(v):
|
309
|
-
raise ValueError(v)
|
310
|
-
return v
|
311
|
-
|
312
|
-
|
313
|
-
def check_non_empty(v: SizedT) -> SizedT:
|
314
|
-
if not len(v):
|
315
|
-
raise ValueError(v)
|
316
|
-
return v
|
317
|
-
|
318
|
-
|
319
|
-
########################################
|
320
|
-
# ../../../omlish/lite/json.py
|
321
|
-
|
322
|
-
|
323
|
-
##
|
324
|
-
|
325
|
-
|
326
|
-
JSON_PRETTY_INDENT = 2
|
327
|
-
|
328
|
-
JSON_PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
|
329
|
-
indent=JSON_PRETTY_INDENT,
|
330
|
-
)
|
331
|
-
|
332
|
-
json_dump_pretty: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_PRETTY_KWARGS) # type: ignore
|
333
|
-
json_dumps_pretty: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_PRETTY_KWARGS)
|
334
|
-
|
335
|
-
|
336
|
-
##
|
337
|
-
|
338
|
-
|
339
|
-
JSON_COMPACT_SEPARATORS = (',', ':')
|
340
|
-
|
341
|
-
JSON_COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
|
342
|
-
indent=None,
|
343
|
-
separators=JSON_COMPACT_SEPARATORS,
|
344
|
-
)
|
345
|
-
|
346
|
-
json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_COMPACT_KWARGS) # type: ignore
|
347
|
-
json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
|
348
|
-
|
349
|
-
|
350
|
-
########################################
|
351
|
-
# ../../../omlish/lite/reflect.py
|
352
|
-
|
353
|
-
|
354
|
-
_GENERIC_ALIAS_TYPES = (
|
355
|
-
ta._GenericAlias, # type: ignore # noqa
|
356
|
-
*([ta._SpecialGenericAlias] if hasattr(ta, '_SpecialGenericAlias') else []), # noqa
|
357
|
-
)
|
358
|
-
|
359
|
-
|
360
|
-
def is_generic_alias(obj, *, origin: ta.Any = None) -> bool:
|
361
|
-
return (
|
362
|
-
isinstance(obj, _GENERIC_ALIAS_TYPES) and
|
363
|
-
(origin is None or ta.get_origin(obj) is origin)
|
364
|
-
)
|
365
|
-
|
366
|
-
|
367
|
-
is_union_alias = functools.partial(is_generic_alias, origin=ta.Union)
|
368
|
-
is_callable_alias = functools.partial(is_generic_alias, origin=ta.Callable)
|
369
|
-
|
370
|
-
|
371
|
-
def is_optional_alias(spec: ta.Any) -> bool:
|
372
|
-
return (
|
373
|
-
isinstance(spec, _GENERIC_ALIAS_TYPES) and # noqa
|
374
|
-
ta.get_origin(spec) is ta.Union and
|
375
|
-
len(ta.get_args(spec)) == 2 and
|
376
|
-
any(a in (None, type(None)) for a in ta.get_args(spec))
|
377
|
-
)
|
378
|
-
|
379
|
-
|
380
|
-
def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
|
381
|
-
[it] = [it for it in ta.get_args(spec) if it not in (None, type(None))]
|
382
|
-
return it
|
383
|
-
|
384
|
-
|
385
|
-
def is_new_type(spec: ta.Any) -> bool:
|
386
|
-
if isinstance(ta.NewType, type):
|
387
|
-
return isinstance(spec, ta.NewType)
|
388
|
-
else:
|
389
|
-
# Before https://github.com/python/cpython/commit/c2f33dfc83ab270412bf243fb21f724037effa1a
|
390
|
-
return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1] # type: ignore # noqa
|
391
|
-
|
392
|
-
|
393
|
-
def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
|
394
|
-
seen = set()
|
395
|
-
todo = list(reversed(cls.__subclasses__()))
|
396
|
-
while todo:
|
397
|
-
cur = todo.pop()
|
398
|
-
if cur in seen:
|
399
|
-
continue
|
400
|
-
seen.add(cur)
|
401
|
-
yield cur
|
402
|
-
todo.extend(reversed(cur.__subclasses__()))
|
403
|
-
|
404
|
-
|
405
|
-
########################################
|
406
|
-
# ../../../omlish/lite/logs.py
|
407
|
-
"""
|
408
|
-
TODO:
|
409
|
-
- translate json keys
|
410
|
-
- debug
|
411
|
-
"""
|
412
|
-
|
413
|
-
|
414
|
-
log = logging.getLogger(__name__)
|
415
|
-
|
416
|
-
|
417
|
-
##
|
418
|
-
|
419
|
-
|
420
|
-
class TidLogFilter(logging.Filter):
|
421
|
-
|
422
|
-
def filter(self, record):
|
423
|
-
record.tid = threading.get_native_id()
|
424
|
-
return True
|
425
|
-
|
426
|
-
|
427
|
-
##
|
428
|
-
|
429
|
-
|
430
|
-
class JsonLogFormatter(logging.Formatter):
|
431
|
-
|
432
|
-
KEYS: ta.Mapping[str, bool] = {
|
433
|
-
'name': False,
|
434
|
-
'msg': False,
|
435
|
-
'args': False,
|
436
|
-
'levelname': False,
|
437
|
-
'levelno': False,
|
438
|
-
'pathname': False,
|
439
|
-
'filename': False,
|
440
|
-
'module': False,
|
441
|
-
'exc_info': True,
|
442
|
-
'exc_text': True,
|
443
|
-
'stack_info': True,
|
444
|
-
'lineno': False,
|
445
|
-
'funcName': False,
|
446
|
-
'created': False,
|
447
|
-
'msecs': False,
|
448
|
-
'relativeCreated': False,
|
449
|
-
'thread': False,
|
450
|
-
'threadName': False,
|
451
|
-
'processName': False,
|
452
|
-
'process': False,
|
453
|
-
}
|
454
|
-
|
455
|
-
def format(self, record: logging.LogRecord) -> str:
|
456
|
-
dct = {
|
457
|
-
k: v
|
458
|
-
for k, o in self.KEYS.items()
|
459
|
-
for v in [getattr(record, k)]
|
460
|
-
if not (o and v is None)
|
461
|
-
}
|
462
|
-
return json_dumps_compact(dct)
|
463
|
-
|
464
|
-
|
465
|
-
##
|
466
|
-
|
467
|
-
|
468
|
-
STANDARD_LOG_FORMAT_PARTS = [
|
469
|
-
('asctime', '%(asctime)-15s'),
|
470
|
-
('process', 'pid=%(process)-6s'),
|
471
|
-
('thread', 'tid=%(thread)x'),
|
472
|
-
('levelname', '%(levelname)s'),
|
473
|
-
('name', '%(name)s'),
|
474
|
-
('separator', '::'),
|
475
|
-
('message', '%(message)s'),
|
476
|
-
]
|
477
|
-
|
478
|
-
|
479
|
-
class StandardLogFormatter(logging.Formatter):
|
480
|
-
|
481
|
-
@staticmethod
|
482
|
-
def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
|
483
|
-
return ' '.join(v for k, v in parts)
|
484
|
-
|
485
|
-
converter = datetime.datetime.fromtimestamp # type: ignore
|
486
|
-
|
487
|
-
def formatTime(self, record, datefmt=None):
|
488
|
-
ct = self.converter(record.created) # type: ignore
|
489
|
-
if datefmt:
|
490
|
-
return ct.strftime(datefmt) # noqa
|
491
|
-
else:
|
492
|
-
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
493
|
-
return '%s.%03d' % (t, record.msecs) # noqa
|
494
|
-
|
495
|
-
|
496
|
-
##
|
497
|
-
|
498
|
-
|
499
|
-
class ProxyLogFilterer(logging.Filterer):
|
500
|
-
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
501
|
-
self._underlying = underlying
|
502
|
-
|
503
|
-
@property
|
504
|
-
def underlying(self) -> logging.Filterer:
|
505
|
-
return self._underlying
|
506
|
-
|
507
|
-
@property
|
508
|
-
def filters(self):
|
509
|
-
return self._underlying.filters
|
510
|
-
|
511
|
-
@filters.setter
|
512
|
-
def filters(self, filters):
|
513
|
-
self._underlying.filters = filters
|
514
|
-
|
515
|
-
def addFilter(self, filter): # noqa
|
516
|
-
self._underlying.addFilter(filter)
|
517
|
-
|
518
|
-
def removeFilter(self, filter): # noqa
|
519
|
-
self._underlying.removeFilter(filter)
|
520
|
-
|
521
|
-
def filter(self, record):
|
522
|
-
return self._underlying.filter(record)
|
523
|
-
|
524
|
-
|
525
|
-
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
526
|
-
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
527
|
-
ProxyLogFilterer.__init__(self, underlying)
|
528
|
-
|
529
|
-
_underlying: logging.Handler
|
530
|
-
|
531
|
-
@property
|
532
|
-
def underlying(self) -> logging.Handler:
|
533
|
-
return self._underlying
|
534
|
-
|
535
|
-
def get_name(self):
|
536
|
-
return self._underlying.get_name()
|
537
|
-
|
538
|
-
def set_name(self, name):
|
539
|
-
self._underlying.set_name(name)
|
540
|
-
|
541
|
-
@property
|
542
|
-
def name(self):
|
543
|
-
return self._underlying.name
|
544
|
-
|
545
|
-
@property
|
546
|
-
def level(self):
|
547
|
-
return self._underlying.level
|
548
|
-
|
549
|
-
@level.setter
|
550
|
-
def level(self, level):
|
551
|
-
self._underlying.level = level
|
552
|
-
|
553
|
-
@property
|
554
|
-
def formatter(self):
|
555
|
-
return self._underlying.formatter
|
556
|
-
|
557
|
-
@formatter.setter
|
558
|
-
def formatter(self, formatter):
|
559
|
-
self._underlying.formatter = formatter
|
560
|
-
|
561
|
-
def createLock(self):
|
562
|
-
self._underlying.createLock()
|
563
|
-
|
564
|
-
def acquire(self):
|
565
|
-
self._underlying.acquire()
|
566
|
-
|
567
|
-
def release(self):
|
568
|
-
self._underlying.release()
|
569
|
-
|
570
|
-
def setLevel(self, level):
|
571
|
-
self._underlying.setLevel(level)
|
572
|
-
|
573
|
-
def format(self, record):
|
574
|
-
return self._underlying.format(record)
|
575
|
-
|
576
|
-
def emit(self, record):
|
577
|
-
self._underlying.emit(record)
|
578
|
-
|
579
|
-
def handle(self, record):
|
580
|
-
return self._underlying.handle(record)
|
581
|
-
|
582
|
-
def setFormatter(self, fmt):
|
583
|
-
self._underlying.setFormatter(fmt)
|
584
|
-
|
585
|
-
def flush(self):
|
586
|
-
self._underlying.flush()
|
587
|
-
|
588
|
-
def close(self):
|
589
|
-
self._underlying.close()
|
590
|
-
|
591
|
-
def handleError(self, record):
|
592
|
-
self._underlying.handleError(record)
|
593
|
-
|
594
|
-
|
595
|
-
##
|
596
|
-
|
597
|
-
|
598
|
-
class StandardLogHandler(ProxyLogHandler):
|
599
|
-
pass
|
600
|
-
|
601
|
-
|
602
|
-
##
|
603
|
-
|
604
|
-
|
605
|
-
@contextlib.contextmanager
|
606
|
-
def _locking_logging_module_lock() -> ta.Iterator[None]:
|
607
|
-
if hasattr(logging, '_acquireLock'):
|
608
|
-
logging._acquireLock() # noqa
|
609
|
-
try:
|
610
|
-
yield
|
611
|
-
finally:
|
612
|
-
logging._releaseLock() # type: ignore # noqa
|
613
|
-
|
614
|
-
elif hasattr(logging, '_lock'):
|
615
|
-
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
616
|
-
with logging._lock: # noqa
|
617
|
-
yield
|
618
|
-
|
619
|
-
else:
|
620
|
-
raise Exception("Can't find lock in logging module")
|
621
|
-
|
622
|
-
|
623
|
-
def configure_standard_logging(
|
624
|
-
level: ta.Union[int, str] = logging.INFO,
|
625
|
-
*,
|
626
|
-
json: bool = False,
|
627
|
-
target: ta.Optional[logging.Logger] = None,
|
628
|
-
force: bool = False,
|
629
|
-
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
630
|
-
) -> ta.Optional[StandardLogHandler]:
|
631
|
-
with _locking_logging_module_lock():
|
632
|
-
if target is None:
|
633
|
-
target = logging.root
|
634
|
-
|
635
|
-
#
|
636
|
-
|
637
|
-
if not force:
|
638
|
-
if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
|
639
|
-
return None
|
640
|
-
|
641
|
-
#
|
642
|
-
|
643
|
-
if handler_factory is not None:
|
644
|
-
handler = handler_factory()
|
645
|
-
else:
|
646
|
-
handler = logging.StreamHandler()
|
647
|
-
|
648
|
-
#
|
649
|
-
|
650
|
-
formatter: logging.Formatter
|
651
|
-
if json:
|
652
|
-
formatter = JsonLogFormatter()
|
653
|
-
else:
|
654
|
-
formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
|
655
|
-
handler.setFormatter(formatter)
|
656
|
-
|
657
|
-
#
|
658
|
-
|
659
|
-
handler.addFilter(TidLogFilter())
|
660
|
-
|
661
|
-
#
|
662
|
-
|
663
|
-
target.addHandler(handler)
|
664
|
-
|
665
|
-
#
|
666
|
-
|
667
|
-
if level is not None:
|
668
|
-
target.setLevel(level)
|
669
|
-
|
670
|
-
#
|
671
|
-
|
672
|
-
return StandardLogHandler(handler)
|
673
|
-
|
674
|
-
|
675
|
-
########################################
|
676
|
-
# ../../../omlish/lite/marshal.py
|
677
|
-
"""
|
678
|
-
TODO:
|
679
|
-
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
680
|
-
- nonstrict toggle
|
681
|
-
"""
|
682
|
-
|
683
|
-
|
684
|
-
##
|
685
|
-
|
686
|
-
|
687
|
-
class ObjMarshaler(abc.ABC):
|
688
|
-
@abc.abstractmethod
|
689
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
690
|
-
raise NotImplementedError
|
691
|
-
|
692
|
-
@abc.abstractmethod
|
693
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
694
|
-
raise NotImplementedError
|
695
|
-
|
696
|
-
|
697
|
-
class NopObjMarshaler(ObjMarshaler):
|
698
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
699
|
-
return o
|
700
|
-
|
701
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
702
|
-
return o
|
703
|
-
|
704
|
-
|
705
|
-
@dc.dataclass()
|
706
|
-
class ProxyObjMarshaler(ObjMarshaler):
|
707
|
-
m: ta.Optional[ObjMarshaler] = None
|
708
|
-
|
709
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
710
|
-
return check_not_none(self.m).marshal(o)
|
711
|
-
|
712
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
713
|
-
return check_not_none(self.m).unmarshal(o)
|
714
|
-
|
715
|
-
|
716
|
-
@dc.dataclass(frozen=True)
|
717
|
-
class CastObjMarshaler(ObjMarshaler):
|
718
|
-
ty: type
|
719
|
-
|
720
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
721
|
-
return o
|
722
|
-
|
723
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
724
|
-
return self.ty(o)
|
725
|
-
|
726
|
-
|
727
|
-
class DynamicObjMarshaler(ObjMarshaler):
|
728
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
729
|
-
return marshal_obj(o)
|
730
|
-
|
731
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
732
|
-
return o
|
733
|
-
|
734
|
-
|
735
|
-
@dc.dataclass(frozen=True)
|
736
|
-
class Base64ObjMarshaler(ObjMarshaler):
|
737
|
-
ty: type
|
738
|
-
|
739
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
740
|
-
return base64.b64encode(o).decode('ascii')
|
741
|
-
|
742
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
743
|
-
return self.ty(base64.b64decode(o))
|
744
|
-
|
745
|
-
|
746
|
-
@dc.dataclass(frozen=True)
|
747
|
-
class EnumObjMarshaler(ObjMarshaler):
|
748
|
-
ty: type
|
749
|
-
|
750
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
751
|
-
return o.name
|
752
|
-
|
753
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
754
|
-
return self.ty.__members__[o] # type: ignore
|
755
|
-
|
756
|
-
|
757
|
-
@dc.dataclass(frozen=True)
|
758
|
-
class OptionalObjMarshaler(ObjMarshaler):
|
759
|
-
item: ObjMarshaler
|
760
|
-
|
761
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
762
|
-
if o is None:
|
763
|
-
return None
|
764
|
-
return self.item.marshal(o)
|
765
|
-
|
766
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
767
|
-
if o is None:
|
768
|
-
return None
|
769
|
-
return self.item.unmarshal(o)
|
770
|
-
|
771
|
-
|
772
|
-
@dc.dataclass(frozen=True)
|
773
|
-
class MappingObjMarshaler(ObjMarshaler):
|
774
|
-
ty: type
|
775
|
-
km: ObjMarshaler
|
776
|
-
vm: ObjMarshaler
|
777
|
-
|
778
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
779
|
-
return {self.km.marshal(k): self.vm.marshal(v) for k, v in o.items()}
|
780
|
-
|
781
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
782
|
-
return self.ty((self.km.unmarshal(k), self.vm.unmarshal(v)) for k, v in o.items())
|
783
|
-
|
784
|
-
|
785
|
-
@dc.dataclass(frozen=True)
|
786
|
-
class IterableObjMarshaler(ObjMarshaler):
|
787
|
-
ty: type
|
788
|
-
item: ObjMarshaler
|
789
|
-
|
790
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
791
|
-
return [self.item.marshal(e) for e in o]
|
792
|
-
|
793
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
794
|
-
return self.ty(self.item.unmarshal(e) for e in o)
|
795
|
-
|
796
|
-
|
797
|
-
@dc.dataclass(frozen=True)
|
798
|
-
class DataclassObjMarshaler(ObjMarshaler):
|
799
|
-
ty: type
|
800
|
-
fs: ta.Mapping[str, ObjMarshaler]
|
801
|
-
nonstrict: bool = False
|
802
|
-
|
803
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
804
|
-
return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
|
805
|
-
|
806
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
807
|
-
return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
|
808
|
-
|
809
|
-
|
810
|
-
@dc.dataclass(frozen=True)
|
811
|
-
class PolymorphicObjMarshaler(ObjMarshaler):
|
812
|
-
class Impl(ta.NamedTuple):
|
813
|
-
ty: type
|
814
|
-
tag: str
|
815
|
-
m: ObjMarshaler
|
816
|
-
|
817
|
-
impls_by_ty: ta.Mapping[type, Impl]
|
818
|
-
impls_by_tag: ta.Mapping[str, Impl]
|
819
|
-
|
820
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
821
|
-
impl = self.impls_by_ty[type(o)]
|
822
|
-
return {impl.tag: impl.m.marshal(o)}
|
823
|
-
|
824
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
825
|
-
[(t, v)] = o.items()
|
826
|
-
impl = self.impls_by_tag[t]
|
827
|
-
return impl.m.unmarshal(v)
|
828
|
-
|
829
|
-
|
830
|
-
@dc.dataclass(frozen=True)
|
831
|
-
class DatetimeObjMarshaler(ObjMarshaler):
|
832
|
-
ty: type
|
833
|
-
|
834
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
835
|
-
return o.isoformat()
|
836
|
-
|
837
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
838
|
-
return self.ty.fromisoformat(o) # type: ignore
|
839
|
-
|
840
|
-
|
841
|
-
class DecimalObjMarshaler(ObjMarshaler):
|
842
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
843
|
-
return str(check_isinstance(o, decimal.Decimal))
|
844
|
-
|
845
|
-
def unmarshal(self, v: ta.Any) -> ta.Any:
|
846
|
-
return decimal.Decimal(check_isinstance(v, str))
|
847
|
-
|
848
|
-
|
849
|
-
class FractionObjMarshaler(ObjMarshaler):
|
850
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
851
|
-
fr = check_isinstance(o, fractions.Fraction)
|
852
|
-
return [fr.numerator, fr.denominator]
|
853
|
-
|
854
|
-
def unmarshal(self, v: ta.Any) -> ta.Any:
|
855
|
-
num, denom = check_isinstance(v, list)
|
856
|
-
return fractions.Fraction(num, denom)
|
857
|
-
|
858
|
-
|
859
|
-
class UuidObjMarshaler(ObjMarshaler):
|
860
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
861
|
-
return str(o)
|
862
|
-
|
863
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
864
|
-
return uuid.UUID(o)
|
865
|
-
|
866
|
-
|
867
|
-
##
|
868
|
-
|
869
|
-
|
870
|
-
_DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
|
871
|
-
**{t: NopObjMarshaler() for t in (type(None),)},
|
872
|
-
**{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
|
873
|
-
**{t: Base64ObjMarshaler(t) for t in (bytes, bytearray)},
|
874
|
-
**{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
|
875
|
-
**{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
|
876
|
-
|
877
|
-
ta.Any: DynamicObjMarshaler(),
|
878
|
-
|
879
|
-
**{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
|
880
|
-
decimal.Decimal: DecimalObjMarshaler(),
|
881
|
-
fractions.Fraction: FractionObjMarshaler(),
|
882
|
-
uuid.UUID: UuidObjMarshaler(),
|
883
|
-
}
|
884
|
-
|
885
|
-
_OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
|
886
|
-
**{t: t for t in (dict,)},
|
887
|
-
**{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
|
888
|
-
}
|
889
|
-
|
890
|
-
_OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
|
891
|
-
**{t: t for t in (list, tuple, set, frozenset)},
|
892
|
-
collections.abc.Set: frozenset,
|
893
|
-
collections.abc.MutableSet: set,
|
894
|
-
collections.abc.Sequence: tuple,
|
895
|
-
collections.abc.MutableSequence: list,
|
896
|
-
}
|
897
|
-
|
898
|
-
|
899
|
-
def _make_obj_marshaler(
|
900
|
-
ty: ta.Any,
|
901
|
-
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
902
|
-
*,
|
903
|
-
nonstrict_dataclasses: bool = False,
|
904
|
-
) -> ObjMarshaler:
|
905
|
-
if isinstance(ty, type):
|
906
|
-
if abc.ABC in ty.__bases__:
|
907
|
-
impls = [ # type: ignore
|
908
|
-
PolymorphicObjMarshaler.Impl(
|
909
|
-
ity,
|
910
|
-
ity.__qualname__,
|
911
|
-
rec(ity),
|
912
|
-
)
|
913
|
-
for ity in deep_subclasses(ty)
|
914
|
-
if abc.ABC not in ity.__bases__
|
915
|
-
]
|
916
|
-
return PolymorphicObjMarshaler(
|
917
|
-
{i.ty: i for i in impls},
|
918
|
-
{i.tag: i for i in impls},
|
919
|
-
)
|
920
|
-
|
921
|
-
if issubclass(ty, enum.Enum):
|
922
|
-
return EnumObjMarshaler(ty)
|
923
|
-
|
924
|
-
if dc.is_dataclass(ty):
|
925
|
-
return DataclassObjMarshaler(
|
926
|
-
ty,
|
927
|
-
{f.name: rec(f.type) for f in dc.fields(ty)},
|
928
|
-
nonstrict=nonstrict_dataclasses,
|
929
|
-
)
|
930
|
-
|
931
|
-
if is_generic_alias(ty):
|
932
|
-
try:
|
933
|
-
mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
|
934
|
-
except KeyError:
|
935
|
-
pass
|
936
|
-
else:
|
937
|
-
k, v = ta.get_args(ty)
|
938
|
-
return MappingObjMarshaler(mt, rec(k), rec(v))
|
939
|
-
|
940
|
-
try:
|
941
|
-
st = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES[ta.get_origin(ty)]
|
942
|
-
except KeyError:
|
943
|
-
pass
|
944
|
-
else:
|
945
|
-
[e] = ta.get_args(ty)
|
946
|
-
return IterableObjMarshaler(st, rec(e))
|
947
|
-
|
948
|
-
if is_union_alias(ty):
|
949
|
-
return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
|
950
|
-
|
951
|
-
raise TypeError(ty)
|
952
|
-
|
953
|
-
|
954
|
-
##
|
955
|
-
|
956
|
-
|
957
|
-
_OBJ_MARSHALERS_LOCK = threading.RLock()
|
958
|
-
|
959
|
-
_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
|
960
|
-
|
961
|
-
_OBJ_MARSHALER_PROXIES: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
|
962
|
-
|
963
|
-
|
964
|
-
def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
|
965
|
-
with _OBJ_MARSHALERS_LOCK:
|
966
|
-
if ty in _OBJ_MARSHALERS:
|
967
|
-
raise KeyError(ty)
|
968
|
-
_OBJ_MARSHALERS[ty] = m
|
969
|
-
|
970
|
-
|
971
|
-
def get_obj_marshaler(
|
972
|
-
ty: ta.Any,
|
973
|
-
*,
|
974
|
-
no_cache: bool = False,
|
975
|
-
**kwargs: ta.Any,
|
976
|
-
) -> ObjMarshaler:
|
977
|
-
with _OBJ_MARSHALERS_LOCK:
|
978
|
-
if not no_cache:
|
979
|
-
try:
|
980
|
-
return _OBJ_MARSHALERS[ty]
|
981
|
-
except KeyError:
|
982
|
-
pass
|
983
|
-
|
984
|
-
try:
|
985
|
-
return _OBJ_MARSHALER_PROXIES[ty]
|
986
|
-
except KeyError:
|
987
|
-
pass
|
988
|
-
|
989
|
-
rec = functools.partial(
|
990
|
-
get_obj_marshaler,
|
991
|
-
no_cache=no_cache,
|
992
|
-
**kwargs,
|
993
|
-
)
|
994
|
-
|
995
|
-
p = ProxyObjMarshaler()
|
996
|
-
_OBJ_MARSHALER_PROXIES[ty] = p
|
997
|
-
try:
|
998
|
-
m = _make_obj_marshaler(ty, rec, **kwargs)
|
999
|
-
finally:
|
1000
|
-
del _OBJ_MARSHALER_PROXIES[ty]
|
1001
|
-
p.m = m
|
1002
|
-
|
1003
|
-
if not no_cache:
|
1004
|
-
_OBJ_MARSHALERS[ty] = m
|
1005
|
-
return m
|
1006
|
-
|
1007
|
-
|
1008
|
-
##
|
1009
|
-
|
1010
|
-
|
1011
|
-
def marshal_obj(o: ta.Any, ty: ta.Any = None) -> ta.Any:
|
1012
|
-
return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
|
1013
|
-
|
1014
|
-
|
1015
|
-
def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
|
1016
|
-
return get_obj_marshaler(ty).unmarshal(o)
|
1017
|
-
|
1018
|
-
|
1019
|
-
########################################
|
1020
|
-
# ../../../omlish/lite/runtime.py
|
1021
|
-
|
1022
|
-
|
1023
|
-
@cached_nullary
|
1024
|
-
def is_debugger_attached() -> bool:
|
1025
|
-
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
1026
|
-
|
1027
|
-
|
1028
|
-
REQUIRED_PYTHON_VERSION = (3, 8)
|
1029
|
-
|
1030
|
-
|
1031
|
-
def check_runtime_version() -> None:
|
1032
|
-
if sys.version_info < REQUIRED_PYTHON_VERSION:
|
1033
|
-
raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
|
1034
|
-
|
1035
|
-
|
1036
|
-
########################################
|
1037
|
-
# ../../../omlish/lite/subprocesses.py
|
1038
|
-
|
1039
|
-
|
1040
|
-
##
|
1041
|
-
|
1042
|
-
|
1043
|
-
_SUBPROCESS_SHELL_WRAP_EXECS = False
|
1044
|
-
|
1045
|
-
|
1046
|
-
def subprocess_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
|
1047
|
-
return ('sh', '-c', ' '.join(map(shlex.quote, args)))
|
1048
|
-
|
1049
|
-
|
1050
|
-
def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
|
1051
|
-
if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
|
1052
|
-
return subprocess_shell_wrap_exec(*args)
|
1053
|
-
else:
|
1054
|
-
return args
|
1055
|
-
|
1056
|
-
|
1057
|
-
def _prepare_subprocess_invocation(
|
1058
|
-
*args: str,
|
1059
|
-
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
1060
|
-
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
1061
|
-
quiet: bool = False,
|
1062
|
-
shell: bool = False,
|
1063
|
-
**kwargs: ta.Any,
|
1064
|
-
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
1065
|
-
log.debug(args)
|
1066
|
-
if extra_env:
|
1067
|
-
log.debug(extra_env)
|
1068
|
-
|
1069
|
-
if extra_env:
|
1070
|
-
env = {**(env if env is not None else os.environ), **extra_env}
|
1071
|
-
|
1072
|
-
if quiet and 'stderr' not in kwargs:
|
1073
|
-
if not log.isEnabledFor(logging.DEBUG):
|
1074
|
-
kwargs['stderr'] = subprocess.DEVNULL
|
1075
|
-
|
1076
|
-
if not shell:
|
1077
|
-
args = subprocess_maybe_shell_wrap_exec(*args)
|
1078
|
-
|
1079
|
-
return args, dict(
|
1080
|
-
env=env,
|
1081
|
-
shell=shell,
|
1082
|
-
**kwargs,
|
1083
|
-
)
|
1084
|
-
|
1085
|
-
|
1086
|
-
def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
|
1087
|
-
args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
|
1088
|
-
return subprocess.check_call(args, **kwargs) # type: ignore
|
1089
|
-
|
1090
|
-
|
1091
|
-
def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
|
1092
|
-
args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
|
1093
|
-
return subprocess.check_output(args, **kwargs)
|
1094
|
-
|
1095
|
-
|
1096
|
-
def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
|
1097
|
-
return subprocess_check_output(*args, **kwargs).decode().strip()
|
1098
|
-
|
1099
|
-
|
1100
|
-
##
|
1101
|
-
|
1102
|
-
|
1103
|
-
DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
|
1104
|
-
FileNotFoundError,
|
1105
|
-
subprocess.CalledProcessError,
|
1106
|
-
)
|
1107
|
-
|
1108
|
-
|
1109
|
-
def subprocess_try_call(
|
1110
|
-
*args: str,
|
1111
|
-
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
1112
|
-
**kwargs: ta.Any,
|
1113
|
-
) -> bool:
|
1114
|
-
try:
|
1115
|
-
subprocess_check_call(*args, **kwargs)
|
1116
|
-
except try_exceptions as e: # noqa
|
1117
|
-
if log.isEnabledFor(logging.DEBUG):
|
1118
|
-
log.exception('command failed')
|
1119
|
-
return False
|
1120
|
-
else:
|
1121
|
-
return True
|
1122
|
-
|
1123
|
-
|
1124
|
-
def subprocess_try_output(
|
1125
|
-
*args: str,
|
1126
|
-
try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
|
1127
|
-
**kwargs: ta.Any,
|
1128
|
-
) -> ta.Optional[bytes]:
|
1129
|
-
try:
|
1130
|
-
return subprocess_check_output(*args, **kwargs)
|
1131
|
-
except try_exceptions as e: # noqa
|
1132
|
-
if log.isEnabledFor(logging.DEBUG):
|
1133
|
-
log.exception('command failed')
|
1134
|
-
return None
|
1135
|
-
|
1136
|
-
|
1137
|
-
def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
1138
|
-
out = subprocess_try_output(*args, **kwargs)
|
1139
|
-
return out.decode().strip() if out is not None else None
|
1140
|
-
|
1141
|
-
|
1142
|
-
##
|
1143
|
-
|
1144
|
-
|
1145
|
-
def subprocess_close(
|
1146
|
-
proc: subprocess.Popen,
|
1147
|
-
timeout: ta.Optional[float] = None,
|
1148
|
-
) -> None:
|
1149
|
-
# TODO: terminate, sleep, kill
|
1150
|
-
if proc.stdout:
|
1151
|
-
proc.stdout.close()
|
1152
|
-
if proc.stderr:
|
1153
|
-
proc.stderr.close()
|
1154
|
-
if proc.stdin:
|
1155
|
-
proc.stdin.close()
|
1156
|
-
|
1157
|
-
proc.wait(timeout)
|
1158
|
-
|
1159
|
-
|
1160
|
-
########################################
|
1161
|
-
# runcommands.py
|
1162
|
-
|
1163
|
-
|
1164
|
-
@dc.dataclass(frozen=True)
|
1165
|
-
class CommandRequest:
|
1166
|
-
cmd: ta.Sequence[str]
|
1167
|
-
in_: ta.Optional[bytes] = None
|
1168
|
-
|
1169
|
-
|
1170
|
-
@dc.dataclass(frozen=True)
|
1171
|
-
class CommandResponse:
|
1172
|
-
req: CommandRequest
|
1173
|
-
rc: int
|
1174
|
-
out: bytes
|
1175
|
-
err: bytes
|
1176
|
-
|
1177
|
-
|
1178
|
-
def _run_commands_loop(input: ta.BinaryIO, output: ta.BinaryIO = sys.stdout.buffer) -> None: # noqa
|
1179
|
-
while (l := input.readline().decode('utf-8').strip()):
|
1180
|
-
req: CommandRequest = unmarshal_obj(json.loads(l), CommandRequest)
|
1181
|
-
proc = subprocess.Popen( # type: ignore
|
1182
|
-
subprocess_maybe_shell_wrap_exec(*req.cmd),
|
1183
|
-
**(dict(stdin=io.BytesIO(req.in_)) if req.in_ is not None else {}),
|
1184
|
-
stdout=subprocess.PIPE,
|
1185
|
-
stderr=subprocess.PIPE,
|
1186
|
-
)
|
1187
|
-
out, err = proc.communicate()
|
1188
|
-
resp = CommandResponse(
|
1189
|
-
req=req,
|
1190
|
-
rc=proc.returncode,
|
1191
|
-
out=out, # noqa
|
1192
|
-
err=err, # noqa
|
1193
|
-
)
|
1194
|
-
output.write(json_dumps_compact(marshal_obj(resp)).encode('utf-8'))
|
1195
|
-
output.write(b'\n')
|
1196
|
-
output.flush()
|
1197
|
-
|
1198
|
-
|
1199
|
-
def run_commands_main() -> None:
|
1200
|
-
bs = post_boostrap()
|
1201
|
-
_run_commands_loop(bs.input)
|