ominfra 0.0.0.dev134__py3-none-any.whl → 0.0.0.dev136__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,1646 @@
1
+ #!/usr/bin/env python3
2
+ # noinspection DuplicatedCode
3
+ # @omlish-lite
4
+ # @omlish-script
5
+ # @omlish-amalg-output main.py
6
+ # ruff: noqa: N802 UP006 UP007 UP036
7
+ import abc
8
+ import base64
9
+ import collections.abc
10
+ import contextlib
11
+ import dataclasses as dc
12
+ import datetime
13
+ import decimal
14
+ import enum
15
+ import fractions
16
+ import functools
17
+ import inspect
18
+ import json
19
+ import logging
20
+ import os
21
+ import platform
22
+ import pwd
23
+ import shlex
24
+ import site
25
+ import struct
26
+ import subprocess
27
+ import sys
28
+ import textwrap
29
+ import threading
30
+ import time
31
+ import types
32
+ import typing as ta
33
+ import uuid
34
+ import weakref # noqa
35
+ import zlib
36
+
37
+
38
+ ########################################
39
+
40
+
41
+ if sys.version_info < (3, 8):
42
+ raise OSError(f'Requires python (3, 8), got {sys.version_info} from {sys.executable}') # noqa
43
+
44
+
45
+ ########################################
46
+
47
+
48
+ # commands/base.py
49
+ CommandInputT = ta.TypeVar('CommandInputT', bound='Command.Input')
50
+ CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
51
+
52
+ # ../../../omlish/lite/cached.py
53
+ T = ta.TypeVar('T')
54
+
55
+ # ../../../omlish/lite/check.py
56
+ SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
57
+
58
+
59
+ ########################################
60
+ # ../commands/base.py
61
+
62
+
63
+ ##
64
+
65
+
66
+ class Command(abc.ABC, ta.Generic[CommandInputT, CommandOutputT]):
67
+ @dc.dataclass(frozen=True)
68
+ class Input(abc.ABC): # noqa
69
+ pass
70
+
71
+ @dc.dataclass(frozen=True)
72
+ class Output(abc.ABC): # noqa
73
+ pass
74
+
75
+ @abc.abstractmethod
76
+ def _execute(self, inp: CommandInputT) -> CommandOutputT:
77
+ raise NotImplementedError
78
+
79
+
80
+ ########################################
81
+ # ../../../pyremote.py
82
+ """
83
+ Basically this: https://mitogen.networkgenomics.com/howitworks.html
84
+ """
85
+
86
+
87
+ ##
88
+
89
+
90
+ _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
91
+ _PYREMOTE_BOOTSTRAP_SRC_FD = 101
92
+
93
+ _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
94
+ _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
95
+
96
+ _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
97
+ _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
98
+ _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
99
+ _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
100
+
101
+ _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
102
+
103
+ _PYREMOTE_BOOTSTRAP_IMPORTS = [
104
+ 'base64',
105
+ 'os',
106
+ 'struct',
107
+ 'sys',
108
+ 'zlib',
109
+ ]
110
+
111
+
112
+ def _pyremote_bootstrap_main(context_name: str) -> None:
113
+ # Get pid
114
+ pid = os.getpid()
115
+
116
+ # Two copies of main src to be sent to parent
117
+ r0, w0 = os.pipe()
118
+ r1, w1 = os.pipe()
119
+
120
+ if (cp := os.fork()):
121
+ # Parent process
122
+
123
+ # Dup original stdin to comm_fd for use as comm channel
124
+ os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
125
+
126
+ # Overwrite stdin (fed to python repl) with first copy of src
127
+ os.dup2(r0, 0)
128
+
129
+ # Dup second copy of src to src_fd to recover after launch
130
+ os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
131
+
132
+ # Close remaining fd's
133
+ for f in [r0, w0, r1, w1]:
134
+ os.close(f)
135
+
136
+ # Save child pid to close after relaunch
137
+ os.environ[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
138
+
139
+ # Save original argv0
140
+ os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = sys.executable
141
+
142
+ # Start repl reading stdin from r0
143
+ os.execl(sys.executable, sys.executable + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
144
+
145
+ else:
146
+ # Child process
147
+
148
+ # Write first ack
149
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
150
+
151
+ # Write pid
152
+ os.write(1, struct.pack('<Q', pid))
153
+
154
+ # Read main src from stdin
155
+ main_z_len = struct.unpack('<I', os.read(0, 4))[0]
156
+ main_src = zlib.decompress(os.fdopen(0, 'rb').read(main_z_len))
157
+
158
+ # Write both copies of main src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely fill
159
+ # and block and need to be drained by pyremote_bootstrap_finalize running in parent.
160
+ for w in [w0, w1]:
161
+ fp = os.fdopen(w, 'wb', 0)
162
+ fp.write(main_src)
163
+ fp.close()
164
+
165
+ # Write second ack
166
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
167
+
168
+ # Exit child
169
+ sys.exit(0)
170
+
171
+
172
+ ##
173
+
174
+
175
+ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
176
+ if any(c in context_name for c in '\'"'):
177
+ raise NameError(context_name)
178
+
179
+ bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
180
+
181
+ for gl in [
182
+ '_PYREMOTE_BOOTSTRAP_INPUT_FD',
183
+ '_PYREMOTE_BOOTSTRAP_SRC_FD',
184
+
185
+ '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
186
+ '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
187
+
188
+ '_PYREMOTE_BOOTSTRAP_ACK0',
189
+ '_PYREMOTE_BOOTSTRAP_ACK1',
190
+
191
+ '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
192
+ ]:
193
+ bs_src = bs_src.replace(gl, repr(globals()[gl]))
194
+
195
+ bs_src = '\n'.join(
196
+ cl
197
+ for l in bs_src.splitlines()
198
+ if (cl := (l.split('#')[0]).rstrip())
199
+ if cl.strip()
200
+ )
201
+
202
+ bs_z = zlib.compress(bs_src.encode('utf-8'))
203
+ bs_z64 = base64.encodebytes(bs_z).replace(b'\n', b'')
204
+
205
+ def dq_repr(o: ta.Any) -> str:
206
+ return '"' + repr(o)[1:-1] + '"'
207
+
208
+ stmts = [
209
+ f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
210
+ f'exec(zlib.decompress(base64.decodebytes({bs_z64!r})))',
211
+ f'_pyremote_bootstrap_main({context_name!r})',
212
+ ]
213
+
214
+ cmd = '; '.join(stmts)
215
+ return cmd
216
+
217
+
218
+ ##
219
+
220
+
221
+ @dc.dataclass(frozen=True)
222
+ class PyremoteEnvInfo:
223
+ sys_base_prefix: str
224
+ sys_byteorder: str
225
+ sys_defaultencoding: str
226
+ sys_exec_prefix: str
227
+ sys_executable: str
228
+ sys_implementation_name: str
229
+ sys_path: ta.List[str]
230
+ sys_platform: str
231
+ sys_prefix: str
232
+ sys_version: str
233
+ sys_version_info: ta.List[ta.Union[int, str]]
234
+
235
+ platform_architecture: ta.List[str]
236
+ platform_machine: str
237
+ platform_platform: str
238
+ platform_processor: str
239
+ platform_system: str
240
+ platform_release: str
241
+ platform_version: str
242
+
243
+ site_userbase: str
244
+
245
+ os_cwd: str
246
+ os_gid: int
247
+ os_loadavg: ta.List[float]
248
+ os_login: ta.Optional[str]
249
+ os_pgrp: int
250
+ os_pid: int
251
+ os_ppid: int
252
+ os_uid: int
253
+
254
+ pw_name: str
255
+ pw_uid: int
256
+ pw_gid: int
257
+ pw_gecos: str
258
+ pw_dir: str
259
+ pw_shell: str
260
+
261
+ env_path: ta.Optional[str]
262
+
263
+
264
+ def _get_pyremote_env_info() -> PyremoteEnvInfo:
265
+ os_uid = os.getuid()
266
+
267
+ pw = pwd.getpwuid(os_uid)
268
+
269
+ os_login: ta.Optional[str]
270
+ try:
271
+ os_login = os.getlogin()
272
+ except OSError:
273
+ os_login = None
274
+
275
+ return PyremoteEnvInfo(
276
+ sys_base_prefix=sys.base_prefix,
277
+ sys_byteorder=sys.byteorder,
278
+ sys_defaultencoding=sys.getdefaultencoding(),
279
+ sys_exec_prefix=sys.exec_prefix,
280
+ sys_executable=sys.executable,
281
+ sys_implementation_name=sys.implementation.name,
282
+ sys_path=sys.path,
283
+ sys_platform=sys.platform,
284
+ sys_prefix=sys.prefix,
285
+ sys_version=sys.version,
286
+ sys_version_info=list(sys.version_info),
287
+
288
+ platform_architecture=list(platform.architecture()),
289
+ platform_machine=platform.machine(),
290
+ platform_platform=platform.platform(),
291
+ platform_processor=platform.processor(),
292
+ platform_system=platform.system(),
293
+ platform_release=platform.release(),
294
+ platform_version=platform.version(),
295
+
296
+ site_userbase=site.getuserbase(),
297
+
298
+ os_cwd=os.getcwd(),
299
+ os_gid=os.getgid(),
300
+ os_loadavg=list(os.getloadavg()),
301
+ os_login=os_login,
302
+ os_pgrp=os.getpgrp(),
303
+ os_pid=os.getpid(),
304
+ os_ppid=os.getppid(),
305
+ os_uid=os_uid,
306
+
307
+ pw_name=pw.pw_name,
308
+ pw_uid=pw.pw_uid,
309
+ pw_gid=pw.pw_gid,
310
+ pw_gecos=pw.pw_gecos,
311
+ pw_dir=pw.pw_dir,
312
+ pw_shell=pw.pw_shell,
313
+
314
+ env_path=os.environ.get('PATH'),
315
+ )
316
+
317
+
318
+ ##
319
+
320
+
321
+ class PyremoteBootstrapDriver:
322
+ def __init__(self, main_src: str) -> None:
323
+ super().__init__()
324
+
325
+ self._main_src = main_src
326
+ self._main_z = zlib.compress(main_src.encode('utf-8'))
327
+
328
+ #
329
+
330
+ @dc.dataclass(frozen=True)
331
+ class Read:
332
+ sz: int
333
+
334
+ @dc.dataclass(frozen=True)
335
+ class Write:
336
+ d: bytes
337
+
338
+ class ProtocolError(Exception):
339
+ pass
340
+
341
+ @dc.dataclass(frozen=True)
342
+ class Result:
343
+ pid: int
344
+ env_info: PyremoteEnvInfo
345
+
346
+ def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
347
+ # Read first ack
348
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
349
+
350
+ # Read pid
351
+ d = yield from self._read(8)
352
+ pid = struct.unpack('<Q', d)[0]
353
+
354
+ # Write main src
355
+ yield from self._write(struct.pack('<I', len(self._main_z)))
356
+ yield from self._write(self._main_z)
357
+
358
+ # Read second and third ack
359
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
360
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
361
+
362
+ # Read env info
363
+ d = yield from self._read(4)
364
+ env_info_json_len = struct.unpack('<I', d)[0]
365
+ d = yield from self._read(env_info_json_len)
366
+ env_info_json = d.decode('utf-8')
367
+ env_info = PyremoteEnvInfo(**json.loads(env_info_json))
368
+
369
+ # Read fourth ack
370
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
371
+
372
+ # Return
373
+ return self.Result(
374
+ pid=pid,
375
+ env_info=env_info,
376
+ )
377
+
378
+ def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
379
+ d = yield self.Read(sz)
380
+ if not isinstance(d, bytes):
381
+ raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
382
+ if len(d) != sz:
383
+ raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
384
+ return d
385
+
386
+ def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
387
+ d = yield from self._read(len(e))
388
+ if d != e:
389
+ raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
390
+
391
+ def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
392
+ i = yield self.Write(d)
393
+ if i is not None:
394
+ raise self.ProtocolError('Unexpected input after write')
395
+
396
+ #
397
+
398
+ def run(self, stdin: ta.IO, stdout: ta.IO) -> Result:
399
+ gen = self.gen()
400
+
401
+ gi: ta.Optional[bytes] = None
402
+ while True:
403
+ try:
404
+ if gi is not None:
405
+ go = gen.send(gi)
406
+ else:
407
+ go = next(gen)
408
+ except StopIteration as e:
409
+ return e.value
410
+
411
+ if isinstance(go, self.Read):
412
+ gi = stdout.read(go.sz)
413
+ elif isinstance(go, self.Write):
414
+ gi = None
415
+ stdin.write(go.d)
416
+ stdin.flush()
417
+ else:
418
+ raise TypeError(go)
419
+
420
+
421
+ ##
422
+
423
+
424
+ @dc.dataclass(frozen=True)
425
+ class PyremotePayloadRuntime:
426
+ input: ta.BinaryIO
427
+ output: ta.BinaryIO
428
+ main_src: str
429
+ env_info: PyremoteEnvInfo
430
+
431
+
432
+ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
433
+ # Restore original argv0
434
+ sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
435
+
436
+ # Read second copy of main src
437
+ r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
438
+ main_src = r1.read().decode('utf-8')
439
+ r1.close()
440
+
441
+ # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in a
442
+ # pipe at once.
443
+ os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
444
+
445
+ # Write third ack
446
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
447
+
448
+ # Write env info
449
+ env_info = _get_pyremote_env_info()
450
+ env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
451
+ os.write(1, struct.pack('<I', len(env_info_json)))
452
+ os.write(1, env_info_json.encode('utf-8'))
453
+
454
+ # Setup IO
455
+ input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
456
+ output = os.fdopen(os.dup(1), 'wb', 0) # noqa
457
+ os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
458
+ os.close(nfd)
459
+
460
+ # Write fourth ack
461
+ output.write(_PYREMOTE_BOOTSTRAP_ACK3)
462
+
463
+ # Return
464
+ return PyremotePayloadRuntime(
465
+ input=input,
466
+ output=output,
467
+ main_src=main_src,
468
+ env_info=env_info,
469
+ )
470
+
471
+
472
+ ########################################
473
+ # ../../../../omlish/lite/cached.py
474
+
475
+
476
+ class _cached_nullary: # noqa
477
+ def __init__(self, fn):
478
+ super().__init__()
479
+ self._fn = fn
480
+ self._value = self._missing = object()
481
+ functools.update_wrapper(self, fn)
482
+
483
+ def __call__(self, *args, **kwargs): # noqa
484
+ if self._value is self._missing:
485
+ self._value = self._fn()
486
+ return self._value
487
+
488
+ def __get__(self, instance, owner): # noqa
489
+ bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
490
+ return bound
491
+
492
+
493
+ def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
494
+ return _cached_nullary(fn)
495
+
496
+
497
+ ########################################
498
+ # ../../../../omlish/lite/check.py
499
+
500
+
501
+ def check_isinstance(v: ta.Any, spec: ta.Union[ta.Type[T], tuple]) -> T:
502
+ if not isinstance(v, spec):
503
+ raise TypeError(v)
504
+ return v
505
+
506
+
507
+ def check_not_isinstance(v: T, spec: ta.Union[type, tuple]) -> T:
508
+ if isinstance(v, spec):
509
+ raise TypeError(v)
510
+ return v
511
+
512
+
513
+ def check_none(v: T) -> None:
514
+ if v is not None:
515
+ raise ValueError(v)
516
+
517
+
518
+ def check_not_none(v: ta.Optional[T]) -> T:
519
+ if v is None:
520
+ raise ValueError
521
+ return v
522
+
523
+
524
+ def check_not(v: ta.Any) -> None:
525
+ if v:
526
+ raise ValueError(v)
527
+ return v
528
+
529
+
530
+ def check_non_empty_str(v: ta.Optional[str]) -> str:
531
+ if not v:
532
+ raise ValueError
533
+ return v
534
+
535
+
536
+ def check_state(v: bool, msg: str = 'Illegal state') -> None:
537
+ if not v:
538
+ raise ValueError(msg)
539
+
540
+
541
+ def check_equal(l: T, r: T) -> T:
542
+ if l != r:
543
+ raise ValueError(l, r)
544
+ return l
545
+
546
+
547
+ def check_not_equal(l: T, r: T) -> T:
548
+ if l == r:
549
+ raise ValueError(l, r)
550
+ return l
551
+
552
+
553
+ def check_is(l: T, r: T) -> T:
554
+ if l is not r:
555
+ raise ValueError(l, r)
556
+ return l
557
+
558
+
559
+ def check_is_not(l: T, r: ta.Any) -> T:
560
+ if l is r:
561
+ raise ValueError(l, r)
562
+ return l
563
+
564
+
565
+ def check_in(v: T, c: ta.Container[T]) -> T:
566
+ if v not in c:
567
+ raise ValueError(v, c)
568
+ return v
569
+
570
+
571
+ def check_not_in(v: T, c: ta.Container[T]) -> T:
572
+ if v in c:
573
+ raise ValueError(v, c)
574
+ return v
575
+
576
+
577
+ def check_single(vs: ta.Iterable[T]) -> T:
578
+ [v] = vs
579
+ return v
580
+
581
+
582
+ def check_empty(v: SizedT) -> SizedT:
583
+ if len(v):
584
+ raise ValueError(v)
585
+ return v
586
+
587
+
588
+ def check_non_empty(v: SizedT) -> SizedT:
589
+ if not len(v):
590
+ raise ValueError(v)
591
+ return v
592
+
593
+
594
+ ########################################
595
+ # ../../../../omlish/lite/json.py
596
+
597
+
598
+ ##
599
+
600
+
601
+ JSON_PRETTY_INDENT = 2
602
+
603
+ JSON_PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
604
+ indent=JSON_PRETTY_INDENT,
605
+ )
606
+
607
+ json_dump_pretty: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_PRETTY_KWARGS) # type: ignore
608
+ json_dumps_pretty: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_PRETTY_KWARGS)
609
+
610
+
611
+ ##
612
+
613
+
614
+ JSON_COMPACT_SEPARATORS = (',', ':')
615
+
616
+ JSON_COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
617
+ indent=None,
618
+ separators=JSON_COMPACT_SEPARATORS,
619
+ )
620
+
621
+ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_COMPACT_KWARGS) # type: ignore
622
+ json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
623
+
624
+
625
+ ########################################
626
+ # ../../../../omlish/lite/reflect.py
627
+
628
+
629
+ _GENERIC_ALIAS_TYPES = (
630
+ ta._GenericAlias, # type: ignore # noqa
631
+ *([ta._SpecialGenericAlias] if hasattr(ta, '_SpecialGenericAlias') else []), # noqa
632
+ )
633
+
634
+
635
+ def is_generic_alias(obj, *, origin: ta.Any = None) -> bool:
636
+ return (
637
+ isinstance(obj, _GENERIC_ALIAS_TYPES) and
638
+ (origin is None or ta.get_origin(obj) is origin)
639
+ )
640
+
641
+
642
+ is_union_alias = functools.partial(is_generic_alias, origin=ta.Union)
643
+ is_callable_alias = functools.partial(is_generic_alias, origin=ta.Callable)
644
+
645
+
646
+ def is_optional_alias(spec: ta.Any) -> bool:
647
+ return (
648
+ isinstance(spec, _GENERIC_ALIAS_TYPES) and # noqa
649
+ ta.get_origin(spec) is ta.Union and
650
+ len(ta.get_args(spec)) == 2 and
651
+ any(a in (None, type(None)) for a in ta.get_args(spec))
652
+ )
653
+
654
+
655
+ def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
656
+ [it] = [it for it in ta.get_args(spec) if it not in (None, type(None))]
657
+ return it
658
+
659
+
660
+ def is_new_type(spec: ta.Any) -> bool:
661
+ if isinstance(ta.NewType, type):
662
+ return isinstance(spec, ta.NewType)
663
+ else:
664
+ # Before https://github.com/python/cpython/commit/c2f33dfc83ab270412bf243fb21f724037effa1a
665
+ return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1] # type: ignore # noqa
666
+
667
+
668
+ def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
669
+ seen = set()
670
+ todo = list(reversed(cls.__subclasses__()))
671
+ while todo:
672
+ cur = todo.pop()
673
+ if cur in seen:
674
+ continue
675
+ seen.add(cur)
676
+ yield cur
677
+ todo.extend(reversed(cur.__subclasses__()))
678
+
679
+
680
+ ########################################
681
+ # ../../../../omlish/lite/logs.py
682
+ """
683
+ TODO:
684
+ - translate json keys
685
+ - debug
686
+ """
687
+
688
+
689
+ log = logging.getLogger(__name__)
690
+
691
+
692
+ ##
693
+
694
+
695
+ class TidLogFilter(logging.Filter):
696
+
697
+ def filter(self, record):
698
+ record.tid = threading.get_native_id()
699
+ return True
700
+
701
+
702
+ ##
703
+
704
+
705
+ class JsonLogFormatter(logging.Formatter):
706
+
707
+ KEYS: ta.Mapping[str, bool] = {
708
+ 'name': False,
709
+ 'msg': False,
710
+ 'args': False,
711
+ 'levelname': False,
712
+ 'levelno': False,
713
+ 'pathname': False,
714
+ 'filename': False,
715
+ 'module': False,
716
+ 'exc_info': True,
717
+ 'exc_text': True,
718
+ 'stack_info': True,
719
+ 'lineno': False,
720
+ 'funcName': False,
721
+ 'created': False,
722
+ 'msecs': False,
723
+ 'relativeCreated': False,
724
+ 'thread': False,
725
+ 'threadName': False,
726
+ 'processName': False,
727
+ 'process': False,
728
+ }
729
+
730
+ def format(self, record: logging.LogRecord) -> str:
731
+ dct = {
732
+ k: v
733
+ for k, o in self.KEYS.items()
734
+ for v in [getattr(record, k)]
735
+ if not (o and v is None)
736
+ }
737
+ return json_dumps_compact(dct)
738
+
739
+
740
+ ##
741
+
742
+
743
+ STANDARD_LOG_FORMAT_PARTS = [
744
+ ('asctime', '%(asctime)-15s'),
745
+ ('process', 'pid=%(process)-6s'),
746
+ ('thread', 'tid=%(thread)x'),
747
+ ('levelname', '%(levelname)s'),
748
+ ('name', '%(name)s'),
749
+ ('separator', '::'),
750
+ ('message', '%(message)s'),
751
+ ]
752
+
753
+
754
+ class StandardLogFormatter(logging.Formatter):
755
+
756
+ @staticmethod
757
+ def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
758
+ return ' '.join(v for k, v in parts)
759
+
760
+ converter = datetime.datetime.fromtimestamp # type: ignore
761
+
762
+ def formatTime(self, record, datefmt=None):
763
+ ct = self.converter(record.created) # type: ignore
764
+ if datefmt:
765
+ return ct.strftime(datefmt) # noqa
766
+ else:
767
+ t = ct.strftime('%Y-%m-%d %H:%M:%S')
768
+ return '%s.%03d' % (t, record.msecs) # noqa
769
+
770
+
771
+ ##
772
+
773
+
774
+ class ProxyLogFilterer(logging.Filterer):
775
+ def __init__(self, underlying: logging.Filterer) -> None: # noqa
776
+ self._underlying = underlying
777
+
778
+ @property
779
+ def underlying(self) -> logging.Filterer:
780
+ return self._underlying
781
+
782
+ @property
783
+ def filters(self):
784
+ return self._underlying.filters
785
+
786
+ @filters.setter
787
+ def filters(self, filters):
788
+ self._underlying.filters = filters
789
+
790
+ def addFilter(self, filter): # noqa
791
+ self._underlying.addFilter(filter)
792
+
793
+ def removeFilter(self, filter): # noqa
794
+ self._underlying.removeFilter(filter)
795
+
796
+ def filter(self, record):
797
+ return self._underlying.filter(record)
798
+
799
+
800
+ class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
801
+ def __init__(self, underlying: logging.Handler) -> None: # noqa
802
+ ProxyLogFilterer.__init__(self, underlying)
803
+
804
+ _underlying: logging.Handler
805
+
806
+ @property
807
+ def underlying(self) -> logging.Handler:
808
+ return self._underlying
809
+
810
+ def get_name(self):
811
+ return self._underlying.get_name()
812
+
813
+ def set_name(self, name):
814
+ self._underlying.set_name(name)
815
+
816
+ @property
817
+ def name(self):
818
+ return self._underlying.name
819
+
820
+ @property
821
+ def level(self):
822
+ return self._underlying.level
823
+
824
+ @level.setter
825
+ def level(self, level):
826
+ self._underlying.level = level
827
+
828
+ @property
829
+ def formatter(self):
830
+ return self._underlying.formatter
831
+
832
+ @formatter.setter
833
+ def formatter(self, formatter):
834
+ self._underlying.formatter = formatter
835
+
836
+ def createLock(self):
837
+ self._underlying.createLock()
838
+
839
+ def acquire(self):
840
+ self._underlying.acquire()
841
+
842
+ def release(self):
843
+ self._underlying.release()
844
+
845
+ def setLevel(self, level):
846
+ self._underlying.setLevel(level)
847
+
848
+ def format(self, record):
849
+ return self._underlying.format(record)
850
+
851
+ def emit(self, record):
852
+ self._underlying.emit(record)
853
+
854
+ def handle(self, record):
855
+ return self._underlying.handle(record)
856
+
857
+ def setFormatter(self, fmt):
858
+ self._underlying.setFormatter(fmt)
859
+
860
+ def flush(self):
861
+ self._underlying.flush()
862
+
863
+ def close(self):
864
+ self._underlying.close()
865
+
866
+ def handleError(self, record):
867
+ self._underlying.handleError(record)
868
+
869
+
870
+ ##
871
+
872
+
873
+ class StandardLogHandler(ProxyLogHandler):
874
+ pass
875
+
876
+
877
+ ##
878
+
879
+
880
+ @contextlib.contextmanager
881
+ def _locking_logging_module_lock() -> ta.Iterator[None]:
882
+ if hasattr(logging, '_acquireLock'):
883
+ logging._acquireLock() # noqa
884
+ try:
885
+ yield
886
+ finally:
887
+ logging._releaseLock() # type: ignore # noqa
888
+
889
+ elif hasattr(logging, '_lock'):
890
+ # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
891
+ with logging._lock: # noqa
892
+ yield
893
+
894
+ else:
895
+ raise Exception("Can't find lock in logging module")
896
+
897
+
898
+ def configure_standard_logging(
899
+ level: ta.Union[int, str] = logging.INFO,
900
+ *,
901
+ json: bool = False,
902
+ target: ta.Optional[logging.Logger] = None,
903
+ force: bool = False,
904
+ handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
905
+ ) -> ta.Optional[StandardLogHandler]:
906
+ with _locking_logging_module_lock():
907
+ if target is None:
908
+ target = logging.root
909
+
910
+ #
911
+
912
+ if not force:
913
+ if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
914
+ return None
915
+
916
+ #
917
+
918
+ if handler_factory is not None:
919
+ handler = handler_factory()
920
+ else:
921
+ handler = logging.StreamHandler()
922
+
923
+ #
924
+
925
+ formatter: logging.Formatter
926
+ if json:
927
+ formatter = JsonLogFormatter()
928
+ else:
929
+ formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
930
+ handler.setFormatter(formatter)
931
+
932
+ #
933
+
934
+ handler.addFilter(TidLogFilter())
935
+
936
+ #
937
+
938
+ target.addHandler(handler)
939
+
940
+ #
941
+
942
+ if level is not None:
943
+ target.setLevel(level)
944
+
945
+ #
946
+
947
+ return StandardLogHandler(handler)
948
+
949
+
950
+ ########################################
951
+ # ../../../../omlish/lite/marshal.py
952
+ """
953
+ TODO:
954
+ - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
955
+ - nonstrict toggle
956
+ """
957
+
958
+
959
+ ##
960
+
961
+
962
+ class ObjMarshaler(abc.ABC):
963
+ @abc.abstractmethod
964
+ def marshal(self, o: ta.Any) -> ta.Any:
965
+ raise NotImplementedError
966
+
967
+ @abc.abstractmethod
968
+ def unmarshal(self, o: ta.Any) -> ta.Any:
969
+ raise NotImplementedError
970
+
971
+
972
+ class NopObjMarshaler(ObjMarshaler):
973
+ def marshal(self, o: ta.Any) -> ta.Any:
974
+ return o
975
+
976
+ def unmarshal(self, o: ta.Any) -> ta.Any:
977
+ return o
978
+
979
+
980
+ @dc.dataclass()
981
+ class ProxyObjMarshaler(ObjMarshaler):
982
+ m: ta.Optional[ObjMarshaler] = None
983
+
984
+ def marshal(self, o: ta.Any) -> ta.Any:
985
+ return check_not_none(self.m).marshal(o)
986
+
987
+ def unmarshal(self, o: ta.Any) -> ta.Any:
988
+ return check_not_none(self.m).unmarshal(o)
989
+
990
+
991
+ @dc.dataclass(frozen=True)
992
+ class CastObjMarshaler(ObjMarshaler):
993
+ ty: type
994
+
995
+ def marshal(self, o: ta.Any) -> ta.Any:
996
+ return o
997
+
998
+ def unmarshal(self, o: ta.Any) -> ta.Any:
999
+ return self.ty(o)
1000
+
1001
+
1002
+ class DynamicObjMarshaler(ObjMarshaler):
1003
+ def marshal(self, o: ta.Any) -> ta.Any:
1004
+ return marshal_obj(o)
1005
+
1006
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1007
+ return o
1008
+
1009
+
1010
+ @dc.dataclass(frozen=True)
1011
+ class Base64ObjMarshaler(ObjMarshaler):
1012
+ ty: type
1013
+
1014
+ def marshal(self, o: ta.Any) -> ta.Any:
1015
+ return base64.b64encode(o).decode('ascii')
1016
+
1017
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1018
+ return self.ty(base64.b64decode(o))
1019
+
1020
+
1021
+ @dc.dataclass(frozen=True)
1022
+ class EnumObjMarshaler(ObjMarshaler):
1023
+ ty: type
1024
+
1025
+ def marshal(self, o: ta.Any) -> ta.Any:
1026
+ return o.name
1027
+
1028
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1029
+ return self.ty.__members__[o] # type: ignore
1030
+
1031
+
1032
+ @dc.dataclass(frozen=True)
1033
+ class OptionalObjMarshaler(ObjMarshaler):
1034
+ item: ObjMarshaler
1035
+
1036
+ def marshal(self, o: ta.Any) -> ta.Any:
1037
+ if o is None:
1038
+ return None
1039
+ return self.item.marshal(o)
1040
+
1041
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1042
+ if o is None:
1043
+ return None
1044
+ return self.item.unmarshal(o)
1045
+
1046
+
1047
+ @dc.dataclass(frozen=True)
1048
+ class MappingObjMarshaler(ObjMarshaler):
1049
+ ty: type
1050
+ km: ObjMarshaler
1051
+ vm: ObjMarshaler
1052
+
1053
+ def marshal(self, o: ta.Any) -> ta.Any:
1054
+ return {self.km.marshal(k): self.vm.marshal(v) for k, v in o.items()}
1055
+
1056
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1057
+ return self.ty((self.km.unmarshal(k), self.vm.unmarshal(v)) for k, v in o.items())
1058
+
1059
+
1060
+ @dc.dataclass(frozen=True)
1061
+ class IterableObjMarshaler(ObjMarshaler):
1062
+ ty: type
1063
+ item: ObjMarshaler
1064
+
1065
+ def marshal(self, o: ta.Any) -> ta.Any:
1066
+ return [self.item.marshal(e) for e in o]
1067
+
1068
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1069
+ return self.ty(self.item.unmarshal(e) for e in o)
1070
+
1071
+
1072
+ @dc.dataclass(frozen=True)
1073
+ class DataclassObjMarshaler(ObjMarshaler):
1074
+ ty: type
1075
+ fs: ta.Mapping[str, ObjMarshaler]
1076
+ nonstrict: bool = False
1077
+
1078
+ def marshal(self, o: ta.Any) -> ta.Any:
1079
+ return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
1080
+
1081
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1082
+ return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
1083
+
1084
+
1085
+ @dc.dataclass(frozen=True)
1086
+ class PolymorphicObjMarshaler(ObjMarshaler):
1087
+ class Impl(ta.NamedTuple):
1088
+ ty: type
1089
+ tag: str
1090
+ m: ObjMarshaler
1091
+
1092
+ impls_by_ty: ta.Mapping[type, Impl]
1093
+ impls_by_tag: ta.Mapping[str, Impl]
1094
+
1095
+ @classmethod
1096
+ def of(cls, impls: ta.Iterable[Impl]) -> 'PolymorphicObjMarshaler':
1097
+ return cls(
1098
+ {i.ty: i for i in impls},
1099
+ {i.tag: i for i in impls},
1100
+ )
1101
+
1102
+ def marshal(self, o: ta.Any) -> ta.Any:
1103
+ impl = self.impls_by_ty[type(o)]
1104
+ return {impl.tag: impl.m.marshal(o)}
1105
+
1106
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1107
+ [(t, v)] = o.items()
1108
+ impl = self.impls_by_tag[t]
1109
+ return impl.m.unmarshal(v)
1110
+
1111
+
1112
+ @dc.dataclass(frozen=True)
1113
+ class DatetimeObjMarshaler(ObjMarshaler):
1114
+ ty: type
1115
+
1116
+ def marshal(self, o: ta.Any) -> ta.Any:
1117
+ return o.isoformat()
1118
+
1119
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1120
+ return self.ty.fromisoformat(o) # type: ignore
1121
+
1122
+
1123
+ class DecimalObjMarshaler(ObjMarshaler):
1124
+ def marshal(self, o: ta.Any) -> ta.Any:
1125
+ return str(check_isinstance(o, decimal.Decimal))
1126
+
1127
+ def unmarshal(self, v: ta.Any) -> ta.Any:
1128
+ return decimal.Decimal(check_isinstance(v, str))
1129
+
1130
+
1131
+ class FractionObjMarshaler(ObjMarshaler):
1132
+ def marshal(self, o: ta.Any) -> ta.Any:
1133
+ fr = check_isinstance(o, fractions.Fraction)
1134
+ return [fr.numerator, fr.denominator]
1135
+
1136
+ def unmarshal(self, v: ta.Any) -> ta.Any:
1137
+ num, denom = check_isinstance(v, list)
1138
+ return fractions.Fraction(num, denom)
1139
+
1140
+
1141
+ class UuidObjMarshaler(ObjMarshaler):
1142
+ def marshal(self, o: ta.Any) -> ta.Any:
1143
+ return str(o)
1144
+
1145
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1146
+ return uuid.UUID(o)
1147
+
1148
+
1149
+ ##
1150
+
1151
+
1152
+ _DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
1153
+ **{t: NopObjMarshaler() for t in (type(None),)},
1154
+ **{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
1155
+ **{t: Base64ObjMarshaler(t) for t in (bytes, bytearray)},
1156
+ **{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
1157
+ **{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
1158
+
1159
+ ta.Any: DynamicObjMarshaler(),
1160
+
1161
+ **{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
1162
+ decimal.Decimal: DecimalObjMarshaler(),
1163
+ fractions.Fraction: FractionObjMarshaler(),
1164
+ uuid.UUID: UuidObjMarshaler(),
1165
+ }
1166
+
1167
+ _OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
1168
+ **{t: t for t in (dict,)},
1169
+ **{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
1170
+ }
1171
+
1172
+ _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
1173
+ **{t: t for t in (list, tuple, set, frozenset)},
1174
+ collections.abc.Set: frozenset,
1175
+ collections.abc.MutableSet: set,
1176
+ collections.abc.Sequence: tuple,
1177
+ collections.abc.MutableSequence: list,
1178
+ }
1179
+
1180
+
1181
+ def _make_obj_marshaler(
1182
+ ty: ta.Any,
1183
+ rec: ta.Callable[[ta.Any], ObjMarshaler],
1184
+ *,
1185
+ nonstrict_dataclasses: bool = False,
1186
+ ) -> ObjMarshaler:
1187
+ if isinstance(ty, type):
1188
+ if abc.ABC in ty.__bases__:
1189
+ return PolymorphicObjMarshaler.of([ # type: ignore
1190
+ PolymorphicObjMarshaler.Impl(
1191
+ ity,
1192
+ ity.__qualname__,
1193
+ rec(ity),
1194
+ )
1195
+ for ity in deep_subclasses(ty)
1196
+ if abc.ABC not in ity.__bases__
1197
+ ])
1198
+
1199
+ if issubclass(ty, enum.Enum):
1200
+ return EnumObjMarshaler(ty)
1201
+
1202
+ if dc.is_dataclass(ty):
1203
+ return DataclassObjMarshaler(
1204
+ ty,
1205
+ {f.name: rec(f.type) for f in dc.fields(ty)},
1206
+ nonstrict=nonstrict_dataclasses,
1207
+ )
1208
+
1209
+ if is_generic_alias(ty):
1210
+ try:
1211
+ mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
1212
+ except KeyError:
1213
+ pass
1214
+ else:
1215
+ k, v = ta.get_args(ty)
1216
+ return MappingObjMarshaler(mt, rec(k), rec(v))
1217
+
1218
+ try:
1219
+ st = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES[ta.get_origin(ty)]
1220
+ except KeyError:
1221
+ pass
1222
+ else:
1223
+ [e] = ta.get_args(ty)
1224
+ return IterableObjMarshaler(st, rec(e))
1225
+
1226
+ if is_union_alias(ty):
1227
+ return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
1228
+
1229
+ raise TypeError(ty)
1230
+
1231
+
1232
+ ##
1233
+
1234
+
1235
+ _OBJ_MARSHALERS_LOCK = threading.RLock()
1236
+
1237
+ _OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
1238
+
1239
+ _OBJ_MARSHALER_PROXIES: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
1240
+
1241
+
1242
+ def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
1243
+ with _OBJ_MARSHALERS_LOCK:
1244
+ if ty in _OBJ_MARSHALERS:
1245
+ raise KeyError(ty)
1246
+ _OBJ_MARSHALERS[ty] = m
1247
+
1248
+
1249
+ def get_obj_marshaler(
1250
+ ty: ta.Any,
1251
+ *,
1252
+ no_cache: bool = False,
1253
+ **kwargs: ta.Any,
1254
+ ) -> ObjMarshaler:
1255
+ with _OBJ_MARSHALERS_LOCK:
1256
+ if not no_cache:
1257
+ try:
1258
+ return _OBJ_MARSHALERS[ty]
1259
+ except KeyError:
1260
+ pass
1261
+
1262
+ try:
1263
+ return _OBJ_MARSHALER_PROXIES[ty]
1264
+ except KeyError:
1265
+ pass
1266
+
1267
+ rec = functools.partial(
1268
+ get_obj_marshaler,
1269
+ no_cache=no_cache,
1270
+ **kwargs,
1271
+ )
1272
+
1273
+ p = ProxyObjMarshaler()
1274
+ _OBJ_MARSHALER_PROXIES[ty] = p
1275
+ try:
1276
+ m = _make_obj_marshaler(ty, rec, **kwargs)
1277
+ finally:
1278
+ del _OBJ_MARSHALER_PROXIES[ty]
1279
+ p.m = m
1280
+
1281
+ if not no_cache:
1282
+ _OBJ_MARSHALERS[ty] = m
1283
+ return m
1284
+
1285
+
1286
+ ##
1287
+
1288
+
1289
+ def marshal_obj(o: ta.Any, ty: ta.Any = None) -> ta.Any:
1290
+ return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
1291
+
1292
+
1293
+ def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
1294
+ return get_obj_marshaler(ty).unmarshal(o)
1295
+
1296
+
1297
+ ########################################
1298
+ # ../../../../omlish/lite/runtime.py
1299
+
1300
+
1301
+ @cached_nullary
1302
+ def is_debugger_attached() -> bool:
1303
+ return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
1304
+
1305
+
1306
+ REQUIRED_PYTHON_VERSION = (3, 8)
1307
+
1308
+
1309
+ def check_runtime_version() -> None:
1310
+ if sys.version_info < REQUIRED_PYTHON_VERSION:
1311
+ raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
1312
+
1313
+
1314
+ ########################################
1315
+ # ../../../../omlish/lite/subprocesses.py
1316
+
1317
+
1318
+ ##
1319
+
1320
+
1321
+ _SUBPROCESS_SHELL_WRAP_EXECS = False
1322
+
1323
+
1324
+ def subprocess_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
1325
+ return ('sh', '-c', ' '.join(map(shlex.quote, args)))
1326
+
1327
+
1328
+ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
1329
+ if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
1330
+ return subprocess_shell_wrap_exec(*args)
1331
+ else:
1332
+ return args
1333
+
1334
+
1335
+ def _prepare_subprocess_invocation(
1336
+ *args: str,
1337
+ env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
1338
+ extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
1339
+ quiet: bool = False,
1340
+ shell: bool = False,
1341
+ **kwargs: ta.Any,
1342
+ ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
1343
+ log.debug(args)
1344
+ if extra_env:
1345
+ log.debug(extra_env)
1346
+
1347
+ if extra_env:
1348
+ env = {**(env if env is not None else os.environ), **extra_env}
1349
+
1350
+ if quiet and 'stderr' not in kwargs:
1351
+ if not log.isEnabledFor(logging.DEBUG):
1352
+ kwargs['stderr'] = subprocess.DEVNULL
1353
+
1354
+ if not shell:
1355
+ args = subprocess_maybe_shell_wrap_exec(*args)
1356
+
1357
+ return args, dict(
1358
+ env=env,
1359
+ shell=shell,
1360
+ **kwargs,
1361
+ )
1362
+
1363
+
1364
+ def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
1365
+ args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
1366
+ return subprocess.check_call(args, **kwargs) # type: ignore
1367
+
1368
+
1369
+ def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
1370
+ args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
1371
+ return subprocess.check_output(args, **kwargs)
1372
+
1373
+
1374
+ def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
1375
+ return subprocess_check_output(*args, **kwargs).decode().strip()
1376
+
1377
+
1378
+ ##
1379
+
1380
+
1381
+ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
1382
+ FileNotFoundError,
1383
+ subprocess.CalledProcessError,
1384
+ )
1385
+
1386
+
1387
+ def subprocess_try_call(
1388
+ *args: str,
1389
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
1390
+ **kwargs: ta.Any,
1391
+ ) -> bool:
1392
+ try:
1393
+ subprocess_check_call(*args, **kwargs)
1394
+ except try_exceptions as e: # noqa
1395
+ if log.isEnabledFor(logging.DEBUG):
1396
+ log.exception('command failed')
1397
+ return False
1398
+ else:
1399
+ return True
1400
+
1401
+
1402
+ def subprocess_try_output(
1403
+ *args: str,
1404
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
1405
+ **kwargs: ta.Any,
1406
+ ) -> ta.Optional[bytes]:
1407
+ try:
1408
+ return subprocess_check_output(*args, **kwargs)
1409
+ except try_exceptions as e: # noqa
1410
+ if log.isEnabledFor(logging.DEBUG):
1411
+ log.exception('command failed')
1412
+ return None
1413
+
1414
+
1415
+ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
1416
+ out = subprocess_try_output(*args, **kwargs)
1417
+ return out.decode().strip() if out is not None else None
1418
+
1419
+
1420
+ ##
1421
+
1422
+
1423
+ def subprocess_close(
1424
+ proc: subprocess.Popen,
1425
+ timeout: ta.Optional[float] = None,
1426
+ ) -> None:
1427
+ # TODO: terminate, sleep, kill
1428
+ if proc.stdout:
1429
+ proc.stdout.close()
1430
+ if proc.stderr:
1431
+ proc.stderr.close()
1432
+ if proc.stdin:
1433
+ proc.stdin.close()
1434
+
1435
+ proc.wait(timeout)
1436
+
1437
+
1438
+ ########################################
1439
+ # ../commands/subprocess.py
1440
+
1441
+
1442
+ ##
1443
+
1444
+
1445
+ class SubprocessCommand(Command['SubprocessCommand.Input', 'SubprocessCommand.Output']):
1446
+ @dc.dataclass(frozen=True)
1447
+ class Input(Command.Input):
1448
+ args: ta.Sequence[str]
1449
+
1450
+ shell: bool = False
1451
+ cwd: ta.Optional[str] = None
1452
+ env: ta.Optional[ta.Mapping[str, str]] = None
1453
+
1454
+ capture_stdout: bool = False
1455
+ capture_stderr: bool = False
1456
+
1457
+ input: ta.Optional[bytes] = None
1458
+ timeout: ta.Optional[float] = None
1459
+
1460
+ def __post_init__(self) -> None:
1461
+ if isinstance(self.args, str):
1462
+ raise TypeError(self.args)
1463
+
1464
+ @dc.dataclass(frozen=True)
1465
+ class Output(Command.Output):
1466
+ rc: int
1467
+ pid: int
1468
+
1469
+ elapsed_s: float
1470
+
1471
+ stdout: ta.Optional[bytes] = None
1472
+ stderr: ta.Optional[bytes] = None
1473
+
1474
+ def _execute(self, inp: Input) -> Output:
1475
+ proc = subprocess.Popen(
1476
+ subprocess_maybe_shell_wrap_exec(*inp.args),
1477
+
1478
+ shell=inp.shell,
1479
+ cwd=inp.cwd,
1480
+ env={**os.environ, **(inp.env or {})},
1481
+
1482
+ stdin=subprocess.PIPE if inp.input is not None else None,
1483
+ stdout=subprocess.PIPE if inp.capture_stdout else None,
1484
+ stderr=subprocess.PIPE if inp.capture_stderr else None,
1485
+ )
1486
+
1487
+ start_time = time.time()
1488
+ stdout, stderr = proc.communicate(
1489
+ input=inp.input,
1490
+ timeout=inp.timeout,
1491
+ )
1492
+ end_time = time.time()
1493
+
1494
+ return SubprocessCommand.Output(
1495
+ rc=proc.returncode,
1496
+ pid=proc.pid,
1497
+
1498
+ elapsed_s=end_time - start_time,
1499
+
1500
+ stdout=stdout, # noqa
1501
+ stderr=stderr, # noqa
1502
+ )
1503
+
1504
+
1505
+ ########################################
1506
+ # main.py
1507
+
1508
+
1509
+ ##
1510
+
1511
+
1512
+ def _send_obj(f: ta.IO, o: ta.Any) -> None:
1513
+ j = json_dumps_compact(marshal_obj(o))
1514
+ d = j.encode('utf-8')
1515
+
1516
+ f.write(struct.pack('<I', len(d)))
1517
+ f.write(d)
1518
+ f.flush()
1519
+
1520
+
1521
+ def _recv_obj(f: ta.IO, ty: type) -> ta.Any:
1522
+ d = f.read(4)
1523
+ if not d:
1524
+ return None
1525
+ if len(d) != 4:
1526
+ raise Exception
1527
+
1528
+ sz = struct.unpack('<I', d)[0]
1529
+ d = f.read(sz)
1530
+ if not d:
1531
+ raise Exception
1532
+
1533
+ j = json.loads(d.decode('utf-8'))
1534
+ return unmarshal_obj(j, ty)
1535
+
1536
+
1537
+ def _remote_main() -> None:
1538
+ rt = pyremote_bootstrap_finalize() # noqa
1539
+
1540
+ while True:
1541
+ i = _recv_obj(rt.input, SubprocessCommand.Input)
1542
+ if i is None:
1543
+ break
1544
+
1545
+ o = SubprocessCommand()._execute(i) # noqa
1546
+
1547
+ _send_obj(rt.output, o)
1548
+
1549
+
1550
+ def _main() -> None:
1551
+ import argparse
1552
+
1553
+ parser = argparse.ArgumentParser()
1554
+
1555
+ parser.add_argument('-s', '--shell')
1556
+ parser.add_argument('-q', '--shell-quote', action='store_true')
1557
+ parser.add_argument('--python', default='python3')
1558
+ parser.add_argument('--_amalg-file')
1559
+
1560
+ args = parser.parse_args()
1561
+
1562
+ #
1563
+
1564
+ self_src = inspect.getsource(sys.modules[__name__])
1565
+ self_src_lines = self_src.splitlines()
1566
+ for l in self_src_lines:
1567
+ if l.startswith('# @omlish-amalg-output '):
1568
+ is_self_amalg = True
1569
+ break
1570
+ else:
1571
+ is_self_amalg = False
1572
+
1573
+ if is_self_amalg:
1574
+ amalg_src = self_src
1575
+ else:
1576
+ amalg_file = args._amalg_file # noqa
1577
+ if amalg_file is None:
1578
+ amalg_file = os.path.join(os.path.dirname(__file__), '_manage.py')
1579
+ with open(amalg_file) as f:
1580
+ amalg_src = f.read()
1581
+
1582
+ #
1583
+
1584
+ remote_src = '\n\n'.join([
1585
+ '__name__ = "__remote__"',
1586
+ amalg_src,
1587
+ '_remote_main()',
1588
+ ])
1589
+
1590
+ #
1591
+
1592
+ bs_src = pyremote_build_bootstrap_cmd(__package__ or 'manage')
1593
+
1594
+ if args.shell is not None:
1595
+ sh_src = f'{args.python} -c {shlex.quote(bs_src)}'
1596
+ if args.shell_quote:
1597
+ sh_src = shlex.quote(sh_src)
1598
+ sh_cmd = f'{args.shell} {sh_src}'
1599
+ cmd = [sh_cmd]
1600
+ shell = True
1601
+ else:
1602
+ cmd = [args.python, '-c', bs_src]
1603
+ shell = False
1604
+
1605
+ proc = subprocess.Popen(
1606
+ subprocess_maybe_shell_wrap_exec(*cmd),
1607
+ shell=shell,
1608
+ stdin=subprocess.PIPE,
1609
+ stdout=subprocess.PIPE,
1610
+ )
1611
+
1612
+ stdin = check_not_none(proc.stdin)
1613
+ stdout = check_not_none(proc.stdout)
1614
+
1615
+ res = PyremoteBootstrapDriver(remote_src).run(stdin, stdout)
1616
+ print(res)
1617
+
1618
+ #
1619
+
1620
+ for ci in [
1621
+ SubprocessCommand.Input(
1622
+ args=['python3', '-'],
1623
+ input=b'print(1)\n',
1624
+ capture_stdout=True,
1625
+ ),
1626
+ SubprocessCommand.Input(
1627
+ args=['uname'],
1628
+ capture_stdout=True,
1629
+ ),
1630
+ ]:
1631
+ _send_obj(stdin, ci)
1632
+
1633
+ o = _recv_obj(stdout, SubprocessCommand.Output)
1634
+
1635
+ print(o)
1636
+
1637
+ try:
1638
+ stdin.close()
1639
+ except BrokenPipeError:
1640
+ pass
1641
+
1642
+ proc.wait()
1643
+
1644
+
1645
+ if __name__ == '__main__':
1646
+ _main()