ominfra 0.0.0.dev134__py3-none-any.whl → 0.0.0.dev136__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1646 @@
1
+ #!/usr/bin/env python3
2
+ # noinspection DuplicatedCode
3
+ # @omlish-lite
4
+ # @omlish-script
5
+ # @omlish-amalg-output main.py
6
+ # ruff: noqa: N802 UP006 UP007 UP036
7
+ import abc
8
+ import base64
9
+ import collections.abc
10
+ import contextlib
11
+ import dataclasses as dc
12
+ import datetime
13
+ import decimal
14
+ import enum
15
+ import fractions
16
+ import functools
17
+ import inspect
18
+ import json
19
+ import logging
20
+ import os
21
+ import platform
22
+ import pwd
23
+ import shlex
24
+ import site
25
+ import struct
26
+ import subprocess
27
+ import sys
28
+ import textwrap
29
+ import threading
30
+ import time
31
+ import types
32
+ import typing as ta
33
+ import uuid
34
+ import weakref # noqa
35
+ import zlib
36
+
37
+
38
+ ########################################
39
+
40
+
41
+ if sys.version_info < (3, 8):
42
+ raise OSError(f'Requires python (3, 8), got {sys.version_info} from {sys.executable}') # noqa
43
+
44
+
45
+ ########################################
46
+
47
+
48
+ # commands/base.py
49
+ CommandInputT = ta.TypeVar('CommandInputT', bound='Command.Input')
50
+ CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
51
+
52
+ # ../../../omlish/lite/cached.py
53
+ T = ta.TypeVar('T')
54
+
55
+ # ../../../omlish/lite/check.py
56
+ SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
57
+
58
+
59
+ ########################################
60
+ # ../commands/base.py
61
+
62
+
63
+ ##
64
+
65
+
66
+ class Command(abc.ABC, ta.Generic[CommandInputT, CommandOutputT]):
67
+ @dc.dataclass(frozen=True)
68
+ class Input(abc.ABC): # noqa
69
+ pass
70
+
71
+ @dc.dataclass(frozen=True)
72
+ class Output(abc.ABC): # noqa
73
+ pass
74
+
75
+ @abc.abstractmethod
76
+ def _execute(self, inp: CommandInputT) -> CommandOutputT:
77
+ raise NotImplementedError
78
+
79
+
80
+ ########################################
81
+ # ../../../pyremote.py
82
+ """
83
+ Basically this: https://mitogen.networkgenomics.com/howitworks.html
84
+ """
85
+
86
+
87
+ ##
88
+
89
+
90
+ _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
91
+ _PYREMOTE_BOOTSTRAP_SRC_FD = 101
92
+
93
+ _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
94
+ _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
95
+
96
+ _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
97
+ _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
98
+ _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
99
+ _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
100
+
101
+ _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
102
+
103
+ _PYREMOTE_BOOTSTRAP_IMPORTS = [
104
+ 'base64',
105
+ 'os',
106
+ 'struct',
107
+ 'sys',
108
+ 'zlib',
109
+ ]
110
+
111
+
112
+ def _pyremote_bootstrap_main(context_name: str) -> None:
113
+ # Get pid
114
+ pid = os.getpid()
115
+
116
+ # Two copies of main src to be sent to parent
117
+ r0, w0 = os.pipe()
118
+ r1, w1 = os.pipe()
119
+
120
+ if (cp := os.fork()):
121
+ # Parent process
122
+
123
+ # Dup original stdin to comm_fd for use as comm channel
124
+ os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
125
+
126
+ # Overwrite stdin (fed to python repl) with first copy of src
127
+ os.dup2(r0, 0)
128
+
129
+ # Dup second copy of src to src_fd to recover after launch
130
+ os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
131
+
132
+ # Close remaining fd's
133
+ for f in [r0, w0, r1, w1]:
134
+ os.close(f)
135
+
136
+ # Save child pid to close after relaunch
137
+ os.environ[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
138
+
139
+ # Save original argv0
140
+ os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = sys.executable
141
+
142
+ # Start repl reading stdin from r0
143
+ os.execl(sys.executable, sys.executable + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
144
+
145
+ else:
146
+ # Child process
147
+
148
+ # Write first ack
149
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
150
+
151
+ # Write pid
152
+ os.write(1, struct.pack('<Q', pid))
153
+
154
+ # Read main src from stdin
155
+ main_z_len = struct.unpack('<I', os.read(0, 4))[0]
156
+ main_src = zlib.decompress(os.fdopen(0, 'rb').read(main_z_len))
157
+
158
+ # Write both copies of main src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely fill
159
+ # and block and need to be drained by pyremote_bootstrap_finalize running in parent.
160
+ for w in [w0, w1]:
161
+ fp = os.fdopen(w, 'wb', 0)
162
+ fp.write(main_src)
163
+ fp.close()
164
+
165
+ # Write second ack
166
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
167
+
168
+ # Exit child
169
+ sys.exit(0)
170
+
171
+
172
+ ##
173
+
174
+
175
+ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
176
+ if any(c in context_name for c in '\'"'):
177
+ raise NameError(context_name)
178
+
179
+ bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
180
+
181
+ for gl in [
182
+ '_PYREMOTE_BOOTSTRAP_INPUT_FD',
183
+ '_PYREMOTE_BOOTSTRAP_SRC_FD',
184
+
185
+ '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
186
+ '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
187
+
188
+ '_PYREMOTE_BOOTSTRAP_ACK0',
189
+ '_PYREMOTE_BOOTSTRAP_ACK1',
190
+
191
+ '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
192
+ ]:
193
+ bs_src = bs_src.replace(gl, repr(globals()[gl]))
194
+
195
+ bs_src = '\n'.join(
196
+ cl
197
+ for l in bs_src.splitlines()
198
+ if (cl := (l.split('#')[0]).rstrip())
199
+ if cl.strip()
200
+ )
201
+
202
+ bs_z = zlib.compress(bs_src.encode('utf-8'))
203
+ bs_z64 = base64.encodebytes(bs_z).replace(b'\n', b'')
204
+
205
+ def dq_repr(o: ta.Any) -> str:
206
+ return '"' + repr(o)[1:-1] + '"'
207
+
208
+ stmts = [
209
+ f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
210
+ f'exec(zlib.decompress(base64.decodebytes({bs_z64!r})))',
211
+ f'_pyremote_bootstrap_main({context_name!r})',
212
+ ]
213
+
214
+ cmd = '; '.join(stmts)
215
+ return cmd
216
+
217
+
218
+ ##
219
+
220
+
221
+ @dc.dataclass(frozen=True)
222
+ class PyremoteEnvInfo:
223
+ sys_base_prefix: str
224
+ sys_byteorder: str
225
+ sys_defaultencoding: str
226
+ sys_exec_prefix: str
227
+ sys_executable: str
228
+ sys_implementation_name: str
229
+ sys_path: ta.List[str]
230
+ sys_platform: str
231
+ sys_prefix: str
232
+ sys_version: str
233
+ sys_version_info: ta.List[ta.Union[int, str]]
234
+
235
+ platform_architecture: ta.List[str]
236
+ platform_machine: str
237
+ platform_platform: str
238
+ platform_processor: str
239
+ platform_system: str
240
+ platform_release: str
241
+ platform_version: str
242
+
243
+ site_userbase: str
244
+
245
+ os_cwd: str
246
+ os_gid: int
247
+ os_loadavg: ta.List[float]
248
+ os_login: ta.Optional[str]
249
+ os_pgrp: int
250
+ os_pid: int
251
+ os_ppid: int
252
+ os_uid: int
253
+
254
+ pw_name: str
255
+ pw_uid: int
256
+ pw_gid: int
257
+ pw_gecos: str
258
+ pw_dir: str
259
+ pw_shell: str
260
+
261
+ env_path: ta.Optional[str]
262
+
263
+
264
+ def _get_pyremote_env_info() -> PyremoteEnvInfo:
265
+ os_uid = os.getuid()
266
+
267
+ pw = pwd.getpwuid(os_uid)
268
+
269
+ os_login: ta.Optional[str]
270
+ try:
271
+ os_login = os.getlogin()
272
+ except OSError:
273
+ os_login = None
274
+
275
+ return PyremoteEnvInfo(
276
+ sys_base_prefix=sys.base_prefix,
277
+ sys_byteorder=sys.byteorder,
278
+ sys_defaultencoding=sys.getdefaultencoding(),
279
+ sys_exec_prefix=sys.exec_prefix,
280
+ sys_executable=sys.executable,
281
+ sys_implementation_name=sys.implementation.name,
282
+ sys_path=sys.path,
283
+ sys_platform=sys.platform,
284
+ sys_prefix=sys.prefix,
285
+ sys_version=sys.version,
286
+ sys_version_info=list(sys.version_info),
287
+
288
+ platform_architecture=list(platform.architecture()),
289
+ platform_machine=platform.machine(),
290
+ platform_platform=platform.platform(),
291
+ platform_processor=platform.processor(),
292
+ platform_system=platform.system(),
293
+ platform_release=platform.release(),
294
+ platform_version=platform.version(),
295
+
296
+ site_userbase=site.getuserbase(),
297
+
298
+ os_cwd=os.getcwd(),
299
+ os_gid=os.getgid(),
300
+ os_loadavg=list(os.getloadavg()),
301
+ os_login=os_login,
302
+ os_pgrp=os.getpgrp(),
303
+ os_pid=os.getpid(),
304
+ os_ppid=os.getppid(),
305
+ os_uid=os_uid,
306
+
307
+ pw_name=pw.pw_name,
308
+ pw_uid=pw.pw_uid,
309
+ pw_gid=pw.pw_gid,
310
+ pw_gecos=pw.pw_gecos,
311
+ pw_dir=pw.pw_dir,
312
+ pw_shell=pw.pw_shell,
313
+
314
+ env_path=os.environ.get('PATH'),
315
+ )
316
+
317
+
318
+ ##
319
+
320
+
321
+ class PyremoteBootstrapDriver:
322
+ def __init__(self, main_src: str) -> None:
323
+ super().__init__()
324
+
325
+ self._main_src = main_src
326
+ self._main_z = zlib.compress(main_src.encode('utf-8'))
327
+
328
+ #
329
+
330
+ @dc.dataclass(frozen=True)
331
+ class Read:
332
+ sz: int
333
+
334
+ @dc.dataclass(frozen=True)
335
+ class Write:
336
+ d: bytes
337
+
338
+ class ProtocolError(Exception):
339
+ pass
340
+
341
+ @dc.dataclass(frozen=True)
342
+ class Result:
343
+ pid: int
344
+ env_info: PyremoteEnvInfo
345
+
346
+ def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
347
+ # Read first ack
348
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
349
+
350
+ # Read pid
351
+ d = yield from self._read(8)
352
+ pid = struct.unpack('<Q', d)[0]
353
+
354
+ # Write main src
355
+ yield from self._write(struct.pack('<I', len(self._main_z)))
356
+ yield from self._write(self._main_z)
357
+
358
+ # Read second and third ack
359
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
360
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
361
+
362
+ # Read env info
363
+ d = yield from self._read(4)
364
+ env_info_json_len = struct.unpack('<I', d)[0]
365
+ d = yield from self._read(env_info_json_len)
366
+ env_info_json = d.decode('utf-8')
367
+ env_info = PyremoteEnvInfo(**json.loads(env_info_json))
368
+
369
+ # Read fourth ack
370
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
371
+
372
+ # Return
373
+ return self.Result(
374
+ pid=pid,
375
+ env_info=env_info,
376
+ )
377
+
378
+ def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
379
+ d = yield self.Read(sz)
380
+ if not isinstance(d, bytes):
381
+ raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
382
+ if len(d) != sz:
383
+ raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
384
+ return d
385
+
386
+ def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
387
+ d = yield from self._read(len(e))
388
+ if d != e:
389
+ raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
390
+
391
+ def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
392
+ i = yield self.Write(d)
393
+ if i is not None:
394
+ raise self.ProtocolError('Unexpected input after write')
395
+
396
+ #
397
+
398
+ def run(self, stdin: ta.IO, stdout: ta.IO) -> Result:
399
+ gen = self.gen()
400
+
401
+ gi: ta.Optional[bytes] = None
402
+ while True:
403
+ try:
404
+ if gi is not None:
405
+ go = gen.send(gi)
406
+ else:
407
+ go = next(gen)
408
+ except StopIteration as e:
409
+ return e.value
410
+
411
+ if isinstance(go, self.Read):
412
+ gi = stdout.read(go.sz)
413
+ elif isinstance(go, self.Write):
414
+ gi = None
415
+ stdin.write(go.d)
416
+ stdin.flush()
417
+ else:
418
+ raise TypeError(go)
419
+
420
+
421
+ ##
422
+
423
+
424
+ @dc.dataclass(frozen=True)
425
+ class PyremotePayloadRuntime:
426
+ input: ta.BinaryIO
427
+ output: ta.BinaryIO
428
+ main_src: str
429
+ env_info: PyremoteEnvInfo
430
+
431
+
432
+ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
433
+ # Restore original argv0
434
+ sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
435
+
436
+ # Read second copy of main src
437
+ r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
438
+ main_src = r1.read().decode('utf-8')
439
+ r1.close()
440
+
441
+ # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in a
442
+ # pipe at once.
443
+ os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
444
+
445
+ # Write third ack
446
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
447
+
448
+ # Write env info
449
+ env_info = _get_pyremote_env_info()
450
+ env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
451
+ os.write(1, struct.pack('<I', len(env_info_json)))
452
+ os.write(1, env_info_json.encode('utf-8'))
453
+
454
+ # Setup IO
455
+ input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
456
+ output = os.fdopen(os.dup(1), 'wb', 0) # noqa
457
+ os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
458
+ os.close(nfd)
459
+
460
+ # Write fourth ack
461
+ output.write(_PYREMOTE_BOOTSTRAP_ACK3)
462
+
463
+ # Return
464
+ return PyremotePayloadRuntime(
465
+ input=input,
466
+ output=output,
467
+ main_src=main_src,
468
+ env_info=env_info,
469
+ )
470
+
471
+
472
+ ########################################
473
+ # ../../../../omlish/lite/cached.py
474
+
475
+
476
+ class _cached_nullary: # noqa
477
+ def __init__(self, fn):
478
+ super().__init__()
479
+ self._fn = fn
480
+ self._value = self._missing = object()
481
+ functools.update_wrapper(self, fn)
482
+
483
+ def __call__(self, *args, **kwargs): # noqa
484
+ if self._value is self._missing:
485
+ self._value = self._fn()
486
+ return self._value
487
+
488
+ def __get__(self, instance, owner): # noqa
489
+ bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
490
+ return bound
491
+
492
+
493
+ def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
494
+ return _cached_nullary(fn)
495
+
496
+
497
+ ########################################
498
+ # ../../../../omlish/lite/check.py
499
+
500
+
501
+ def check_isinstance(v: ta.Any, spec: ta.Union[ta.Type[T], tuple]) -> T:
502
+ if not isinstance(v, spec):
503
+ raise TypeError(v)
504
+ return v
505
+
506
+
507
+ def check_not_isinstance(v: T, spec: ta.Union[type, tuple]) -> T:
508
+ if isinstance(v, spec):
509
+ raise TypeError(v)
510
+ return v
511
+
512
+
513
+ def check_none(v: T) -> None:
514
+ if v is not None:
515
+ raise ValueError(v)
516
+
517
+
518
+ def check_not_none(v: ta.Optional[T]) -> T:
519
+ if v is None:
520
+ raise ValueError
521
+ return v
522
+
523
+
524
+ def check_not(v: ta.Any) -> None:
525
+ if v:
526
+ raise ValueError(v)
527
+ return v
528
+
529
+
530
+ def check_non_empty_str(v: ta.Optional[str]) -> str:
531
+ if not v:
532
+ raise ValueError
533
+ return v
534
+
535
+
536
+ def check_state(v: bool, msg: str = 'Illegal state') -> None:
537
+ if not v:
538
+ raise ValueError(msg)
539
+
540
+
541
+ def check_equal(l: T, r: T) -> T:
542
+ if l != r:
543
+ raise ValueError(l, r)
544
+ return l
545
+
546
+
547
+ def check_not_equal(l: T, r: T) -> T:
548
+ if l == r:
549
+ raise ValueError(l, r)
550
+ return l
551
+
552
+
553
+ def check_is(l: T, r: T) -> T:
554
+ if l is not r:
555
+ raise ValueError(l, r)
556
+ return l
557
+
558
+
559
+ def check_is_not(l: T, r: ta.Any) -> T:
560
+ if l is r:
561
+ raise ValueError(l, r)
562
+ return l
563
+
564
+
565
+ def check_in(v: T, c: ta.Container[T]) -> T:
566
+ if v not in c:
567
+ raise ValueError(v, c)
568
+ return v
569
+
570
+
571
+ def check_not_in(v: T, c: ta.Container[T]) -> T:
572
+ if v in c:
573
+ raise ValueError(v, c)
574
+ return v
575
+
576
+
577
+ def check_single(vs: ta.Iterable[T]) -> T:
578
+ [v] = vs
579
+ return v
580
+
581
+
582
+ def check_empty(v: SizedT) -> SizedT:
583
+ if len(v):
584
+ raise ValueError(v)
585
+ return v
586
+
587
+
588
+ def check_non_empty(v: SizedT) -> SizedT:
589
+ if not len(v):
590
+ raise ValueError(v)
591
+ return v
592
+
593
+
594
+ ########################################
595
+ # ../../../../omlish/lite/json.py
596
+
597
+
598
+ ##
599
+
600
+
601
+ JSON_PRETTY_INDENT = 2
602
+
603
+ JSON_PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
604
+ indent=JSON_PRETTY_INDENT,
605
+ )
606
+
607
+ json_dump_pretty: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_PRETTY_KWARGS) # type: ignore
608
+ json_dumps_pretty: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_PRETTY_KWARGS)
609
+
610
+
611
+ ##
612
+
613
+
614
+ JSON_COMPACT_SEPARATORS = (',', ':')
615
+
616
+ JSON_COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
617
+ indent=None,
618
+ separators=JSON_COMPACT_SEPARATORS,
619
+ )
620
+
621
+ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_COMPACT_KWARGS) # type: ignore
622
+ json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
623
+
624
+
625
+ ########################################
626
+ # ../../../../omlish/lite/reflect.py
627
+
628
+
629
+ _GENERIC_ALIAS_TYPES = (
630
+ ta._GenericAlias, # type: ignore # noqa
631
+ *([ta._SpecialGenericAlias] if hasattr(ta, '_SpecialGenericAlias') else []), # noqa
632
+ )
633
+
634
+
635
+ def is_generic_alias(obj, *, origin: ta.Any = None) -> bool:
636
+ return (
637
+ isinstance(obj, _GENERIC_ALIAS_TYPES) and
638
+ (origin is None or ta.get_origin(obj) is origin)
639
+ )
640
+
641
+
642
+ is_union_alias = functools.partial(is_generic_alias, origin=ta.Union)
643
+ is_callable_alias = functools.partial(is_generic_alias, origin=ta.Callable)
644
+
645
+
646
+ def is_optional_alias(spec: ta.Any) -> bool:
647
+ return (
648
+ isinstance(spec, _GENERIC_ALIAS_TYPES) and # noqa
649
+ ta.get_origin(spec) is ta.Union and
650
+ len(ta.get_args(spec)) == 2 and
651
+ any(a in (None, type(None)) for a in ta.get_args(spec))
652
+ )
653
+
654
+
655
+ def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
656
+ [it] = [it for it in ta.get_args(spec) if it not in (None, type(None))]
657
+ return it
658
+
659
+
660
+ def is_new_type(spec: ta.Any) -> bool:
661
+ if isinstance(ta.NewType, type):
662
+ return isinstance(spec, ta.NewType)
663
+ else:
664
+ # Before https://github.com/python/cpython/commit/c2f33dfc83ab270412bf243fb21f724037effa1a
665
+ return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1] # type: ignore # noqa
666
+
667
+
668
+ def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
669
+ seen = set()
670
+ todo = list(reversed(cls.__subclasses__()))
671
+ while todo:
672
+ cur = todo.pop()
673
+ if cur in seen:
674
+ continue
675
+ seen.add(cur)
676
+ yield cur
677
+ todo.extend(reversed(cur.__subclasses__()))
678
+
679
+
680
+ ########################################
681
+ # ../../../../omlish/lite/logs.py
682
+ """
683
+ TODO:
684
+ - translate json keys
685
+ - debug
686
+ """
687
+
688
+
689
+ log = logging.getLogger(__name__)
690
+
691
+
692
+ ##
693
+
694
+
695
+ class TidLogFilter(logging.Filter):
696
+
697
+ def filter(self, record):
698
+ record.tid = threading.get_native_id()
699
+ return True
700
+
701
+
702
+ ##
703
+
704
+
705
+ class JsonLogFormatter(logging.Formatter):
706
+
707
+ KEYS: ta.Mapping[str, bool] = {
708
+ 'name': False,
709
+ 'msg': False,
710
+ 'args': False,
711
+ 'levelname': False,
712
+ 'levelno': False,
713
+ 'pathname': False,
714
+ 'filename': False,
715
+ 'module': False,
716
+ 'exc_info': True,
717
+ 'exc_text': True,
718
+ 'stack_info': True,
719
+ 'lineno': False,
720
+ 'funcName': False,
721
+ 'created': False,
722
+ 'msecs': False,
723
+ 'relativeCreated': False,
724
+ 'thread': False,
725
+ 'threadName': False,
726
+ 'processName': False,
727
+ 'process': False,
728
+ }
729
+
730
+ def format(self, record: logging.LogRecord) -> str:
731
+ dct = {
732
+ k: v
733
+ for k, o in self.KEYS.items()
734
+ for v in [getattr(record, k)]
735
+ if not (o and v is None)
736
+ }
737
+ return json_dumps_compact(dct)
738
+
739
+
740
+ ##
741
+
742
+
743
+ STANDARD_LOG_FORMAT_PARTS = [
744
+ ('asctime', '%(asctime)-15s'),
745
+ ('process', 'pid=%(process)-6s'),
746
+ ('thread', 'tid=%(thread)x'),
747
+ ('levelname', '%(levelname)s'),
748
+ ('name', '%(name)s'),
749
+ ('separator', '::'),
750
+ ('message', '%(message)s'),
751
+ ]
752
+
753
+
754
+ class StandardLogFormatter(logging.Formatter):
755
+
756
+ @staticmethod
757
+ def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
758
+ return ' '.join(v for k, v in parts)
759
+
760
+ converter = datetime.datetime.fromtimestamp # type: ignore
761
+
762
+ def formatTime(self, record, datefmt=None):
763
+ ct = self.converter(record.created) # type: ignore
764
+ if datefmt:
765
+ return ct.strftime(datefmt) # noqa
766
+ else:
767
+ t = ct.strftime('%Y-%m-%d %H:%M:%S')
768
+ return '%s.%03d' % (t, record.msecs) # noqa
769
+
770
+
771
+ ##
772
+
773
+
774
+ class ProxyLogFilterer(logging.Filterer):
775
+ def __init__(self, underlying: logging.Filterer) -> None: # noqa
776
+ self._underlying = underlying
777
+
778
+ @property
779
+ def underlying(self) -> logging.Filterer:
780
+ return self._underlying
781
+
782
+ @property
783
+ def filters(self):
784
+ return self._underlying.filters
785
+
786
+ @filters.setter
787
+ def filters(self, filters):
788
+ self._underlying.filters = filters
789
+
790
+ def addFilter(self, filter): # noqa
791
+ self._underlying.addFilter(filter)
792
+
793
+ def removeFilter(self, filter): # noqa
794
+ self._underlying.removeFilter(filter)
795
+
796
+ def filter(self, record):
797
+ return self._underlying.filter(record)
798
+
799
+
800
+ class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
801
+ def __init__(self, underlying: logging.Handler) -> None: # noqa
802
+ ProxyLogFilterer.__init__(self, underlying)
803
+
804
+ _underlying: logging.Handler
805
+
806
+ @property
807
+ def underlying(self) -> logging.Handler:
808
+ return self._underlying
809
+
810
+ def get_name(self):
811
+ return self._underlying.get_name()
812
+
813
+ def set_name(self, name):
814
+ self._underlying.set_name(name)
815
+
816
+ @property
817
+ def name(self):
818
+ return self._underlying.name
819
+
820
+ @property
821
+ def level(self):
822
+ return self._underlying.level
823
+
824
+ @level.setter
825
+ def level(self, level):
826
+ self._underlying.level = level
827
+
828
+ @property
829
+ def formatter(self):
830
+ return self._underlying.formatter
831
+
832
+ @formatter.setter
833
+ def formatter(self, formatter):
834
+ self._underlying.formatter = formatter
835
+
836
+ def createLock(self):
837
+ self._underlying.createLock()
838
+
839
+ def acquire(self):
840
+ self._underlying.acquire()
841
+
842
+ def release(self):
843
+ self._underlying.release()
844
+
845
+ def setLevel(self, level):
846
+ self._underlying.setLevel(level)
847
+
848
+ def format(self, record):
849
+ return self._underlying.format(record)
850
+
851
+ def emit(self, record):
852
+ self._underlying.emit(record)
853
+
854
+ def handle(self, record):
855
+ return self._underlying.handle(record)
856
+
857
+ def setFormatter(self, fmt):
858
+ self._underlying.setFormatter(fmt)
859
+
860
+ def flush(self):
861
+ self._underlying.flush()
862
+
863
+ def close(self):
864
+ self._underlying.close()
865
+
866
+ def handleError(self, record):
867
+ self._underlying.handleError(record)
868
+
869
+
870
+ ##
871
+
872
+
873
+ class StandardLogHandler(ProxyLogHandler):
874
+ pass
875
+
876
+
877
+ ##
878
+
879
+
880
+ @contextlib.contextmanager
881
+ def _locking_logging_module_lock() -> ta.Iterator[None]:
882
+ if hasattr(logging, '_acquireLock'):
883
+ logging._acquireLock() # noqa
884
+ try:
885
+ yield
886
+ finally:
887
+ logging._releaseLock() # type: ignore # noqa
888
+
889
+ elif hasattr(logging, '_lock'):
890
+ # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
891
+ with logging._lock: # noqa
892
+ yield
893
+
894
+ else:
895
+ raise Exception("Can't find lock in logging module")
896
+
897
+
898
+ def configure_standard_logging(
899
+ level: ta.Union[int, str] = logging.INFO,
900
+ *,
901
+ json: bool = False,
902
+ target: ta.Optional[logging.Logger] = None,
903
+ force: bool = False,
904
+ handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
905
+ ) -> ta.Optional[StandardLogHandler]:
906
+ with _locking_logging_module_lock():
907
+ if target is None:
908
+ target = logging.root
909
+
910
+ #
911
+
912
+ if not force:
913
+ if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
914
+ return None
915
+
916
+ #
917
+
918
+ if handler_factory is not None:
919
+ handler = handler_factory()
920
+ else:
921
+ handler = logging.StreamHandler()
922
+
923
+ #
924
+
925
+ formatter: logging.Formatter
926
+ if json:
927
+ formatter = JsonLogFormatter()
928
+ else:
929
+ formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
930
+ handler.setFormatter(formatter)
931
+
932
+ #
933
+
934
+ handler.addFilter(TidLogFilter())
935
+
936
+ #
937
+
938
+ target.addHandler(handler)
939
+
940
+ #
941
+
942
+ if level is not None:
943
+ target.setLevel(level)
944
+
945
+ #
946
+
947
+ return StandardLogHandler(handler)
948
+
949
+
950
+ ########################################
951
+ # ../../../../omlish/lite/marshal.py
952
+ """
953
+ TODO:
954
+ - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
955
+ - nonstrict toggle
956
+ """
957
+
958
+
959
+ ##
960
+
961
+
962
+ class ObjMarshaler(abc.ABC):
963
+ @abc.abstractmethod
964
+ def marshal(self, o: ta.Any) -> ta.Any:
965
+ raise NotImplementedError
966
+
967
+ @abc.abstractmethod
968
+ def unmarshal(self, o: ta.Any) -> ta.Any:
969
+ raise NotImplementedError
970
+
971
+
972
+ class NopObjMarshaler(ObjMarshaler):
973
+ def marshal(self, o: ta.Any) -> ta.Any:
974
+ return o
975
+
976
+ def unmarshal(self, o: ta.Any) -> ta.Any:
977
+ return o
978
+
979
+
980
+ @dc.dataclass()
981
+ class ProxyObjMarshaler(ObjMarshaler):
982
+ m: ta.Optional[ObjMarshaler] = None
983
+
984
+ def marshal(self, o: ta.Any) -> ta.Any:
985
+ return check_not_none(self.m).marshal(o)
986
+
987
+ def unmarshal(self, o: ta.Any) -> ta.Any:
988
+ return check_not_none(self.m).unmarshal(o)
989
+
990
+
991
+ @dc.dataclass(frozen=True)
992
+ class CastObjMarshaler(ObjMarshaler):
993
+ ty: type
994
+
995
+ def marshal(self, o: ta.Any) -> ta.Any:
996
+ return o
997
+
998
+ def unmarshal(self, o: ta.Any) -> ta.Any:
999
+ return self.ty(o)
1000
+
1001
+
1002
+ class DynamicObjMarshaler(ObjMarshaler):
1003
+ def marshal(self, o: ta.Any) -> ta.Any:
1004
+ return marshal_obj(o)
1005
+
1006
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1007
+ return o
1008
+
1009
+
1010
+ @dc.dataclass(frozen=True)
1011
+ class Base64ObjMarshaler(ObjMarshaler):
1012
+ ty: type
1013
+
1014
+ def marshal(self, o: ta.Any) -> ta.Any:
1015
+ return base64.b64encode(o).decode('ascii')
1016
+
1017
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1018
+ return self.ty(base64.b64decode(o))
1019
+
1020
+
1021
+ @dc.dataclass(frozen=True)
1022
+ class EnumObjMarshaler(ObjMarshaler):
1023
+ ty: type
1024
+
1025
+ def marshal(self, o: ta.Any) -> ta.Any:
1026
+ return o.name
1027
+
1028
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1029
+ return self.ty.__members__[o] # type: ignore
1030
+
1031
+
1032
+ @dc.dataclass(frozen=True)
1033
+ class OptionalObjMarshaler(ObjMarshaler):
1034
+ item: ObjMarshaler
1035
+
1036
+ def marshal(self, o: ta.Any) -> ta.Any:
1037
+ if o is None:
1038
+ return None
1039
+ return self.item.marshal(o)
1040
+
1041
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1042
+ if o is None:
1043
+ return None
1044
+ return self.item.unmarshal(o)
1045
+
1046
+
1047
+ @dc.dataclass(frozen=True)
1048
+ class MappingObjMarshaler(ObjMarshaler):
1049
+ ty: type
1050
+ km: ObjMarshaler
1051
+ vm: ObjMarshaler
1052
+
1053
+ def marshal(self, o: ta.Any) -> ta.Any:
1054
+ return {self.km.marshal(k): self.vm.marshal(v) for k, v in o.items()}
1055
+
1056
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1057
+ return self.ty((self.km.unmarshal(k), self.vm.unmarshal(v)) for k, v in o.items())
1058
+
1059
+
1060
+ @dc.dataclass(frozen=True)
1061
+ class IterableObjMarshaler(ObjMarshaler):
1062
+ ty: type
1063
+ item: ObjMarshaler
1064
+
1065
+ def marshal(self, o: ta.Any) -> ta.Any:
1066
+ return [self.item.marshal(e) for e in o]
1067
+
1068
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1069
+ return self.ty(self.item.unmarshal(e) for e in o)
1070
+
1071
+
1072
+ @dc.dataclass(frozen=True)
1073
+ class DataclassObjMarshaler(ObjMarshaler):
1074
+ ty: type
1075
+ fs: ta.Mapping[str, ObjMarshaler]
1076
+ nonstrict: bool = False
1077
+
1078
+ def marshal(self, o: ta.Any) -> ta.Any:
1079
+ return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
1080
+
1081
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1082
+ return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
1083
+
1084
+
1085
+ @dc.dataclass(frozen=True)
1086
+ class PolymorphicObjMarshaler(ObjMarshaler):
1087
+ class Impl(ta.NamedTuple):
1088
+ ty: type
1089
+ tag: str
1090
+ m: ObjMarshaler
1091
+
1092
+ impls_by_ty: ta.Mapping[type, Impl]
1093
+ impls_by_tag: ta.Mapping[str, Impl]
1094
+
1095
+ @classmethod
1096
+ def of(cls, impls: ta.Iterable[Impl]) -> 'PolymorphicObjMarshaler':
1097
+ return cls(
1098
+ {i.ty: i for i in impls},
1099
+ {i.tag: i for i in impls},
1100
+ )
1101
+
1102
+ def marshal(self, o: ta.Any) -> ta.Any:
1103
+ impl = self.impls_by_ty[type(o)]
1104
+ return {impl.tag: impl.m.marshal(o)}
1105
+
1106
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1107
+ [(t, v)] = o.items()
1108
+ impl = self.impls_by_tag[t]
1109
+ return impl.m.unmarshal(v)
1110
+
1111
+
1112
+ @dc.dataclass(frozen=True)
1113
+ class DatetimeObjMarshaler(ObjMarshaler):
1114
+ ty: type
1115
+
1116
+ def marshal(self, o: ta.Any) -> ta.Any:
1117
+ return o.isoformat()
1118
+
1119
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1120
+ return self.ty.fromisoformat(o) # type: ignore
1121
+
1122
+
1123
+ class DecimalObjMarshaler(ObjMarshaler):
1124
+ def marshal(self, o: ta.Any) -> ta.Any:
1125
+ return str(check_isinstance(o, decimal.Decimal))
1126
+
1127
+ def unmarshal(self, v: ta.Any) -> ta.Any:
1128
+ return decimal.Decimal(check_isinstance(v, str))
1129
+
1130
+
1131
+ class FractionObjMarshaler(ObjMarshaler):
1132
+ def marshal(self, o: ta.Any) -> ta.Any:
1133
+ fr = check_isinstance(o, fractions.Fraction)
1134
+ return [fr.numerator, fr.denominator]
1135
+
1136
+ def unmarshal(self, v: ta.Any) -> ta.Any:
1137
+ num, denom = check_isinstance(v, list)
1138
+ return fractions.Fraction(num, denom)
1139
+
1140
+
1141
+ class UuidObjMarshaler(ObjMarshaler):
1142
+ def marshal(self, o: ta.Any) -> ta.Any:
1143
+ return str(o)
1144
+
1145
+ def unmarshal(self, o: ta.Any) -> ta.Any:
1146
+ return uuid.UUID(o)
1147
+
1148
+
1149
+ ##
1150
+
1151
+
1152
+ _DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
1153
+ **{t: NopObjMarshaler() for t in (type(None),)},
1154
+ **{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
1155
+ **{t: Base64ObjMarshaler(t) for t in (bytes, bytearray)},
1156
+ **{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
1157
+ **{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
1158
+
1159
+ ta.Any: DynamicObjMarshaler(),
1160
+
1161
+ **{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
1162
+ decimal.Decimal: DecimalObjMarshaler(),
1163
+ fractions.Fraction: FractionObjMarshaler(),
1164
+ uuid.UUID: UuidObjMarshaler(),
1165
+ }
1166
+
1167
+ _OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
1168
+ **{t: t for t in (dict,)},
1169
+ **{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
1170
+ }
1171
+
1172
+ _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
1173
+ **{t: t for t in (list, tuple, set, frozenset)},
1174
+ collections.abc.Set: frozenset,
1175
+ collections.abc.MutableSet: set,
1176
+ collections.abc.Sequence: tuple,
1177
+ collections.abc.MutableSequence: list,
1178
+ }
1179
+
1180
+
1181
+ def _make_obj_marshaler(
1182
+ ty: ta.Any,
1183
+ rec: ta.Callable[[ta.Any], ObjMarshaler],
1184
+ *,
1185
+ nonstrict_dataclasses: bool = False,
1186
+ ) -> ObjMarshaler:
1187
+ if isinstance(ty, type):
1188
+ if abc.ABC in ty.__bases__:
1189
+ return PolymorphicObjMarshaler.of([ # type: ignore
1190
+ PolymorphicObjMarshaler.Impl(
1191
+ ity,
1192
+ ity.__qualname__,
1193
+ rec(ity),
1194
+ )
1195
+ for ity in deep_subclasses(ty)
1196
+ if abc.ABC not in ity.__bases__
1197
+ ])
1198
+
1199
+ if issubclass(ty, enum.Enum):
1200
+ return EnumObjMarshaler(ty)
1201
+
1202
+ if dc.is_dataclass(ty):
1203
+ return DataclassObjMarshaler(
1204
+ ty,
1205
+ {f.name: rec(f.type) for f in dc.fields(ty)},
1206
+ nonstrict=nonstrict_dataclasses,
1207
+ )
1208
+
1209
+ if is_generic_alias(ty):
1210
+ try:
1211
+ mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
1212
+ except KeyError:
1213
+ pass
1214
+ else:
1215
+ k, v = ta.get_args(ty)
1216
+ return MappingObjMarshaler(mt, rec(k), rec(v))
1217
+
1218
+ try:
1219
+ st = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES[ta.get_origin(ty)]
1220
+ except KeyError:
1221
+ pass
1222
+ else:
1223
+ [e] = ta.get_args(ty)
1224
+ return IterableObjMarshaler(st, rec(e))
1225
+
1226
+ if is_union_alias(ty):
1227
+ return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
1228
+
1229
+ raise TypeError(ty)
1230
+
1231
+
1232
+ ##
1233
+
1234
+
1235
+ _OBJ_MARSHALERS_LOCK = threading.RLock()
1236
+
1237
+ _OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
1238
+
1239
+ _OBJ_MARSHALER_PROXIES: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
1240
+
1241
+
1242
+ def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
1243
+ with _OBJ_MARSHALERS_LOCK:
1244
+ if ty in _OBJ_MARSHALERS:
1245
+ raise KeyError(ty)
1246
+ _OBJ_MARSHALERS[ty] = m
1247
+
1248
+
1249
+ def get_obj_marshaler(
1250
+ ty: ta.Any,
1251
+ *,
1252
+ no_cache: bool = False,
1253
+ **kwargs: ta.Any,
1254
+ ) -> ObjMarshaler:
1255
+ with _OBJ_MARSHALERS_LOCK:
1256
+ if not no_cache:
1257
+ try:
1258
+ return _OBJ_MARSHALERS[ty]
1259
+ except KeyError:
1260
+ pass
1261
+
1262
+ try:
1263
+ return _OBJ_MARSHALER_PROXIES[ty]
1264
+ except KeyError:
1265
+ pass
1266
+
1267
+ rec = functools.partial(
1268
+ get_obj_marshaler,
1269
+ no_cache=no_cache,
1270
+ **kwargs,
1271
+ )
1272
+
1273
+ p = ProxyObjMarshaler()
1274
+ _OBJ_MARSHALER_PROXIES[ty] = p
1275
+ try:
1276
+ m = _make_obj_marshaler(ty, rec, **kwargs)
1277
+ finally:
1278
+ del _OBJ_MARSHALER_PROXIES[ty]
1279
+ p.m = m
1280
+
1281
+ if not no_cache:
1282
+ _OBJ_MARSHALERS[ty] = m
1283
+ return m
1284
+
1285
+
1286
+ ##
1287
+
1288
+
1289
+ def marshal_obj(o: ta.Any, ty: ta.Any = None) -> ta.Any:
1290
+ return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
1291
+
1292
+
1293
+ def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
1294
+ return get_obj_marshaler(ty).unmarshal(o)
1295
+
1296
+
1297
+ ########################################
1298
+ # ../../../../omlish/lite/runtime.py
1299
+
1300
+
1301
+ @cached_nullary
1302
+ def is_debugger_attached() -> bool:
1303
+ return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
1304
+
1305
+
1306
+ REQUIRED_PYTHON_VERSION = (3, 8)
1307
+
1308
+
1309
+ def check_runtime_version() -> None:
1310
+ if sys.version_info < REQUIRED_PYTHON_VERSION:
1311
+ raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
1312
+
1313
+
1314
+ ########################################
1315
+ # ../../../../omlish/lite/subprocesses.py
1316
+
1317
+
1318
+ ##
1319
+
1320
+
1321
+ _SUBPROCESS_SHELL_WRAP_EXECS = False
1322
+
1323
+
1324
+ def subprocess_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
1325
+ return ('sh', '-c', ' '.join(map(shlex.quote, args)))
1326
+
1327
+
1328
+ def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
1329
+ if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
1330
+ return subprocess_shell_wrap_exec(*args)
1331
+ else:
1332
+ return args
1333
+
1334
+
1335
+ def _prepare_subprocess_invocation(
1336
+ *args: str,
1337
+ env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
1338
+ extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
1339
+ quiet: bool = False,
1340
+ shell: bool = False,
1341
+ **kwargs: ta.Any,
1342
+ ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
1343
+ log.debug(args)
1344
+ if extra_env:
1345
+ log.debug(extra_env)
1346
+
1347
+ if extra_env:
1348
+ env = {**(env if env is not None else os.environ), **extra_env}
1349
+
1350
+ if quiet and 'stderr' not in kwargs:
1351
+ if not log.isEnabledFor(logging.DEBUG):
1352
+ kwargs['stderr'] = subprocess.DEVNULL
1353
+
1354
+ if not shell:
1355
+ args = subprocess_maybe_shell_wrap_exec(*args)
1356
+
1357
+ return args, dict(
1358
+ env=env,
1359
+ shell=shell,
1360
+ **kwargs,
1361
+ )
1362
+
1363
+
1364
+ def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
1365
+ args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
1366
+ return subprocess.check_call(args, **kwargs) # type: ignore
1367
+
1368
+
1369
+ def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
1370
+ args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
1371
+ return subprocess.check_output(args, **kwargs)
1372
+
1373
+
1374
+ def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
1375
+ return subprocess_check_output(*args, **kwargs).decode().strip()
1376
+
1377
+
1378
+ ##
1379
+
1380
+
1381
+ DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
1382
+ FileNotFoundError,
1383
+ subprocess.CalledProcessError,
1384
+ )
1385
+
1386
+
1387
+ def subprocess_try_call(
1388
+ *args: str,
1389
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
1390
+ **kwargs: ta.Any,
1391
+ ) -> bool:
1392
+ try:
1393
+ subprocess_check_call(*args, **kwargs)
1394
+ except try_exceptions as e: # noqa
1395
+ if log.isEnabledFor(logging.DEBUG):
1396
+ log.exception('command failed')
1397
+ return False
1398
+ else:
1399
+ return True
1400
+
1401
+
1402
+ def subprocess_try_output(
1403
+ *args: str,
1404
+ try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
1405
+ **kwargs: ta.Any,
1406
+ ) -> ta.Optional[bytes]:
1407
+ try:
1408
+ return subprocess_check_output(*args, **kwargs)
1409
+ except try_exceptions as e: # noqa
1410
+ if log.isEnabledFor(logging.DEBUG):
1411
+ log.exception('command failed')
1412
+ return None
1413
+
1414
+
1415
+ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
1416
+ out = subprocess_try_output(*args, **kwargs)
1417
+ return out.decode().strip() if out is not None else None
1418
+
1419
+
1420
+ ##
1421
+
1422
+
1423
+ def subprocess_close(
1424
+ proc: subprocess.Popen,
1425
+ timeout: ta.Optional[float] = None,
1426
+ ) -> None:
1427
+ # TODO: terminate, sleep, kill
1428
+ if proc.stdout:
1429
+ proc.stdout.close()
1430
+ if proc.stderr:
1431
+ proc.stderr.close()
1432
+ if proc.stdin:
1433
+ proc.stdin.close()
1434
+
1435
+ proc.wait(timeout)
1436
+
1437
+
1438
+ ########################################
1439
+ # ../commands/subprocess.py
1440
+
1441
+
1442
+ ##
1443
+
1444
+
1445
+ class SubprocessCommand(Command['SubprocessCommand.Input', 'SubprocessCommand.Output']):
1446
+ @dc.dataclass(frozen=True)
1447
+ class Input(Command.Input):
1448
+ args: ta.Sequence[str]
1449
+
1450
+ shell: bool = False
1451
+ cwd: ta.Optional[str] = None
1452
+ env: ta.Optional[ta.Mapping[str, str]] = None
1453
+
1454
+ capture_stdout: bool = False
1455
+ capture_stderr: bool = False
1456
+
1457
+ input: ta.Optional[bytes] = None
1458
+ timeout: ta.Optional[float] = None
1459
+
1460
+ def __post_init__(self) -> None:
1461
+ if isinstance(self.args, str):
1462
+ raise TypeError(self.args)
1463
+
1464
+ @dc.dataclass(frozen=True)
1465
+ class Output(Command.Output):
1466
+ rc: int
1467
+ pid: int
1468
+
1469
+ elapsed_s: float
1470
+
1471
+ stdout: ta.Optional[bytes] = None
1472
+ stderr: ta.Optional[bytes] = None
1473
+
1474
+ def _execute(self, inp: Input) -> Output:
1475
+ proc = subprocess.Popen(
1476
+ subprocess_maybe_shell_wrap_exec(*inp.args),
1477
+
1478
+ shell=inp.shell,
1479
+ cwd=inp.cwd,
1480
+ env={**os.environ, **(inp.env or {})},
1481
+
1482
+ stdin=subprocess.PIPE if inp.input is not None else None,
1483
+ stdout=subprocess.PIPE if inp.capture_stdout else None,
1484
+ stderr=subprocess.PIPE if inp.capture_stderr else None,
1485
+ )
1486
+
1487
+ start_time = time.time()
1488
+ stdout, stderr = proc.communicate(
1489
+ input=inp.input,
1490
+ timeout=inp.timeout,
1491
+ )
1492
+ end_time = time.time()
1493
+
1494
+ return SubprocessCommand.Output(
1495
+ rc=proc.returncode,
1496
+ pid=proc.pid,
1497
+
1498
+ elapsed_s=end_time - start_time,
1499
+
1500
+ stdout=stdout, # noqa
1501
+ stderr=stderr, # noqa
1502
+ )
1503
+
1504
+
1505
+ ########################################
1506
+ # main.py
1507
+
1508
+
1509
+ ##
1510
+
1511
+
1512
+ def _send_obj(f: ta.IO, o: ta.Any) -> None:
1513
+ j = json_dumps_compact(marshal_obj(o))
1514
+ d = j.encode('utf-8')
1515
+
1516
+ f.write(struct.pack('<I', len(d)))
1517
+ f.write(d)
1518
+ f.flush()
1519
+
1520
+
1521
+ def _recv_obj(f: ta.IO, ty: type) -> ta.Any:
1522
+ d = f.read(4)
1523
+ if not d:
1524
+ return None
1525
+ if len(d) != 4:
1526
+ raise Exception
1527
+
1528
+ sz = struct.unpack('<I', d)[0]
1529
+ d = f.read(sz)
1530
+ if not d:
1531
+ raise Exception
1532
+
1533
+ j = json.loads(d.decode('utf-8'))
1534
+ return unmarshal_obj(j, ty)
1535
+
1536
+
1537
+ def _remote_main() -> None:
1538
+ rt = pyremote_bootstrap_finalize() # noqa
1539
+
1540
+ while True:
1541
+ i = _recv_obj(rt.input, SubprocessCommand.Input)
1542
+ if i is None:
1543
+ break
1544
+
1545
+ o = SubprocessCommand()._execute(i) # noqa
1546
+
1547
+ _send_obj(rt.output, o)
1548
+
1549
+
1550
+ def _main() -> None:
1551
+ import argparse
1552
+
1553
+ parser = argparse.ArgumentParser()
1554
+
1555
+ parser.add_argument('-s', '--shell')
1556
+ parser.add_argument('-q', '--shell-quote', action='store_true')
1557
+ parser.add_argument('--python', default='python3')
1558
+ parser.add_argument('--_amalg-file')
1559
+
1560
+ args = parser.parse_args()
1561
+
1562
+ #
1563
+
1564
+ self_src = inspect.getsource(sys.modules[__name__])
1565
+ self_src_lines = self_src.splitlines()
1566
+ for l in self_src_lines:
1567
+ if l.startswith('# @omlish-amalg-output '):
1568
+ is_self_amalg = True
1569
+ break
1570
+ else:
1571
+ is_self_amalg = False
1572
+
1573
+ if is_self_amalg:
1574
+ amalg_src = self_src
1575
+ else:
1576
+ amalg_file = args._amalg_file # noqa
1577
+ if amalg_file is None:
1578
+ amalg_file = os.path.join(os.path.dirname(__file__), '_manage.py')
1579
+ with open(amalg_file) as f:
1580
+ amalg_src = f.read()
1581
+
1582
+ #
1583
+
1584
+ remote_src = '\n\n'.join([
1585
+ '__name__ = "__remote__"',
1586
+ amalg_src,
1587
+ '_remote_main()',
1588
+ ])
1589
+
1590
+ #
1591
+
1592
+ bs_src = pyremote_build_bootstrap_cmd(__package__ or 'manage')
1593
+
1594
+ if args.shell is not None:
1595
+ sh_src = f'{args.python} -c {shlex.quote(bs_src)}'
1596
+ if args.shell_quote:
1597
+ sh_src = shlex.quote(sh_src)
1598
+ sh_cmd = f'{args.shell} {sh_src}'
1599
+ cmd = [sh_cmd]
1600
+ shell = True
1601
+ else:
1602
+ cmd = [args.python, '-c', bs_src]
1603
+ shell = False
1604
+
1605
+ proc = subprocess.Popen(
1606
+ subprocess_maybe_shell_wrap_exec(*cmd),
1607
+ shell=shell,
1608
+ stdin=subprocess.PIPE,
1609
+ stdout=subprocess.PIPE,
1610
+ )
1611
+
1612
+ stdin = check_not_none(proc.stdin)
1613
+ stdout = check_not_none(proc.stdout)
1614
+
1615
+ res = PyremoteBootstrapDriver(remote_src).run(stdin, stdout)
1616
+ print(res)
1617
+
1618
+ #
1619
+
1620
+ for ci in [
1621
+ SubprocessCommand.Input(
1622
+ args=['python3', '-'],
1623
+ input=b'print(1)\n',
1624
+ capture_stdout=True,
1625
+ ),
1626
+ SubprocessCommand.Input(
1627
+ args=['uname'],
1628
+ capture_stdout=True,
1629
+ ),
1630
+ ]:
1631
+ _send_obj(stdin, ci)
1632
+
1633
+ o = _recv_obj(stdout, SubprocessCommand.Output)
1634
+
1635
+ print(o)
1636
+
1637
+ try:
1638
+ stdin.close()
1639
+ except BrokenPipeError:
1640
+ pass
1641
+
1642
+ proc.wait()
1643
+
1644
+
1645
+ if __name__ == '__main__':
1646
+ _main()