ominfra 0.0.0.dev141__py3-none-any.whl → 0.0.0.dev143__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/manage/__init__.py +10 -0
- ominfra/manage/__main__.py +4 -0
- ominfra/manage/bootstrap.py +11 -0
- ominfra/manage/bootstrap_.py +18 -0
- ominfra/manage/commands/base.py +133 -1
- ominfra/manage/commands/execution.py +23 -0
- ominfra/manage/commands/inject.py +122 -0
- ominfra/manage/commands/marshal.py +26 -0
- ominfra/manage/commands/subprocess.py +18 -17
- ominfra/manage/config.py +10 -0
- ominfra/manage/inject.py +55 -0
- ominfra/manage/main.py +54 -132
- ominfra/manage/marshal.py +12 -0
- ominfra/manage/remote/__init__.py +0 -0
- ominfra/manage/remote/channel.py +52 -0
- ominfra/manage/remote/config.py +12 -0
- ominfra/manage/remote/execution.py +193 -0
- ominfra/manage/remote/inject.py +29 -0
- ominfra/manage/{payload.py → remote/payload.py} +7 -1
- ominfra/manage/{spawning.py → remote/spawning.py} +31 -35
- ominfra/pyremote.py +53 -14
- ominfra/scripts/journald2aws.py +216 -131
- ominfra/scripts/manage.py +2180 -452
- ominfra/scripts/supervisor.py +203 -130
- ominfra/supervisor/inject.py +2 -1
- {ominfra-0.0.0.dev141.dist-info → ominfra-0.0.0.dev143.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev141.dist-info → ominfra-0.0.0.dev143.dist-info}/RECORD +31 -17
- {ominfra-0.0.0.dev141.dist-info → ominfra-0.0.0.dev143.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev141.dist-info → ominfra-0.0.0.dev143.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev141.dist-info → ominfra-0.0.0.dev143.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev141.dist-info → ominfra-0.0.0.dev143.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -31,6 +31,7 @@ import subprocess
|
|
31
31
|
import sys
|
32
32
|
import threading
|
33
33
|
import time
|
34
|
+
import traceback
|
34
35
|
import types
|
35
36
|
import typing as ta
|
36
37
|
import uuid
|
@@ -48,10 +49,6 @@ if sys.version_info < (3, 8):
|
|
48
49
|
########################################
|
49
50
|
|
50
51
|
|
51
|
-
# commands/base.py
|
52
|
-
CommandT = ta.TypeVar('CommandT', bound='Command')
|
53
|
-
CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
|
54
|
-
|
55
52
|
# ../../omlish/lite/cached.py
|
56
53
|
T = ta.TypeVar('T')
|
57
54
|
CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
@@ -59,34 +56,39 @@ CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
|
59
56
|
# ../../omlish/lite/check.py
|
60
57
|
SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
|
61
58
|
|
59
|
+
# commands/base.py
|
60
|
+
CommandT = ta.TypeVar('CommandT', bound='Command')
|
61
|
+
CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
|
62
62
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
63
|
+
# ../../omlish/lite/inject.py
|
64
|
+
U = ta.TypeVar('U')
|
65
|
+
InjectorKeyCls = ta.Union[type, ta.NewType]
|
66
|
+
InjectorProviderFn = ta.Callable[['Injector'], ta.Any]
|
67
|
+
InjectorProviderFnMap = ta.Mapping['InjectorKey', 'InjectorProviderFn']
|
68
|
+
InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
|
68
69
|
|
70
|
+
# ../../omlish/lite/subprocesses.py
|
71
|
+
SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']
|
69
72
|
|
70
|
-
@dc.dataclass(frozen=True)
|
71
|
-
class Command(abc.ABC, ta.Generic[CommandOutputT]):
|
72
|
-
@dc.dataclass(frozen=True)
|
73
|
-
class Output(abc.ABC): # noqa
|
74
|
-
pass
|
75
73
|
|
74
|
+
########################################
|
75
|
+
# ../config.py
|
76
76
|
|
77
|
-
##
|
78
77
|
|
78
|
+
@dc.dataclass(frozen=True)
|
79
|
+
class MainConfig:
|
80
|
+
log_level: ta.Optional[str] = 'INFO'
|
79
81
|
|
80
|
-
|
81
|
-
@abc.abstractmethod
|
82
|
-
def execute(self, i: CommandT) -> CommandOutputT:
|
83
|
-
raise NotImplementedError
|
82
|
+
debug: bool = False
|
84
83
|
|
85
84
|
|
86
85
|
########################################
|
87
86
|
# ../../pyremote.py
|
88
87
|
"""
|
89
88
|
Basically this: https://mitogen.networkgenomics.com/howitworks.html
|
89
|
+
|
90
|
+
TODO:
|
91
|
+
- log: ta.Optional[logging.Logger] = None + log.debug's
|
90
92
|
"""
|
91
93
|
|
92
94
|
|
@@ -97,6 +99,9 @@ Basically this: https://mitogen.networkgenomics.com/howitworks.html
|
|
97
99
|
class PyremoteBootstrapOptions:
|
98
100
|
debug: bool = False
|
99
101
|
|
102
|
+
DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
|
103
|
+
main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
|
104
|
+
|
100
105
|
|
101
106
|
##
|
102
107
|
|
@@ -251,12 +256,14 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
251
256
|
os.close(f)
|
252
257
|
|
253
258
|
# Save vars
|
254
|
-
os.environ
|
255
|
-
|
256
|
-
|
259
|
+
env = os.environ
|
260
|
+
exe = sys.executable
|
261
|
+
env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
|
262
|
+
env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
|
263
|
+
env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
|
257
264
|
|
258
265
|
# Start repl reading stdin from r0
|
259
|
-
os.execl(
|
266
|
+
os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
|
260
267
|
|
261
268
|
else:
|
262
269
|
# Child process
|
@@ -320,12 +327,12 @@ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
|
|
320
327
|
if cl.strip()
|
321
328
|
)
|
322
329
|
|
323
|
-
bs_z = zlib.compress(bs_src.encode('utf-8'))
|
324
|
-
|
330
|
+
bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
|
331
|
+
bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
|
325
332
|
|
326
333
|
stmts = [
|
327
334
|
f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
|
328
|
-
f'exec(zlib.decompress(base64.
|
335
|
+
f'exec(zlib.decompress(base64.b85decode({bs_z85!r})))',
|
329
336
|
f'_pyremote_bootstrap_main({context_name!r})',
|
330
337
|
]
|
331
338
|
|
@@ -411,6 +418,10 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
411
418
|
os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
|
412
419
|
os.close(nfd)
|
413
420
|
|
421
|
+
if (mn := options.main_name_override) is not None:
|
422
|
+
# Inspections like typing.get_type_hints need an entry in sys.modules.
|
423
|
+
sys.modules[mn] = sys.modules['__main__']
|
424
|
+
|
414
425
|
# Write fourth ack
|
415
426
|
output.write(_PYREMOTE_BOOTSTRAP_ACK3)
|
416
427
|
|
@@ -429,14 +440,41 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
429
440
|
|
430
441
|
|
431
442
|
class PyremoteBootstrapDriver:
|
432
|
-
def __init__(
|
443
|
+
def __init__(
|
444
|
+
self,
|
445
|
+
main_src: ta.Union[str, ta.Sequence[str]],
|
446
|
+
options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
|
447
|
+
) -> None:
|
433
448
|
super().__init__()
|
434
449
|
|
435
450
|
self._main_src = main_src
|
436
|
-
self._main_z = zlib.compress(main_src.encode('utf-8'))
|
437
|
-
|
438
451
|
self._options = options
|
452
|
+
|
453
|
+
self._prepared_main_src = self._prepare_main_src(main_src, options)
|
454
|
+
self._main_z = zlib.compress(self._prepared_main_src.encode('utf-8'))
|
455
|
+
|
439
456
|
self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
|
457
|
+
#
|
458
|
+
|
459
|
+
@classmethod
|
460
|
+
def _prepare_main_src(
|
461
|
+
cls,
|
462
|
+
main_src: ta.Union[str, ta.Sequence[str]],
|
463
|
+
options: PyremoteBootstrapOptions,
|
464
|
+
) -> str:
|
465
|
+
parts: ta.List[str]
|
466
|
+
if isinstance(main_src, str):
|
467
|
+
parts = [main_src]
|
468
|
+
else:
|
469
|
+
parts = list(main_src)
|
470
|
+
|
471
|
+
if (mn := options.main_name_override) is not None:
|
472
|
+
parts.insert(0, f'__name__ = {mn!r}')
|
473
|
+
|
474
|
+
if len(parts) == 1:
|
475
|
+
return parts[0]
|
476
|
+
else:
|
477
|
+
return '\n\n'.join(parts)
|
440
478
|
|
441
479
|
#
|
442
480
|
|
@@ -514,7 +552,7 @@ class PyremoteBootstrapDriver:
|
|
514
552
|
|
515
553
|
#
|
516
554
|
|
517
|
-
def run(self,
|
555
|
+
def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
|
518
556
|
gen = self.gen()
|
519
557
|
|
520
558
|
gi: ta.Optional[bytes] = None
|
@@ -528,12 +566,12 @@ class PyremoteBootstrapDriver:
|
|
528
566
|
return e.value
|
529
567
|
|
530
568
|
if isinstance(go, self.Read):
|
531
|
-
if len(gi :=
|
569
|
+
if len(gi := input.read(go.sz)) != go.sz:
|
532
570
|
raise EOFError
|
533
571
|
elif isinstance(go, self.Write):
|
534
572
|
gi = None
|
535
|
-
|
536
|
-
|
573
|
+
output.write(go.d)
|
574
|
+
output.flush()
|
537
575
|
else:
|
538
576
|
raise TypeError(go)
|
539
577
|
|
@@ -697,6 +735,99 @@ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON
|
|
697
735
|
json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
|
698
736
|
|
699
737
|
|
738
|
+
########################################
|
739
|
+
# ../../../omlish/lite/maybes.py
|
740
|
+
|
741
|
+
|
742
|
+
class Maybe(ta.Generic[T]):
|
743
|
+
@property
|
744
|
+
@abc.abstractmethod
|
745
|
+
def present(self) -> bool:
|
746
|
+
raise NotImplementedError
|
747
|
+
|
748
|
+
@abc.abstractmethod
|
749
|
+
def must(self) -> T:
|
750
|
+
raise NotImplementedError
|
751
|
+
|
752
|
+
@classmethod
|
753
|
+
def just(cls, v: T) -> 'Maybe[T]':
|
754
|
+
return tuple.__new__(_Maybe, (v,)) # noqa
|
755
|
+
|
756
|
+
_empty: ta.ClassVar['Maybe']
|
757
|
+
|
758
|
+
@classmethod
|
759
|
+
def empty(cls) -> 'Maybe[T]':
|
760
|
+
return Maybe._empty
|
761
|
+
|
762
|
+
|
763
|
+
class _Maybe(Maybe[T], tuple):
|
764
|
+
__slots__ = ()
|
765
|
+
|
766
|
+
def __init_subclass__(cls, **kwargs):
|
767
|
+
raise TypeError
|
768
|
+
|
769
|
+
@property
|
770
|
+
def present(self) -> bool:
|
771
|
+
return bool(self)
|
772
|
+
|
773
|
+
def must(self) -> T:
|
774
|
+
if not self:
|
775
|
+
raise ValueError
|
776
|
+
return self[0]
|
777
|
+
|
778
|
+
|
779
|
+
Maybe._empty = tuple.__new__(_Maybe, ()) # noqa
|
780
|
+
|
781
|
+
|
782
|
+
########################################
|
783
|
+
# ../../../omlish/lite/pycharm.py
|
784
|
+
|
785
|
+
|
786
|
+
DEFAULT_PYCHARM_VERSION = '242.23726.102'
|
787
|
+
|
788
|
+
|
789
|
+
@dc.dataclass(frozen=True)
|
790
|
+
class PycharmRemoteDebug:
|
791
|
+
port: int
|
792
|
+
host: ta.Optional[str] = 'localhost'
|
793
|
+
install_version: ta.Optional[str] = DEFAULT_PYCHARM_VERSION
|
794
|
+
|
795
|
+
|
796
|
+
def pycharm_debug_connect(prd: PycharmRemoteDebug) -> None:
|
797
|
+
if prd.install_version is not None:
|
798
|
+
import subprocess
|
799
|
+
import sys
|
800
|
+
subprocess.check_call([
|
801
|
+
sys.executable,
|
802
|
+
'-mpip',
|
803
|
+
'install',
|
804
|
+
f'pydevd-pycharm~={prd.install_version}',
|
805
|
+
])
|
806
|
+
|
807
|
+
pydevd_pycharm = __import__('pydevd_pycharm') # noqa
|
808
|
+
pydevd_pycharm.settrace(
|
809
|
+
prd.host,
|
810
|
+
port=prd.port,
|
811
|
+
stdoutToServer=True,
|
812
|
+
stderrToServer=True,
|
813
|
+
)
|
814
|
+
|
815
|
+
|
816
|
+
def pycharm_debug_preamble(prd: PycharmRemoteDebug) -> str:
|
817
|
+
import inspect
|
818
|
+
import textwrap
|
819
|
+
|
820
|
+
return textwrap.dedent(f"""
|
821
|
+
{inspect.getsource(pycharm_debug_connect)}
|
822
|
+
|
823
|
+
pycharm_debug_connect(PycharmRemoteDebug(
|
824
|
+
{prd.port!r},
|
825
|
+
host={prd.host!r},
|
826
|
+
install_version={prd.install_version!r},
|
827
|
+
))
|
828
|
+
""")
|
829
|
+
|
830
|
+
|
700
831
|
########################################
|
701
832
|
# ../../../omlish/lite/reflect.py
|
702
833
|
|
@@ -753,208 +884,1335 @@ def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
|
|
753
884
|
|
754
885
|
|
755
886
|
########################################
|
756
|
-
#
|
887
|
+
# ../../../omlish/lite/strings.py
|
757
888
|
|
758
889
|
|
759
|
-
|
760
|
-
def _get_self_src() -> str:
|
761
|
-
return inspect.getsource(sys.modules[__name__])
|
890
|
+
##
|
762
891
|
|
763
892
|
|
764
|
-
def
|
765
|
-
|
766
|
-
|
767
|
-
|
768
|
-
|
893
|
+
def camel_case(name: str, lower: bool = False) -> str:
|
894
|
+
if not name:
|
895
|
+
return ''
|
896
|
+
s = ''.join(map(str.capitalize, name.split('_'))) # noqa
|
897
|
+
if lower:
|
898
|
+
s = s[0].lower() + s[1:]
|
899
|
+
return s
|
769
900
|
|
770
901
|
|
771
|
-
|
772
|
-
|
773
|
-
return
|
902
|
+
def snake_case(name: str) -> str:
|
903
|
+
uppers: list[int | None] = [i for i, c in enumerate(name) if c.isupper()]
|
904
|
+
return '_'.join([name[l:r].lower() for l, r in zip([None, *uppers], [*uppers, None])]).strip('_')
|
774
905
|
|
775
906
|
|
776
|
-
|
777
|
-
if file is not None:
|
778
|
-
with open(file) as f:
|
779
|
-
return f.read()
|
907
|
+
##
|
780
908
|
|
781
|
-
if _is_self_amalg():
|
782
|
-
return _get_self_src()
|
783
909
|
|
784
|
-
|
785
|
-
return
|
910
|
+
def is_dunder(name: str) -> bool:
|
911
|
+
return (
|
912
|
+
name[:2] == name[-2:] == '__' and
|
913
|
+
name[2:3] != '_' and
|
914
|
+
name[-3:-2] != '_' and
|
915
|
+
len(name) > 4
|
916
|
+
)
|
786
917
|
|
787
918
|
|
788
|
-
|
789
|
-
|
790
|
-
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
919
|
+
def is_sunder(name: str) -> bool:
|
920
|
+
return (
|
921
|
+
name[0] == name[-1] == '_' and
|
922
|
+
name[1:2] != '_' and
|
923
|
+
name[-2:-1] != '_' and
|
924
|
+
len(name) > 2
|
925
|
+
)
|
795
926
|
|
796
927
|
|
797
|
-
|
928
|
+
##
|
929
|
+
|
930
|
+
|
931
|
+
def attr_repr(obj: ta.Any, *attrs: str) -> str:
|
932
|
+
return f'{type(obj).__name__}({", ".join(f"{attr}={getattr(obj, attr)!r}" for attr in attrs)})'
|
798
933
|
|
799
934
|
|
800
935
|
##
|
801
936
|
|
802
937
|
|
803
|
-
|
938
|
+
FORMAT_NUM_BYTES_SUFFIXES: ta.Sequence[str] = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB']
|
804
939
|
|
805
|
-
def filter(self, record):
|
806
|
-
record.tid = threading.get_native_id()
|
807
|
-
return True
|
808
940
|
|
941
|
+
def format_num_bytes(num_bytes: int) -> str:
|
942
|
+
for i, suffix in enumerate(FORMAT_NUM_BYTES_SUFFIXES):
|
943
|
+
value = num_bytes / 1024 ** i
|
944
|
+
if num_bytes < 1024 ** (i + 1):
|
945
|
+
if value.is_integer():
|
946
|
+
return f'{int(value)}{suffix}'
|
947
|
+
else:
|
948
|
+
return f'{value:.2f}{suffix}'
|
809
949
|
|
810
|
-
|
950
|
+
return f'{num_bytes / 1024 ** (len(FORMAT_NUM_BYTES_SUFFIXES) - 1):.2f}{FORMAT_NUM_BYTES_SUFFIXES[-1]}'
|
811
951
|
|
812
952
|
|
813
|
-
|
953
|
+
########################################
|
954
|
+
# ../commands/base.py
|
814
955
|
|
815
|
-
KEYS: ta.Mapping[str, bool] = {
|
816
|
-
'name': False,
|
817
|
-
'msg': False,
|
818
|
-
'args': False,
|
819
|
-
'levelname': False,
|
820
|
-
'levelno': False,
|
821
|
-
'pathname': False,
|
822
|
-
'filename': False,
|
823
|
-
'module': False,
|
824
|
-
'exc_info': True,
|
825
|
-
'exc_text': True,
|
826
|
-
'stack_info': True,
|
827
|
-
'lineno': False,
|
828
|
-
'funcName': False,
|
829
|
-
'created': False,
|
830
|
-
'msecs': False,
|
831
|
-
'relativeCreated': False,
|
832
|
-
'thread': False,
|
833
|
-
'threadName': False,
|
834
|
-
'processName': False,
|
835
|
-
'process': False,
|
836
|
-
}
|
837
956
|
|
838
|
-
|
839
|
-
|
840
|
-
|
841
|
-
|
842
|
-
|
843
|
-
|
844
|
-
|
845
|
-
|
957
|
+
##
|
958
|
+
|
959
|
+
|
960
|
+
@dc.dataclass(frozen=True)
|
961
|
+
class Command(abc.ABC, ta.Generic[CommandOutputT]):
|
962
|
+
@dc.dataclass(frozen=True)
|
963
|
+
class Output(abc.ABC): # noqa
|
964
|
+
pass
|
965
|
+
|
966
|
+
@ta.final
|
967
|
+
def execute(self, executor: 'CommandExecutor') -> CommandOutputT:
|
968
|
+
return check_isinstance(executor.execute(self), self.Output) # type: ignore[return-value]
|
846
969
|
|
847
970
|
|
848
971
|
##
|
849
972
|
|
850
973
|
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
|
855
|
-
('levelname', '%(levelname)s'),
|
856
|
-
('name', '%(name)s'),
|
857
|
-
('separator', '::'),
|
858
|
-
('message', '%(message)s'),
|
859
|
-
]
|
974
|
+
@dc.dataclass(frozen=True)
|
975
|
+
class CommandException:
|
976
|
+
name: str
|
977
|
+
repr: str
|
860
978
|
|
979
|
+
traceback: ta.Optional[str] = None
|
861
980
|
|
862
|
-
|
981
|
+
exc: ta.Optional[ta.Any] = None # Exception
|
863
982
|
|
864
|
-
|
865
|
-
def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
|
866
|
-
return ' '.join(v for k, v in parts)
|
983
|
+
cmd: ta.Optional[Command] = None
|
867
984
|
|
868
|
-
|
985
|
+
@classmethod
|
986
|
+
def of(
|
987
|
+
cls,
|
988
|
+
exc: Exception,
|
989
|
+
*,
|
990
|
+
omit_exc_object: bool = False,
|
869
991
|
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
|
875
|
-
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
876
|
-
return '%s.%03d' % (t, record.msecs) # noqa
|
992
|
+
cmd: ta.Optional[Command] = None,
|
993
|
+
) -> 'CommandException':
|
994
|
+
return CommandException(
|
995
|
+
name=type(exc).__qualname__,
|
996
|
+
repr=repr(exc),
|
877
997
|
|
998
|
+
traceback=(
|
999
|
+
''.join(traceback.format_tb(exc.__traceback__))
|
1000
|
+
if getattr(exc, '__traceback__', None) is not None else None
|
1001
|
+
),
|
878
1002
|
|
879
|
-
|
1003
|
+
exc=None if omit_exc_object else exc,
|
880
1004
|
|
1005
|
+
cmd=cmd,
|
1006
|
+
)
|
881
1007
|
|
882
|
-
class ProxyLogFilterer(logging.Filterer):
|
883
|
-
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
884
|
-
self._underlying = underlying
|
885
1008
|
|
1009
|
+
class CommandOutputOrException(abc.ABC, ta.Generic[CommandOutputT]):
|
886
1010
|
@property
|
887
|
-
|
888
|
-
|
1011
|
+
@abc.abstractmethod
|
1012
|
+
def output(self) -> ta.Optional[CommandOutputT]:
|
1013
|
+
raise NotImplementedError
|
889
1014
|
|
890
1015
|
@property
|
891
|
-
|
892
|
-
|
1016
|
+
@abc.abstractmethod
|
1017
|
+
def exception(self) -> ta.Optional[CommandException]:
|
1018
|
+
raise NotImplementedError
|
893
1019
|
|
894
|
-
@filters.setter
|
895
|
-
def filters(self, filters):
|
896
|
-
self._underlying.filters = filters
|
897
1020
|
|
898
|
-
|
899
|
-
|
1021
|
+
@dc.dataclass(frozen=True)
|
1022
|
+
class CommandOutputOrExceptionData(CommandOutputOrException):
|
1023
|
+
output: ta.Optional[Command.Output] = None
|
1024
|
+
exception: ta.Optional[CommandException] = None
|
900
1025
|
|
901
|
-
def removeFilter(self, filter): # noqa
|
902
|
-
self._underlying.removeFilter(filter)
|
903
1026
|
|
904
|
-
|
905
|
-
|
1027
|
+
class CommandExecutor(abc.ABC, ta.Generic[CommandT, CommandOutputT]):
|
1028
|
+
@abc.abstractmethod
|
1029
|
+
def execute(self, cmd: CommandT) -> CommandOutputT:
|
1030
|
+
raise NotImplementedError
|
906
1031
|
|
1032
|
+
def try_execute(
|
1033
|
+
self,
|
1034
|
+
cmd: CommandT,
|
1035
|
+
*,
|
1036
|
+
log: ta.Optional[logging.Logger] = None,
|
1037
|
+
omit_exc_object: bool = False,
|
1038
|
+
) -> CommandOutputOrException[CommandOutputT]:
|
1039
|
+
try:
|
1040
|
+
o = self.execute(cmd)
|
907
1041
|
|
908
|
-
|
909
|
-
|
910
|
-
|
1042
|
+
except Exception as e: # noqa
|
1043
|
+
if log is not None:
|
1044
|
+
log.exception('Exception executing command: %r', type(cmd))
|
911
1045
|
|
912
|
-
|
1046
|
+
return CommandOutputOrExceptionData(exception=CommandException.of(
|
1047
|
+
e,
|
1048
|
+
omit_exc_object=omit_exc_object,
|
1049
|
+
cmd=cmd,
|
1050
|
+
))
|
913
1051
|
|
914
|
-
|
915
|
-
|
916
|
-
return self._underlying
|
1052
|
+
else:
|
1053
|
+
return CommandOutputOrExceptionData(output=o)
|
917
1054
|
|
918
|
-
def get_name(self):
|
919
|
-
return self._underlying.get_name()
|
920
1055
|
|
921
|
-
|
922
|
-
self._underlying.set_name(name)
|
1056
|
+
##
|
923
1057
|
|
924
|
-
@property
|
925
|
-
def name(self):
|
926
|
-
return self._underlying.name
|
927
1058
|
|
928
|
-
|
929
|
-
|
930
|
-
|
1059
|
+
@dc.dataclass(frozen=True)
|
1060
|
+
class CommandRegistration:
|
1061
|
+
command_cls: ta.Type[Command]
|
931
1062
|
|
932
|
-
|
933
|
-
def level(self, level):
|
934
|
-
self._underlying.level = level
|
1063
|
+
name: ta.Optional[str] = None
|
935
1064
|
|
936
1065
|
@property
|
937
|
-
def
|
938
|
-
|
1066
|
+
def name_or_default(self) -> str:
|
1067
|
+
if not (cls_name := self.command_cls.__name__).endswith('Command'):
|
1068
|
+
raise NameError(cls_name)
|
1069
|
+
return snake_case(cls_name[:-len('Command')])
|
939
1070
|
|
940
|
-
@formatter.setter
|
941
|
-
def formatter(self, formatter):
|
942
|
-
self._underlying.formatter = formatter
|
943
1071
|
|
944
|
-
|
945
|
-
self._underlying.createLock()
|
1072
|
+
CommandRegistrations = ta.NewType('CommandRegistrations', ta.Sequence[CommandRegistration])
|
946
1073
|
|
947
|
-
def acquire(self):
|
948
|
-
self._underlying.acquire()
|
949
1074
|
|
950
|
-
|
951
|
-
self._underlying.release()
|
1075
|
+
##
|
952
1076
|
|
953
|
-
def setLevel(self, level):
|
954
|
-
self._underlying.setLevel(level)
|
955
1077
|
|
956
|
-
|
957
|
-
|
1078
|
+
@dc.dataclass(frozen=True)
|
1079
|
+
class CommandExecutorRegistration:
|
1080
|
+
command_cls: ta.Type[Command]
|
1081
|
+
executor_cls: ta.Type[CommandExecutor]
|
1082
|
+
|
1083
|
+
|
1084
|
+
CommandExecutorRegistrations = ta.NewType('CommandExecutorRegistrations', ta.Sequence[CommandExecutorRegistration])
|
1085
|
+
|
1086
|
+
|
1087
|
+
##
|
1088
|
+
|
1089
|
+
|
1090
|
+
CommandNameMap = ta.NewType('CommandNameMap', ta.Mapping[str, ta.Type[Command]])
|
1091
|
+
|
1092
|
+
|
1093
|
+
def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
1094
|
+
dct: ta.Dict[str, ta.Type[Command]] = {}
|
1095
|
+
cr: CommandRegistration
|
1096
|
+
for cr in crs:
|
1097
|
+
if (name := cr.name_or_default) in dct:
|
1098
|
+
raise NameError(name)
|
1099
|
+
dct[name] = cr.command_cls
|
1100
|
+
return CommandNameMap(dct)
|
1101
|
+
|
1102
|
+
|
1103
|
+
########################################
|
1104
|
+
# ../remote/config.py
|
1105
|
+
|
1106
|
+
|
1107
|
+
@dc.dataclass(frozen=True)
|
1108
|
+
class RemoteConfig:
|
1109
|
+
payload_file: ta.Optional[str] = None
|
1110
|
+
|
1111
|
+
pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
|
1112
|
+
|
1113
|
+
|
1114
|
+
########################################
|
1115
|
+
# ../remote/payload.py
|
1116
|
+
|
1117
|
+
|
1118
|
+
RemoteExecutionPayloadFile = ta.NewType('RemoteExecutionPayloadFile', str)
|
1119
|
+
|
1120
|
+
|
1121
|
+
@cached_nullary
|
1122
|
+
def _get_self_src() -> str:
|
1123
|
+
return inspect.getsource(sys.modules[__name__])
|
1124
|
+
|
1125
|
+
|
1126
|
+
def _is_src_amalg(src: str) -> bool:
|
1127
|
+
for l in src.splitlines(): # noqa
|
1128
|
+
if l.startswith('# @omlish-amalg-output '):
|
1129
|
+
return True
|
1130
|
+
return False
|
1131
|
+
|
1132
|
+
|
1133
|
+
@cached_nullary
|
1134
|
+
def _is_self_amalg() -> bool:
|
1135
|
+
return _is_src_amalg(_get_self_src())
|
1136
|
+
|
1137
|
+
|
1138
|
+
def get_remote_payload_src(
|
1139
|
+
*,
|
1140
|
+
file: ta.Optional[RemoteExecutionPayloadFile],
|
1141
|
+
) -> str:
|
1142
|
+
if file is not None:
|
1143
|
+
with open(file) as f:
|
1144
|
+
return f.read()
|
1145
|
+
|
1146
|
+
if _is_self_amalg():
|
1147
|
+
return _get_self_src()
|
1148
|
+
|
1149
|
+
import importlib.resources
|
1150
|
+
return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
|
1151
|
+
|
1152
|
+
|
1153
|
+
########################################
|
1154
|
+
# ../../../omlish/lite/inject.py
|
1155
|
+
|
1156
|
+
|
1157
|
+
###
|
1158
|
+
# types
|
1159
|
+
|
1160
|
+
|
1161
|
+
@dc.dataclass(frozen=True)
|
1162
|
+
class InjectorKey(ta.Generic[T]):
|
1163
|
+
# Before PEP-560 typing.Generic was a metaclass with a __new__ that takes a 'cls' arg, so instantiating a dataclass
|
1164
|
+
# with kwargs (such as through dc.replace) causes `TypeError: __new__() got multiple values for argument 'cls'`.
|
1165
|
+
# See:
|
1166
|
+
# - https://github.com/python/cpython/commit/d911e40e788fb679723d78b6ea11cabf46caed5a
|
1167
|
+
# - https://gist.github.com/wrmsr/4468b86efe9f373b6b114bfe85b98fd3
|
1168
|
+
cls_: InjectorKeyCls
|
1169
|
+
|
1170
|
+
tag: ta.Any = None
|
1171
|
+
array: bool = False
|
1172
|
+
|
1173
|
+
|
1174
|
+
def is_valid_injector_key_cls(cls: ta.Any) -> bool:
|
1175
|
+
return isinstance(cls, type) or is_new_type(cls)
|
1176
|
+
|
1177
|
+
|
1178
|
+
def check_valid_injector_key_cls(cls: T) -> T:
|
1179
|
+
if not is_valid_injector_key_cls(cls):
|
1180
|
+
raise TypeError(cls)
|
1181
|
+
return cls
|
1182
|
+
|
1183
|
+
|
1184
|
+
##
|
1185
|
+
|
1186
|
+
|
1187
|
+
class InjectorProvider(abc.ABC):
|
1188
|
+
@abc.abstractmethod
|
1189
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1190
|
+
raise NotImplementedError
|
1191
|
+
|
1192
|
+
|
1193
|
+
##
|
1194
|
+
|
1195
|
+
|
1196
|
+
@dc.dataclass(frozen=True)
|
1197
|
+
class InjectorBinding:
|
1198
|
+
key: InjectorKey
|
1199
|
+
provider: InjectorProvider
|
1200
|
+
|
1201
|
+
|
1202
|
+
class InjectorBindings(abc.ABC):
|
1203
|
+
@abc.abstractmethod
|
1204
|
+
def bindings(self) -> ta.Iterator[InjectorBinding]:
|
1205
|
+
raise NotImplementedError
|
1206
|
+
|
1207
|
+
##
|
1208
|
+
|
1209
|
+
|
1210
|
+
class Injector(abc.ABC):
|
1211
|
+
@abc.abstractmethod
|
1212
|
+
def try_provide(self, key: ta.Any) -> Maybe[ta.Any]:
|
1213
|
+
raise NotImplementedError
|
1214
|
+
|
1215
|
+
@abc.abstractmethod
|
1216
|
+
def provide(self, key: ta.Any) -> ta.Any:
|
1217
|
+
raise NotImplementedError
|
1218
|
+
|
1219
|
+
@abc.abstractmethod
|
1220
|
+
def provide_kwargs(
|
1221
|
+
self,
|
1222
|
+
obj: ta.Any,
|
1223
|
+
*,
|
1224
|
+
skip_args: int = 0,
|
1225
|
+
skip_kwargs: ta.Optional[ta.Iterable[ta.Any]] = None,
|
1226
|
+
) -> ta.Mapping[str, ta.Any]:
|
1227
|
+
raise NotImplementedError
|
1228
|
+
|
1229
|
+
@abc.abstractmethod
|
1230
|
+
def inject(
|
1231
|
+
self,
|
1232
|
+
obj: ta.Any,
|
1233
|
+
*,
|
1234
|
+
args: ta.Optional[ta.Sequence[ta.Any]] = None,
|
1235
|
+
kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
1236
|
+
) -> ta.Any:
|
1237
|
+
raise NotImplementedError
|
1238
|
+
|
1239
|
+
def __getitem__(
|
1240
|
+
self,
|
1241
|
+
target: ta.Union[InjectorKey[T], ta.Type[T]],
|
1242
|
+
) -> T:
|
1243
|
+
return self.provide(target)
|
1244
|
+
|
1245
|
+
|
1246
|
+
###
|
1247
|
+
# exceptions
|
1248
|
+
|
1249
|
+
|
1250
|
+
class InjectorError(Exception):
|
1251
|
+
pass
|
1252
|
+
|
1253
|
+
|
1254
|
+
@dc.dataclass()
|
1255
|
+
class InjectorKeyError(InjectorError):
|
1256
|
+
key: InjectorKey
|
1257
|
+
|
1258
|
+
source: ta.Any = None
|
1259
|
+
name: ta.Optional[str] = None
|
1260
|
+
|
1261
|
+
|
1262
|
+
class UnboundInjectorKeyError(InjectorKeyError):
|
1263
|
+
pass
|
1264
|
+
|
1265
|
+
|
1266
|
+
class DuplicateInjectorKeyError(InjectorKeyError):
|
1267
|
+
pass
|
1268
|
+
|
1269
|
+
|
1270
|
+
class CyclicDependencyInjectorKeyError(InjectorKeyError):
|
1271
|
+
pass
|
1272
|
+
|
1273
|
+
|
1274
|
+
###
|
1275
|
+
# keys
|
1276
|
+
|
1277
|
+
|
1278
|
+
def as_injector_key(o: ta.Any) -> InjectorKey:
|
1279
|
+
if o is inspect.Parameter.empty:
|
1280
|
+
raise TypeError(o)
|
1281
|
+
if isinstance(o, InjectorKey):
|
1282
|
+
return o
|
1283
|
+
if is_valid_injector_key_cls(o):
|
1284
|
+
return InjectorKey(o)
|
1285
|
+
raise TypeError(o)
|
1286
|
+
|
1287
|
+
|
1288
|
+
###
|
1289
|
+
# providers
|
1290
|
+
|
1291
|
+
|
1292
|
+
@dc.dataclass(frozen=True)
|
1293
|
+
class FnInjectorProvider(InjectorProvider):
|
1294
|
+
fn: ta.Any
|
1295
|
+
|
1296
|
+
def __post_init__(self) -> None:
|
1297
|
+
check_not_isinstance(self.fn, type)
|
1298
|
+
|
1299
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1300
|
+
def pfn(i: Injector) -> ta.Any:
|
1301
|
+
return i.inject(self.fn)
|
1302
|
+
|
1303
|
+
return pfn
|
1304
|
+
|
1305
|
+
|
1306
|
+
@dc.dataclass(frozen=True)
|
1307
|
+
class CtorInjectorProvider(InjectorProvider):
|
1308
|
+
cls_: type
|
1309
|
+
|
1310
|
+
def __post_init__(self) -> None:
|
1311
|
+
check_isinstance(self.cls_, type)
|
1312
|
+
|
1313
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1314
|
+
def pfn(i: Injector) -> ta.Any:
|
1315
|
+
return i.inject(self.cls_)
|
1316
|
+
|
1317
|
+
return pfn
|
1318
|
+
|
1319
|
+
|
1320
|
+
@dc.dataclass(frozen=True)
|
1321
|
+
class ConstInjectorProvider(InjectorProvider):
|
1322
|
+
v: ta.Any
|
1323
|
+
|
1324
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1325
|
+
return lambda _: self.v
|
1326
|
+
|
1327
|
+
|
1328
|
+
@dc.dataclass(frozen=True)
|
1329
|
+
class SingletonInjectorProvider(InjectorProvider):
|
1330
|
+
p: InjectorProvider
|
1331
|
+
|
1332
|
+
def __post_init__(self) -> None:
|
1333
|
+
check_isinstance(self.p, InjectorProvider)
|
1334
|
+
|
1335
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1336
|
+
v = not_set = object()
|
1337
|
+
|
1338
|
+
def pfn(i: Injector) -> ta.Any:
|
1339
|
+
nonlocal v
|
1340
|
+
if v is not_set:
|
1341
|
+
v = ufn(i)
|
1342
|
+
return v
|
1343
|
+
|
1344
|
+
ufn = self.p.provider_fn()
|
1345
|
+
return pfn
|
1346
|
+
|
1347
|
+
|
1348
|
+
@dc.dataclass(frozen=True)
|
1349
|
+
class LinkInjectorProvider(InjectorProvider):
|
1350
|
+
k: InjectorKey
|
1351
|
+
|
1352
|
+
def __post_init__(self) -> None:
|
1353
|
+
check_isinstance(self.k, InjectorKey)
|
1354
|
+
|
1355
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1356
|
+
def pfn(i: Injector) -> ta.Any:
|
1357
|
+
return i.provide(self.k)
|
1358
|
+
|
1359
|
+
return pfn
|
1360
|
+
|
1361
|
+
|
1362
|
+
@dc.dataclass(frozen=True)
|
1363
|
+
class ArrayInjectorProvider(InjectorProvider):
|
1364
|
+
ps: ta.Sequence[InjectorProvider]
|
1365
|
+
|
1366
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1367
|
+
ps = [p.provider_fn() for p in self.ps]
|
1368
|
+
|
1369
|
+
def pfn(i: Injector) -> ta.Any:
|
1370
|
+
rv = []
|
1371
|
+
for ep in ps:
|
1372
|
+
o = ep(i)
|
1373
|
+
rv.append(o)
|
1374
|
+
return rv
|
1375
|
+
|
1376
|
+
return pfn
|
1377
|
+
|
1378
|
+
|
1379
|
+
###
|
1380
|
+
# bindings
|
1381
|
+
|
1382
|
+
|
1383
|
+
@dc.dataclass(frozen=True)
|
1384
|
+
class _InjectorBindings(InjectorBindings):
|
1385
|
+
bs: ta.Optional[ta.Sequence[InjectorBinding]] = None
|
1386
|
+
ps: ta.Optional[ta.Sequence[InjectorBindings]] = None
|
1387
|
+
|
1388
|
+
def bindings(self) -> ta.Iterator[InjectorBinding]:
|
1389
|
+
if self.bs is not None:
|
1390
|
+
yield from self.bs
|
1391
|
+
if self.ps is not None:
|
1392
|
+
for p in self.ps:
|
1393
|
+
yield from p.bindings()
|
1394
|
+
|
1395
|
+
|
1396
|
+
def as_injector_bindings(*args: InjectorBindingOrBindings) -> InjectorBindings:
|
1397
|
+
bs: ta.List[InjectorBinding] = []
|
1398
|
+
ps: ta.List[InjectorBindings] = []
|
1399
|
+
|
1400
|
+
for a in args:
|
1401
|
+
if isinstance(a, InjectorBindings):
|
1402
|
+
ps.append(a)
|
1403
|
+
elif isinstance(a, InjectorBinding):
|
1404
|
+
bs.append(a)
|
1405
|
+
else:
|
1406
|
+
raise TypeError(a)
|
1407
|
+
|
1408
|
+
return _InjectorBindings(
|
1409
|
+
bs or None,
|
1410
|
+
ps or None,
|
1411
|
+
)
|
1412
|
+
|
1413
|
+
|
1414
|
+
##
|
1415
|
+
|
1416
|
+
|
1417
|
+
@dc.dataclass(frozen=True)
|
1418
|
+
class OverridesInjectorBindings(InjectorBindings):
|
1419
|
+
p: InjectorBindings
|
1420
|
+
m: ta.Mapping[InjectorKey, InjectorBinding]
|
1421
|
+
|
1422
|
+
def bindings(self) -> ta.Iterator[InjectorBinding]:
|
1423
|
+
for b in self.p.bindings():
|
1424
|
+
yield self.m.get(b.key, b)
|
1425
|
+
|
1426
|
+
|
1427
|
+
def injector_override(p: InjectorBindings, *args: InjectorBindingOrBindings) -> InjectorBindings:
|
1428
|
+
m: ta.Dict[InjectorKey, InjectorBinding] = {}
|
1429
|
+
|
1430
|
+
for b in as_injector_bindings(*args).bindings():
|
1431
|
+
if b.key in m:
|
1432
|
+
raise DuplicateInjectorKeyError(b.key)
|
1433
|
+
m[b.key] = b
|
1434
|
+
|
1435
|
+
return OverridesInjectorBindings(p, m)
|
1436
|
+
|
1437
|
+
|
1438
|
+
##
|
1439
|
+
|
1440
|
+
|
1441
|
+
def build_injector_provider_map(bs: InjectorBindings) -> ta.Mapping[InjectorKey, InjectorProvider]:
|
1442
|
+
pm: ta.Dict[InjectorKey, InjectorProvider] = {}
|
1443
|
+
am: ta.Dict[InjectorKey, ta.List[InjectorProvider]] = {}
|
1444
|
+
|
1445
|
+
for b in bs.bindings():
|
1446
|
+
if b.key.array:
|
1447
|
+
al = am.setdefault(b.key, [])
|
1448
|
+
if isinstance(b.provider, ArrayInjectorProvider):
|
1449
|
+
al.extend(b.provider.ps)
|
1450
|
+
else:
|
1451
|
+
al.append(b.provider)
|
1452
|
+
else:
|
1453
|
+
if b.key in pm:
|
1454
|
+
raise KeyError(b.key)
|
1455
|
+
pm[b.key] = b.provider
|
1456
|
+
|
1457
|
+
if am:
|
1458
|
+
for k, aps in am.items():
|
1459
|
+
pm[k] = ArrayInjectorProvider(aps)
|
1460
|
+
|
1461
|
+
return pm
|
1462
|
+
|
1463
|
+
|
1464
|
+
###
|
1465
|
+
# inspection
|
1466
|
+
|
1467
|
+
|
1468
|
+
class _InjectionInspection(ta.NamedTuple):
|
1469
|
+
signature: inspect.Signature
|
1470
|
+
type_hints: ta.Mapping[str, ta.Any]
|
1471
|
+
args_offset: int
|
1472
|
+
|
1473
|
+
|
1474
|
+
_INJECTION_INSPECTION_CACHE: ta.MutableMapping[ta.Any, _InjectionInspection] = weakref.WeakKeyDictionary()
|
1475
|
+
|
1476
|
+
|
1477
|
+
def _do_injection_inspect(obj: ta.Any) -> _InjectionInspection:
|
1478
|
+
tgt = obj
|
1479
|
+
if isinstance(tgt, type) and tgt.__init__ is not object.__init__: # type: ignore[misc]
|
1480
|
+
# Python 3.8's inspect.signature can't handle subclasses overriding __new__, always generating *args/**kwargs.
|
1481
|
+
# - https://bugs.python.org/issue40897
|
1482
|
+
# - https://github.com/python/cpython/commit/df7c62980d15acd3125dfbd81546dad359f7add7
|
1483
|
+
tgt = tgt.__init__ # type: ignore[misc]
|
1484
|
+
has_generic_base = True
|
1485
|
+
else:
|
1486
|
+
has_generic_base = False
|
1487
|
+
|
1488
|
+
# inspect.signature(eval_str=True) was added in 3.10 and we have to support 3.8, so we have to get_type_hints to
|
1489
|
+
# eval str annotations *in addition to* getting the signature for parameter information.
|
1490
|
+
uw = tgt
|
1491
|
+
has_partial = False
|
1492
|
+
while True:
|
1493
|
+
if isinstance(uw, functools.partial):
|
1494
|
+
has_partial = True
|
1495
|
+
uw = uw.func
|
1496
|
+
else:
|
1497
|
+
if (uw2 := inspect.unwrap(uw)) is uw:
|
1498
|
+
break
|
1499
|
+
uw = uw2
|
1500
|
+
|
1501
|
+
if has_generic_base and has_partial:
|
1502
|
+
raise InjectorError(
|
1503
|
+
'Injector inspection does not currently support both a typing.Generic base and a functools.partial: '
|
1504
|
+
f'{obj}',
|
1505
|
+
)
|
1506
|
+
|
1507
|
+
return _InjectionInspection(
|
1508
|
+
inspect.signature(tgt),
|
1509
|
+
ta.get_type_hints(uw),
|
1510
|
+
1 if has_generic_base else 0,
|
1511
|
+
)
|
1512
|
+
|
1513
|
+
|
1514
|
+
def _injection_inspect(obj: ta.Any) -> _InjectionInspection:
|
1515
|
+
try:
|
1516
|
+
return _INJECTION_INSPECTION_CACHE[obj]
|
1517
|
+
except TypeError:
|
1518
|
+
return _do_injection_inspect(obj)
|
1519
|
+
except KeyError:
|
1520
|
+
pass
|
1521
|
+
insp = _do_injection_inspect(obj)
|
1522
|
+
_INJECTION_INSPECTION_CACHE[obj] = insp
|
1523
|
+
return insp
|
1524
|
+
|
1525
|
+
|
1526
|
+
class InjectionKwarg(ta.NamedTuple):
|
1527
|
+
name: str
|
1528
|
+
key: InjectorKey
|
1529
|
+
has_default: bool
|
1530
|
+
|
1531
|
+
|
1532
|
+
class InjectionKwargsTarget(ta.NamedTuple):
|
1533
|
+
obj: ta.Any
|
1534
|
+
kwargs: ta.Sequence[InjectionKwarg]
|
1535
|
+
|
1536
|
+
|
1537
|
+
def build_injection_kwargs_target(
|
1538
|
+
obj: ta.Any,
|
1539
|
+
*,
|
1540
|
+
skip_args: int = 0,
|
1541
|
+
skip_kwargs: ta.Optional[ta.Iterable[str]] = None,
|
1542
|
+
raw_optional: bool = False,
|
1543
|
+
) -> InjectionKwargsTarget:
|
1544
|
+
insp = _injection_inspect(obj)
|
1545
|
+
|
1546
|
+
params = list(insp.signature.parameters.values())
|
1547
|
+
|
1548
|
+
skip_names: ta.Set[str] = set()
|
1549
|
+
if skip_kwargs is not None:
|
1550
|
+
skip_names.update(check_not_isinstance(skip_kwargs, str))
|
1551
|
+
|
1552
|
+
seen: ta.Set[InjectorKey] = set()
|
1553
|
+
kws: ta.List[InjectionKwarg] = []
|
1554
|
+
for p in params[insp.args_offset + skip_args:]:
|
1555
|
+
if p.name in skip_names:
|
1556
|
+
continue
|
1557
|
+
|
1558
|
+
if p.annotation is inspect.Signature.empty:
|
1559
|
+
if p.default is not inspect.Parameter.empty:
|
1560
|
+
raise KeyError(f'{obj}, {p.name}')
|
1561
|
+
continue
|
1562
|
+
|
1563
|
+
if p.kind not in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY):
|
1564
|
+
raise TypeError(insp)
|
1565
|
+
|
1566
|
+
# 3.8 inspect.signature doesn't eval_str but typing.get_type_hints does, so prefer that.
|
1567
|
+
ann = insp.type_hints.get(p.name, p.annotation)
|
1568
|
+
if (
|
1569
|
+
not raw_optional and
|
1570
|
+
is_optional_alias(ann)
|
1571
|
+
):
|
1572
|
+
ann = get_optional_alias_arg(ann)
|
1573
|
+
|
1574
|
+
k = as_injector_key(ann)
|
1575
|
+
|
1576
|
+
if k in seen:
|
1577
|
+
raise DuplicateInjectorKeyError(k)
|
1578
|
+
seen.add(k)
|
1579
|
+
|
1580
|
+
kws.append(InjectionKwarg(
|
1581
|
+
p.name,
|
1582
|
+
k,
|
1583
|
+
p.default is not inspect.Parameter.empty,
|
1584
|
+
))
|
1585
|
+
|
1586
|
+
return InjectionKwargsTarget(
|
1587
|
+
obj,
|
1588
|
+
kws,
|
1589
|
+
)
|
1590
|
+
|
1591
|
+
|
1592
|
+
###
|
1593
|
+
# injector
|
1594
|
+
|
1595
|
+
|
1596
|
+
_INJECTOR_INJECTOR_KEY: InjectorKey[Injector] = InjectorKey(Injector)
|
1597
|
+
|
1598
|
+
|
1599
|
+
@dc.dataclass(frozen=True)
|
1600
|
+
class _InjectorEager:
|
1601
|
+
key: InjectorKey
|
1602
|
+
|
1603
|
+
|
1604
|
+
_INJECTOR_EAGER_ARRAY_KEY: InjectorKey[_InjectorEager] = InjectorKey(_InjectorEager, array=True)
|
1605
|
+
|
1606
|
+
|
1607
|
+
class _Injector(Injector):
|
1608
|
+
def __init__(self, bs: InjectorBindings, p: ta.Optional[Injector] = None) -> None:
|
1609
|
+
super().__init__()
|
1610
|
+
|
1611
|
+
self._bs = check_isinstance(bs, InjectorBindings)
|
1612
|
+
self._p: ta.Optional[Injector] = check_isinstance(p, (Injector, type(None)))
|
1613
|
+
|
1614
|
+
self._pfm = {k: v.provider_fn() for k, v in build_injector_provider_map(bs).items()}
|
1615
|
+
|
1616
|
+
if _INJECTOR_INJECTOR_KEY in self._pfm:
|
1617
|
+
raise DuplicateInjectorKeyError(_INJECTOR_INJECTOR_KEY)
|
1618
|
+
|
1619
|
+
self.__cur_req: ta.Optional[_Injector._Request] = None
|
1620
|
+
|
1621
|
+
if _INJECTOR_EAGER_ARRAY_KEY in self._pfm:
|
1622
|
+
for e in self.provide(_INJECTOR_EAGER_ARRAY_KEY):
|
1623
|
+
self.provide(e.key)
|
1624
|
+
|
1625
|
+
class _Request:
|
1626
|
+
def __init__(self, injector: '_Injector') -> None:
|
1627
|
+
super().__init__()
|
1628
|
+
self._injector = injector
|
1629
|
+
self._provisions: ta.Dict[InjectorKey, Maybe] = {}
|
1630
|
+
self._seen_keys: ta.Set[InjectorKey] = set()
|
1631
|
+
|
1632
|
+
def handle_key(self, key: InjectorKey) -> Maybe[Maybe]:
|
1633
|
+
try:
|
1634
|
+
return Maybe.just(self._provisions[key])
|
1635
|
+
except KeyError:
|
1636
|
+
pass
|
1637
|
+
if key in self._seen_keys:
|
1638
|
+
raise CyclicDependencyInjectorKeyError(key)
|
1639
|
+
self._seen_keys.add(key)
|
1640
|
+
return Maybe.empty()
|
1641
|
+
|
1642
|
+
def handle_provision(self, key: InjectorKey, mv: Maybe) -> Maybe:
|
1643
|
+
check_in(key, self._seen_keys)
|
1644
|
+
check_not_in(key, self._provisions)
|
1645
|
+
self._provisions[key] = mv
|
1646
|
+
return mv
|
1647
|
+
|
1648
|
+
@contextlib.contextmanager
|
1649
|
+
def _current_request(self) -> ta.Generator[_Request, None, None]:
|
1650
|
+
if (cr := self.__cur_req) is not None:
|
1651
|
+
yield cr
|
1652
|
+
return
|
1653
|
+
|
1654
|
+
cr = self._Request(self)
|
1655
|
+
try:
|
1656
|
+
self.__cur_req = cr
|
1657
|
+
yield cr
|
1658
|
+
finally:
|
1659
|
+
self.__cur_req = None
|
1660
|
+
|
1661
|
+
def try_provide(self, key: ta.Any) -> Maybe[ta.Any]:
|
1662
|
+
key = as_injector_key(key)
|
1663
|
+
|
1664
|
+
cr: _Injector._Request
|
1665
|
+
with self._current_request() as cr:
|
1666
|
+
if (rv := cr.handle_key(key)).present:
|
1667
|
+
return rv.must()
|
1668
|
+
|
1669
|
+
if key == _INJECTOR_INJECTOR_KEY:
|
1670
|
+
return cr.handle_provision(key, Maybe.just(self))
|
1671
|
+
|
1672
|
+
fn = self._pfm.get(key)
|
1673
|
+
if fn is not None:
|
1674
|
+
return cr.handle_provision(key, Maybe.just(fn(self)))
|
1675
|
+
|
1676
|
+
if self._p is not None:
|
1677
|
+
pv = self._p.try_provide(key)
|
1678
|
+
if pv is not None:
|
1679
|
+
return cr.handle_provision(key, Maybe.empty())
|
1680
|
+
|
1681
|
+
return cr.handle_provision(key, Maybe.empty())
|
1682
|
+
|
1683
|
+
def provide(self, key: ta.Any) -> ta.Any:
|
1684
|
+
v = self.try_provide(key)
|
1685
|
+
if v.present:
|
1686
|
+
return v.must()
|
1687
|
+
raise UnboundInjectorKeyError(key)
|
1688
|
+
|
1689
|
+
def provide_kwargs(
|
1690
|
+
self,
|
1691
|
+
obj: ta.Any,
|
1692
|
+
*,
|
1693
|
+
skip_args: int = 0,
|
1694
|
+
skip_kwargs: ta.Optional[ta.Iterable[ta.Any]] = None,
|
1695
|
+
) -> ta.Mapping[str, ta.Any]:
|
1696
|
+
kt = build_injection_kwargs_target(
|
1697
|
+
obj,
|
1698
|
+
skip_args=skip_args,
|
1699
|
+
skip_kwargs=skip_kwargs,
|
1700
|
+
)
|
1701
|
+
|
1702
|
+
ret: ta.Dict[str, ta.Any] = {}
|
1703
|
+
for kw in kt.kwargs:
|
1704
|
+
if kw.has_default:
|
1705
|
+
if not (mv := self.try_provide(kw.key)).present:
|
1706
|
+
continue
|
1707
|
+
v = mv.must()
|
1708
|
+
else:
|
1709
|
+
v = self.provide(kw.key)
|
1710
|
+
ret[kw.name] = v
|
1711
|
+
return ret
|
1712
|
+
|
1713
|
+
def inject(
|
1714
|
+
self,
|
1715
|
+
obj: ta.Any,
|
1716
|
+
*,
|
1717
|
+
args: ta.Optional[ta.Sequence[ta.Any]] = None,
|
1718
|
+
kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
1719
|
+
) -> ta.Any:
|
1720
|
+
provided = self.provide_kwargs(
|
1721
|
+
obj,
|
1722
|
+
skip_args=len(args) if args is not None else 0,
|
1723
|
+
skip_kwargs=kwargs if kwargs is not None else None,
|
1724
|
+
)
|
1725
|
+
|
1726
|
+
return obj(
|
1727
|
+
*(args if args is not None else ()),
|
1728
|
+
**(kwargs if kwargs is not None else {}),
|
1729
|
+
**provided,
|
1730
|
+
)
|
1731
|
+
|
1732
|
+
|
1733
|
+
###
|
1734
|
+
# binder
|
1735
|
+
|
1736
|
+
|
1737
|
+
class InjectorBinder:
|
1738
|
+
def __new__(cls, *args, **kwargs): # noqa
|
1739
|
+
raise TypeError
|
1740
|
+
|
1741
|
+
_FN_TYPES: ta.Tuple[type, ...] = (
|
1742
|
+
types.FunctionType,
|
1743
|
+
types.MethodType,
|
1744
|
+
|
1745
|
+
classmethod,
|
1746
|
+
staticmethod,
|
1747
|
+
|
1748
|
+
functools.partial,
|
1749
|
+
functools.partialmethod,
|
1750
|
+
)
|
1751
|
+
|
1752
|
+
@classmethod
|
1753
|
+
def _is_fn(cls, obj: ta.Any) -> bool:
|
1754
|
+
return isinstance(obj, cls._FN_TYPES)
|
1755
|
+
|
1756
|
+
@classmethod
|
1757
|
+
def bind_as_fn(cls, icls: ta.Type[T]) -> ta.Type[T]:
|
1758
|
+
check_isinstance(icls, type)
|
1759
|
+
if icls not in cls._FN_TYPES:
|
1760
|
+
cls._FN_TYPES = (*cls._FN_TYPES, icls)
|
1761
|
+
return icls
|
1762
|
+
|
1763
|
+
_BANNED_BIND_TYPES: ta.Tuple[type, ...] = (
|
1764
|
+
InjectorProvider,
|
1765
|
+
)
|
1766
|
+
|
1767
|
+
@classmethod
|
1768
|
+
def bind(
|
1769
|
+
cls,
|
1770
|
+
obj: ta.Any,
|
1771
|
+
*,
|
1772
|
+
key: ta.Any = None,
|
1773
|
+
tag: ta.Any = None,
|
1774
|
+
array: ta.Optional[bool] = None, # noqa
|
1775
|
+
|
1776
|
+
to_fn: ta.Any = None,
|
1777
|
+
to_ctor: ta.Any = None,
|
1778
|
+
to_const: ta.Any = None,
|
1779
|
+
to_key: ta.Any = None,
|
1780
|
+
|
1781
|
+
singleton: bool = False,
|
1782
|
+
|
1783
|
+
eager: bool = False,
|
1784
|
+
) -> InjectorBindingOrBindings:
|
1785
|
+
if obj is None or obj is inspect.Parameter.empty:
|
1786
|
+
raise TypeError(obj)
|
1787
|
+
if isinstance(obj, cls._BANNED_BIND_TYPES):
|
1788
|
+
raise TypeError(obj)
|
1789
|
+
|
1790
|
+
##
|
1791
|
+
|
1792
|
+
if key is not None:
|
1793
|
+
key = as_injector_key(key)
|
1794
|
+
|
1795
|
+
##
|
1796
|
+
|
1797
|
+
has_to = (
|
1798
|
+
to_fn is not None or
|
1799
|
+
to_ctor is not None or
|
1800
|
+
to_const is not None or
|
1801
|
+
to_key is not None
|
1802
|
+
)
|
1803
|
+
if isinstance(obj, InjectorKey):
|
1804
|
+
if key is None:
|
1805
|
+
key = obj
|
1806
|
+
elif isinstance(obj, type):
|
1807
|
+
if not has_to:
|
1808
|
+
to_ctor = obj
|
1809
|
+
if key is None:
|
1810
|
+
key = InjectorKey(obj)
|
1811
|
+
elif cls._is_fn(obj) and not has_to:
|
1812
|
+
to_fn = obj
|
1813
|
+
if key is None:
|
1814
|
+
insp = _injection_inspect(obj)
|
1815
|
+
key_cls: ta.Any = check_valid_injector_key_cls(check_not_none(insp.type_hints.get('return')))
|
1816
|
+
key = InjectorKey(key_cls)
|
1817
|
+
else:
|
1818
|
+
if to_const is not None:
|
1819
|
+
raise TypeError('Cannot bind instance with to_const')
|
1820
|
+
to_const = obj
|
1821
|
+
if key is None:
|
1822
|
+
key = InjectorKey(type(obj))
|
1823
|
+
del has_to
|
1824
|
+
|
1825
|
+
##
|
1826
|
+
|
1827
|
+
if tag is not None:
|
1828
|
+
if key.tag is not None:
|
1829
|
+
raise TypeError('Tag already set')
|
1830
|
+
key = dc.replace(key, tag=tag)
|
1831
|
+
|
1832
|
+
if array is not None:
|
1833
|
+
key = dc.replace(key, array=array)
|
1834
|
+
|
1835
|
+
##
|
1836
|
+
|
1837
|
+
providers: ta.List[InjectorProvider] = []
|
1838
|
+
if to_fn is not None:
|
1839
|
+
providers.append(FnInjectorProvider(to_fn))
|
1840
|
+
if to_ctor is not None:
|
1841
|
+
providers.append(CtorInjectorProvider(to_ctor))
|
1842
|
+
if to_const is not None:
|
1843
|
+
providers.append(ConstInjectorProvider(to_const))
|
1844
|
+
if to_key is not None:
|
1845
|
+
providers.append(LinkInjectorProvider(as_injector_key(to_key)))
|
1846
|
+
if not providers:
|
1847
|
+
raise TypeError('Must specify provider')
|
1848
|
+
if len(providers) > 1:
|
1849
|
+
raise TypeError('May not specify multiple providers')
|
1850
|
+
provider, = providers
|
1851
|
+
|
1852
|
+
##
|
1853
|
+
|
1854
|
+
if singleton:
|
1855
|
+
provider = SingletonInjectorProvider(provider)
|
1856
|
+
|
1857
|
+
binding = InjectorBinding(key, provider)
|
1858
|
+
|
1859
|
+
##
|
1860
|
+
|
1861
|
+
extras: ta.List[InjectorBinding] = []
|
1862
|
+
|
1863
|
+
if eager:
|
1864
|
+
extras.append(bind_injector_eager_key(key))
|
1865
|
+
|
1866
|
+
##
|
1867
|
+
|
1868
|
+
if extras:
|
1869
|
+
return as_injector_bindings(binding, *extras)
|
1870
|
+
else:
|
1871
|
+
return binding
|
1872
|
+
|
1873
|
+
|
1874
|
+
###
|
1875
|
+
# injection helpers
|
1876
|
+
|
1877
|
+
|
1878
|
+
def make_injector_factory(
|
1879
|
+
fn: ta.Callable[..., T],
|
1880
|
+
cls: U,
|
1881
|
+
ann: ta.Any = None,
|
1882
|
+
) -> ta.Callable[..., U]:
|
1883
|
+
if ann is None:
|
1884
|
+
ann = cls
|
1885
|
+
|
1886
|
+
def outer(injector: Injector) -> ann:
|
1887
|
+
def inner(*args, **kwargs):
|
1888
|
+
return injector.inject(fn, args=args, kwargs=kwargs)
|
1889
|
+
return cls(inner) # type: ignore
|
1890
|
+
|
1891
|
+
return outer
|
1892
|
+
|
1893
|
+
|
1894
|
+
def bind_injector_array(
|
1895
|
+
obj: ta.Any = None,
|
1896
|
+
*,
|
1897
|
+
tag: ta.Any = None,
|
1898
|
+
) -> InjectorBindingOrBindings:
|
1899
|
+
key = as_injector_key(obj)
|
1900
|
+
if tag is not None:
|
1901
|
+
if key.tag is not None:
|
1902
|
+
raise ValueError('Must not specify multiple tags')
|
1903
|
+
key = dc.replace(key, tag=tag)
|
1904
|
+
|
1905
|
+
if key.array:
|
1906
|
+
raise ValueError('Key must not be array')
|
1907
|
+
|
1908
|
+
return InjectorBinding(
|
1909
|
+
dc.replace(key, array=True),
|
1910
|
+
ArrayInjectorProvider([]),
|
1911
|
+
)
|
1912
|
+
|
1913
|
+
|
1914
|
+
def make_injector_array_type(
|
1915
|
+
ele: ta.Union[InjectorKey, InjectorKeyCls],
|
1916
|
+
cls: U,
|
1917
|
+
ann: ta.Any = None,
|
1918
|
+
) -> ta.Callable[..., U]:
|
1919
|
+
if isinstance(ele, InjectorKey):
|
1920
|
+
if not ele.array:
|
1921
|
+
raise InjectorError('Provided key must be array', ele)
|
1922
|
+
key = ele
|
1923
|
+
else:
|
1924
|
+
key = dc.replace(as_injector_key(ele), array=True)
|
1925
|
+
|
1926
|
+
if ann is None:
|
1927
|
+
ann = cls
|
1928
|
+
|
1929
|
+
def inner(injector: Injector) -> ann:
|
1930
|
+
return cls(injector.provide(key)) # type: ignore[operator]
|
1931
|
+
|
1932
|
+
return inner
|
1933
|
+
|
1934
|
+
|
1935
|
+
def bind_injector_eager_key(key: ta.Any) -> InjectorBinding:
|
1936
|
+
return InjectorBinding(_INJECTOR_EAGER_ARRAY_KEY, ConstInjectorProvider(_InjectorEager(as_injector_key(key))))
|
1937
|
+
|
1938
|
+
|
1939
|
+
##
|
1940
|
+
|
1941
|
+
|
1942
|
+
class Injection:
|
1943
|
+
def __new__(cls, *args, **kwargs): # noqa
|
1944
|
+
raise TypeError
|
1945
|
+
|
1946
|
+
# keys
|
1947
|
+
|
1948
|
+
@classmethod
|
1949
|
+
def as_key(cls, o: ta.Any) -> InjectorKey:
|
1950
|
+
return as_injector_key(o)
|
1951
|
+
|
1952
|
+
@classmethod
|
1953
|
+
def array(cls, o: ta.Any) -> InjectorKey:
|
1954
|
+
return dc.replace(as_injector_key(o), array=True)
|
1955
|
+
|
1956
|
+
@classmethod
|
1957
|
+
def tag(cls, o: ta.Any, t: ta.Any) -> InjectorKey:
|
1958
|
+
return dc.replace(as_injector_key(o), tag=t)
|
1959
|
+
|
1960
|
+
# bindings
|
1961
|
+
|
1962
|
+
@classmethod
|
1963
|
+
def as_bindings(cls, *args: InjectorBindingOrBindings) -> InjectorBindings:
|
1964
|
+
return as_injector_bindings(*args)
|
1965
|
+
|
1966
|
+
@classmethod
|
1967
|
+
def override(cls, p: InjectorBindings, *args: InjectorBindingOrBindings) -> InjectorBindings:
|
1968
|
+
return injector_override(p, *args)
|
1969
|
+
|
1970
|
+
# injector
|
1971
|
+
|
1972
|
+
@classmethod
|
1973
|
+
def create_injector(cls, *args: InjectorBindingOrBindings, parent: ta.Optional[Injector] = None) -> Injector:
|
1974
|
+
return _Injector(as_injector_bindings(*args), parent)
|
1975
|
+
|
1976
|
+
# binder
|
1977
|
+
|
1978
|
+
@classmethod
|
1979
|
+
def bind(
|
1980
|
+
cls,
|
1981
|
+
obj: ta.Any,
|
1982
|
+
*,
|
1983
|
+
key: ta.Any = None,
|
1984
|
+
tag: ta.Any = None,
|
1985
|
+
array: ta.Optional[bool] = None, # noqa
|
1986
|
+
|
1987
|
+
to_fn: ta.Any = None,
|
1988
|
+
to_ctor: ta.Any = None,
|
1989
|
+
to_const: ta.Any = None,
|
1990
|
+
to_key: ta.Any = None,
|
1991
|
+
|
1992
|
+
singleton: bool = False,
|
1993
|
+
|
1994
|
+
eager: bool = False,
|
1995
|
+
) -> InjectorBindingOrBindings:
|
1996
|
+
return InjectorBinder.bind(
|
1997
|
+
obj,
|
1998
|
+
|
1999
|
+
key=key,
|
2000
|
+
tag=tag,
|
2001
|
+
array=array,
|
2002
|
+
|
2003
|
+
to_fn=to_fn,
|
2004
|
+
to_ctor=to_ctor,
|
2005
|
+
to_const=to_const,
|
2006
|
+
to_key=to_key,
|
2007
|
+
|
2008
|
+
singleton=singleton,
|
2009
|
+
|
2010
|
+
eager=eager,
|
2011
|
+
)
|
2012
|
+
|
2013
|
+
# helpers
|
2014
|
+
|
2015
|
+
@classmethod
|
2016
|
+
def bind_factory(
|
2017
|
+
cls,
|
2018
|
+
fn: ta.Callable[..., T],
|
2019
|
+
cls_: U,
|
2020
|
+
ann: ta.Any = None,
|
2021
|
+
) -> InjectorBindingOrBindings:
|
2022
|
+
return cls.bind(make_injector_factory(fn, cls_, ann))
|
2023
|
+
|
2024
|
+
@classmethod
|
2025
|
+
def bind_array(
|
2026
|
+
cls,
|
2027
|
+
obj: ta.Any = None,
|
2028
|
+
*,
|
2029
|
+
tag: ta.Any = None,
|
2030
|
+
) -> InjectorBindingOrBindings:
|
2031
|
+
return bind_injector_array(obj, tag=tag)
|
2032
|
+
|
2033
|
+
@classmethod
|
2034
|
+
def bind_array_type(
|
2035
|
+
cls,
|
2036
|
+
ele: ta.Union[InjectorKey, InjectorKeyCls],
|
2037
|
+
cls_: U,
|
2038
|
+
ann: ta.Any = None,
|
2039
|
+
) -> InjectorBindingOrBindings:
|
2040
|
+
return cls.bind(make_injector_array_type(ele, cls_, ann))
|
2041
|
+
|
2042
|
+
|
2043
|
+
inj = Injection
|
2044
|
+
|
2045
|
+
|
2046
|
+
########################################
|
2047
|
+
# ../../../omlish/lite/logs.py
|
2048
|
+
"""
|
2049
|
+
TODO:
|
2050
|
+
- translate json keys
|
2051
|
+
- debug
|
2052
|
+
"""
|
2053
|
+
|
2054
|
+
|
2055
|
+
log = logging.getLogger(__name__)
|
2056
|
+
|
2057
|
+
|
2058
|
+
##
|
2059
|
+
|
2060
|
+
|
2061
|
+
class TidLogFilter(logging.Filter):
|
2062
|
+
|
2063
|
+
def filter(self, record):
|
2064
|
+
record.tid = threading.get_native_id()
|
2065
|
+
return True
|
2066
|
+
|
2067
|
+
|
2068
|
+
##
|
2069
|
+
|
2070
|
+
|
2071
|
+
class JsonLogFormatter(logging.Formatter):
|
2072
|
+
|
2073
|
+
KEYS: ta.Mapping[str, bool] = {
|
2074
|
+
'name': False,
|
2075
|
+
'msg': False,
|
2076
|
+
'args': False,
|
2077
|
+
'levelname': False,
|
2078
|
+
'levelno': False,
|
2079
|
+
'pathname': False,
|
2080
|
+
'filename': False,
|
2081
|
+
'module': False,
|
2082
|
+
'exc_info': True,
|
2083
|
+
'exc_text': True,
|
2084
|
+
'stack_info': True,
|
2085
|
+
'lineno': False,
|
2086
|
+
'funcName': False,
|
2087
|
+
'created': False,
|
2088
|
+
'msecs': False,
|
2089
|
+
'relativeCreated': False,
|
2090
|
+
'thread': False,
|
2091
|
+
'threadName': False,
|
2092
|
+
'processName': False,
|
2093
|
+
'process': False,
|
2094
|
+
}
|
2095
|
+
|
2096
|
+
def format(self, record: logging.LogRecord) -> str:
|
2097
|
+
dct = {
|
2098
|
+
k: v
|
2099
|
+
for k, o in self.KEYS.items()
|
2100
|
+
for v in [getattr(record, k)]
|
2101
|
+
if not (o and v is None)
|
2102
|
+
}
|
2103
|
+
return json_dumps_compact(dct)
|
2104
|
+
|
2105
|
+
|
2106
|
+
##
|
2107
|
+
|
2108
|
+
|
2109
|
+
STANDARD_LOG_FORMAT_PARTS = [
|
2110
|
+
('asctime', '%(asctime)-15s'),
|
2111
|
+
('process', 'pid=%(process)-6s'),
|
2112
|
+
('thread', 'tid=%(thread)x'),
|
2113
|
+
('levelname', '%(levelname)s'),
|
2114
|
+
('name', '%(name)s'),
|
2115
|
+
('separator', '::'),
|
2116
|
+
('message', '%(message)s'),
|
2117
|
+
]
|
2118
|
+
|
2119
|
+
|
2120
|
+
class StandardLogFormatter(logging.Formatter):
|
2121
|
+
|
2122
|
+
@staticmethod
|
2123
|
+
def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
|
2124
|
+
return ' '.join(v for k, v in parts)
|
2125
|
+
|
2126
|
+
converter = datetime.datetime.fromtimestamp # type: ignore
|
2127
|
+
|
2128
|
+
def formatTime(self, record, datefmt=None):
|
2129
|
+
ct = self.converter(record.created) # type: ignore
|
2130
|
+
if datefmt:
|
2131
|
+
return ct.strftime(datefmt) # noqa
|
2132
|
+
else:
|
2133
|
+
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
2134
|
+
return '%s.%03d' % (t, record.msecs) # noqa
|
2135
|
+
|
2136
|
+
|
2137
|
+
##
|
2138
|
+
|
2139
|
+
|
2140
|
+
class ProxyLogFilterer(logging.Filterer):
|
2141
|
+
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
2142
|
+
self._underlying = underlying
|
2143
|
+
|
2144
|
+
@property
|
2145
|
+
def underlying(self) -> logging.Filterer:
|
2146
|
+
return self._underlying
|
2147
|
+
|
2148
|
+
@property
|
2149
|
+
def filters(self):
|
2150
|
+
return self._underlying.filters
|
2151
|
+
|
2152
|
+
@filters.setter
|
2153
|
+
def filters(self, filters):
|
2154
|
+
self._underlying.filters = filters
|
2155
|
+
|
2156
|
+
def addFilter(self, filter): # noqa
|
2157
|
+
self._underlying.addFilter(filter)
|
2158
|
+
|
2159
|
+
def removeFilter(self, filter): # noqa
|
2160
|
+
self._underlying.removeFilter(filter)
|
2161
|
+
|
2162
|
+
def filter(self, record):
|
2163
|
+
return self._underlying.filter(record)
|
2164
|
+
|
2165
|
+
|
2166
|
+
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
2167
|
+
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
2168
|
+
ProxyLogFilterer.__init__(self, underlying)
|
2169
|
+
|
2170
|
+
_underlying: logging.Handler
|
2171
|
+
|
2172
|
+
@property
|
2173
|
+
def underlying(self) -> logging.Handler:
|
2174
|
+
return self._underlying
|
2175
|
+
|
2176
|
+
def get_name(self):
|
2177
|
+
return self._underlying.get_name()
|
2178
|
+
|
2179
|
+
def set_name(self, name):
|
2180
|
+
self._underlying.set_name(name)
|
2181
|
+
|
2182
|
+
@property
|
2183
|
+
def name(self):
|
2184
|
+
return self._underlying.name
|
2185
|
+
|
2186
|
+
@property
|
2187
|
+
def level(self):
|
2188
|
+
return self._underlying.level
|
2189
|
+
|
2190
|
+
@level.setter
|
2191
|
+
def level(self, level):
|
2192
|
+
self._underlying.level = level
|
2193
|
+
|
2194
|
+
@property
|
2195
|
+
def formatter(self):
|
2196
|
+
return self._underlying.formatter
|
2197
|
+
|
2198
|
+
@formatter.setter
|
2199
|
+
def formatter(self, formatter):
|
2200
|
+
self._underlying.formatter = formatter
|
2201
|
+
|
2202
|
+
def createLock(self):
|
2203
|
+
self._underlying.createLock()
|
2204
|
+
|
2205
|
+
def acquire(self):
|
2206
|
+
self._underlying.acquire()
|
2207
|
+
|
2208
|
+
def release(self):
|
2209
|
+
self._underlying.release()
|
2210
|
+
|
2211
|
+
def setLevel(self, level):
|
2212
|
+
self._underlying.setLevel(level)
|
2213
|
+
|
2214
|
+
def format(self, record):
|
2215
|
+
return self._underlying.format(record)
|
958
2216
|
|
959
2217
|
def emit(self, record):
|
960
2218
|
self._underlying.emit(record)
|
@@ -1067,21 +2325,26 @@ TODO:
|
|
1067
2325
|
##
|
1068
2326
|
|
1069
2327
|
|
2328
|
+
@dc.dataclass(frozen=True)
|
2329
|
+
class ObjMarshalOptions:
|
2330
|
+
raw_bytes: bool = False
|
2331
|
+
|
2332
|
+
|
1070
2333
|
class ObjMarshaler(abc.ABC):
|
1071
2334
|
@abc.abstractmethod
|
1072
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2335
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1073
2336
|
raise NotImplementedError
|
1074
2337
|
|
1075
2338
|
@abc.abstractmethod
|
1076
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2339
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1077
2340
|
raise NotImplementedError
|
1078
2341
|
|
1079
2342
|
|
1080
2343
|
class NopObjMarshaler(ObjMarshaler):
|
1081
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2344
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1082
2345
|
return o
|
1083
2346
|
|
1084
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2347
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1085
2348
|
return o
|
1086
2349
|
|
1087
2350
|
|
@@ -1089,29 +2352,29 @@ class NopObjMarshaler(ObjMarshaler):
|
|
1089
2352
|
class ProxyObjMarshaler(ObjMarshaler):
|
1090
2353
|
m: ta.Optional[ObjMarshaler] = None
|
1091
2354
|
|
1092
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
1093
|
-
return check_not_none(self.m).marshal(o)
|
2355
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2356
|
+
return check_not_none(self.m).marshal(o, opts)
|
1094
2357
|
|
1095
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
1096
|
-
return check_not_none(self.m).unmarshal(o)
|
2358
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2359
|
+
return check_not_none(self.m).unmarshal(o, opts)
|
1097
2360
|
|
1098
2361
|
|
1099
2362
|
@dc.dataclass(frozen=True)
|
1100
2363
|
class CastObjMarshaler(ObjMarshaler):
|
1101
2364
|
ty: type
|
1102
2365
|
|
1103
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2366
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1104
2367
|
return o
|
1105
2368
|
|
1106
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2369
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1107
2370
|
return self.ty(o)
|
1108
2371
|
|
1109
2372
|
|
1110
2373
|
class DynamicObjMarshaler(ObjMarshaler):
|
1111
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2374
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1112
2375
|
return marshal_obj(o)
|
1113
2376
|
|
1114
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2377
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1115
2378
|
return o
|
1116
2379
|
|
1117
2380
|
|
@@ -1119,21 +2382,36 @@ class DynamicObjMarshaler(ObjMarshaler):
|
|
1119
2382
|
class Base64ObjMarshaler(ObjMarshaler):
|
1120
2383
|
ty: type
|
1121
2384
|
|
1122
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2385
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1123
2386
|
return base64.b64encode(o).decode('ascii')
|
1124
2387
|
|
1125
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2388
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1126
2389
|
return self.ty(base64.b64decode(o))
|
1127
2390
|
|
1128
2391
|
|
2392
|
+
@dc.dataclass(frozen=True)
|
2393
|
+
class BytesSwitchedObjMarshaler(ObjMarshaler):
|
2394
|
+
m: ObjMarshaler
|
2395
|
+
|
2396
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2397
|
+
if opts.raw_bytes:
|
2398
|
+
return o
|
2399
|
+
return self.m.marshal(o, opts)
|
2400
|
+
|
2401
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2402
|
+
if opts.raw_bytes:
|
2403
|
+
return o
|
2404
|
+
return self.m.unmarshal(o, opts)
|
2405
|
+
|
2406
|
+
|
1129
2407
|
@dc.dataclass(frozen=True)
|
1130
2408
|
class EnumObjMarshaler(ObjMarshaler):
|
1131
2409
|
ty: type
|
1132
2410
|
|
1133
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2411
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1134
2412
|
return o.name
|
1135
2413
|
|
1136
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2414
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1137
2415
|
return self.ty.__members__[o] # type: ignore
|
1138
2416
|
|
1139
2417
|
|
@@ -1141,15 +2419,15 @@ class EnumObjMarshaler(ObjMarshaler):
|
|
1141
2419
|
class OptionalObjMarshaler(ObjMarshaler):
|
1142
2420
|
item: ObjMarshaler
|
1143
2421
|
|
1144
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2422
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1145
2423
|
if o is None:
|
1146
2424
|
return None
|
1147
|
-
return self.item.marshal(o)
|
2425
|
+
return self.item.marshal(o, opts)
|
1148
2426
|
|
1149
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2427
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1150
2428
|
if o is None:
|
1151
2429
|
return None
|
1152
|
-
return self.item.unmarshal(o)
|
2430
|
+
return self.item.unmarshal(o, opts)
|
1153
2431
|
|
1154
2432
|
|
1155
2433
|
@dc.dataclass(frozen=True)
|
@@ -1158,11 +2436,11 @@ class MappingObjMarshaler(ObjMarshaler):
|
|
1158
2436
|
km: ObjMarshaler
|
1159
2437
|
vm: ObjMarshaler
|
1160
2438
|
|
1161
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
1162
|
-
return {self.km.marshal(k): self.vm.marshal(v) for k, v in o.items()}
|
2439
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2440
|
+
return {self.km.marshal(k, opts): self.vm.marshal(v, opts) for k, v in o.items()}
|
1163
2441
|
|
1164
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
1165
|
-
return self.ty((self.km.unmarshal(k), self.vm.unmarshal(v)) for k, v in o.items())
|
2442
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2443
|
+
return self.ty((self.km.unmarshal(k, opts), self.vm.unmarshal(v, opts)) for k, v in o.items())
|
1166
2444
|
|
1167
2445
|
|
1168
2446
|
@dc.dataclass(frozen=True)
|
@@ -1170,11 +2448,11 @@ class IterableObjMarshaler(ObjMarshaler):
|
|
1170
2448
|
ty: type
|
1171
2449
|
item: ObjMarshaler
|
1172
2450
|
|
1173
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
1174
|
-
return [self.item.marshal(e) for e in o]
|
2451
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2452
|
+
return [self.item.marshal(e, opts) for e in o]
|
1175
2453
|
|
1176
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
1177
|
-
return self.ty(self.item.unmarshal(e) for e in o)
|
2454
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2455
|
+
return self.ty(self.item.unmarshal(e, opts) for e in o)
|
1178
2456
|
|
1179
2457
|
|
1180
2458
|
@dc.dataclass(frozen=True)
|
@@ -1183,11 +2461,11 @@ class DataclassObjMarshaler(ObjMarshaler):
|
|
1183
2461
|
fs: ta.Mapping[str, ObjMarshaler]
|
1184
2462
|
nonstrict: bool = False
|
1185
2463
|
|
1186
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
1187
|
-
return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
|
2464
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2465
|
+
return {k: m.marshal(getattr(o, k), opts) for k, m in self.fs.items()}
|
1188
2466
|
|
1189
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
1190
|
-
return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
|
2467
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
2468
|
+
return self.ty(**{k: self.fs[k].unmarshal(v, opts) for k, v in o.items() if not self.nonstrict or k in self.fs})
|
1191
2469
|
|
1192
2470
|
|
1193
2471
|
@dc.dataclass(frozen=True)
|
@@ -1207,50 +2485,50 @@ class PolymorphicObjMarshaler(ObjMarshaler):
|
|
1207
2485
|
{i.tag: i for i in impls},
|
1208
2486
|
)
|
1209
2487
|
|
1210
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2488
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1211
2489
|
impl = self.impls_by_ty[type(o)]
|
1212
|
-
return {impl.tag: impl.m.marshal(o)}
|
2490
|
+
return {impl.tag: impl.m.marshal(o, opts)}
|
1213
2491
|
|
1214
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2492
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1215
2493
|
[(t, v)] = o.items()
|
1216
2494
|
impl = self.impls_by_tag[t]
|
1217
|
-
return impl.m.unmarshal(v)
|
2495
|
+
return impl.m.unmarshal(v, opts)
|
1218
2496
|
|
1219
2497
|
|
1220
2498
|
@dc.dataclass(frozen=True)
|
1221
2499
|
class DatetimeObjMarshaler(ObjMarshaler):
|
1222
2500
|
ty: type
|
1223
2501
|
|
1224
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2502
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1225
2503
|
return o.isoformat()
|
1226
2504
|
|
1227
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2505
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1228
2506
|
return self.ty.fromisoformat(o) # type: ignore
|
1229
2507
|
|
1230
2508
|
|
1231
2509
|
class DecimalObjMarshaler(ObjMarshaler):
|
1232
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2510
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1233
2511
|
return str(check_isinstance(o, decimal.Decimal))
|
1234
2512
|
|
1235
|
-
def unmarshal(self, v: ta.Any) -> ta.Any:
|
2513
|
+
def unmarshal(self, v: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1236
2514
|
return decimal.Decimal(check_isinstance(v, str))
|
1237
2515
|
|
1238
2516
|
|
1239
2517
|
class FractionObjMarshaler(ObjMarshaler):
|
1240
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2518
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1241
2519
|
fr = check_isinstance(o, fractions.Fraction)
|
1242
2520
|
return [fr.numerator, fr.denominator]
|
1243
2521
|
|
1244
|
-
def unmarshal(self, v: ta.Any) -> ta.Any:
|
2522
|
+
def unmarshal(self, v: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1245
2523
|
num, denom = check_isinstance(v, list)
|
1246
2524
|
return fractions.Fraction(num, denom)
|
1247
2525
|
|
1248
2526
|
|
1249
2527
|
class UuidObjMarshaler(ObjMarshaler):
|
1250
|
-
def marshal(self, o: ta.Any) -> ta.Any:
|
2528
|
+
def marshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1251
2529
|
return str(o)
|
1252
2530
|
|
1253
|
-
def unmarshal(self, o: ta.Any) -> ta.Any:
|
2531
|
+
def unmarshal(self, o: ta.Any, opts: ObjMarshalOptions) -> ta.Any:
|
1254
2532
|
return uuid.UUID(o)
|
1255
2533
|
|
1256
2534
|
|
@@ -1260,7 +2538,7 @@ class UuidObjMarshaler(ObjMarshaler):
|
|
1260
2538
|
_DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
|
1261
2539
|
**{t: NopObjMarshaler() for t in (type(None),)},
|
1262
2540
|
**{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
|
1263
|
-
**{t: Base64ObjMarshaler(t) for t in (bytes, bytearray)},
|
2541
|
+
**{t: BytesSwitchedObjMarshaler(Base64ObjMarshaler(t)) for t in (bytes, bytearray)},
|
1264
2542
|
**{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
|
1265
2543
|
**{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
|
1266
2544
|
|
@@ -1277,146 +2555,313 @@ _OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
|
|
1277
2555
|
**{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
|
1278
2556
|
}
|
1279
2557
|
|
1280
|
-
_OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
|
1281
|
-
**{t: t for t in (list, tuple, set, frozenset)},
|
1282
|
-
collections.abc.Set: frozenset,
|
1283
|
-
collections.abc.MutableSet: set,
|
1284
|
-
collections.abc.Sequence: tuple,
|
1285
|
-
collections.abc.MutableSequence: list,
|
1286
|
-
}
|
2558
|
+
_OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
|
2559
|
+
**{t: t for t in (list, tuple, set, frozenset)},
|
2560
|
+
collections.abc.Set: frozenset,
|
2561
|
+
collections.abc.MutableSet: set,
|
2562
|
+
collections.abc.Sequence: tuple,
|
2563
|
+
collections.abc.MutableSequence: list,
|
2564
|
+
}
|
2565
|
+
|
2566
|
+
|
2567
|
+
##
|
2568
|
+
|
2569
|
+
|
2570
|
+
class ObjMarshalerManager:
|
2571
|
+
def __init__(
|
2572
|
+
self,
|
2573
|
+
*,
|
2574
|
+
default_options: ObjMarshalOptions = ObjMarshalOptions(),
|
2575
|
+
|
2576
|
+
default_obj_marshalers: ta.Dict[ta.Any, ObjMarshaler] = _DEFAULT_OBJ_MARSHALERS, # noqa
|
2577
|
+
generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES, # noqa
|
2578
|
+
generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES, # noqa
|
2579
|
+
) -> None:
|
2580
|
+
super().__init__()
|
2581
|
+
|
2582
|
+
self._default_options = default_options
|
2583
|
+
|
2584
|
+
self._obj_marshalers = dict(default_obj_marshalers)
|
2585
|
+
self._generic_mapping_types = generic_mapping_types
|
2586
|
+
self._generic_iterable_types = generic_iterable_types
|
2587
|
+
|
2588
|
+
self._lock = threading.RLock()
|
2589
|
+
self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
|
2590
|
+
self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
|
2591
|
+
|
2592
|
+
#
|
2593
|
+
|
2594
|
+
def make_obj_marshaler(
|
2595
|
+
self,
|
2596
|
+
ty: ta.Any,
|
2597
|
+
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
2598
|
+
*,
|
2599
|
+
nonstrict_dataclasses: bool = False,
|
2600
|
+
) -> ObjMarshaler:
|
2601
|
+
if isinstance(ty, type):
|
2602
|
+
if abc.ABC in ty.__bases__:
|
2603
|
+
return PolymorphicObjMarshaler.of([ # type: ignore
|
2604
|
+
PolymorphicObjMarshaler.Impl(
|
2605
|
+
ity,
|
2606
|
+
ity.__qualname__,
|
2607
|
+
rec(ity),
|
2608
|
+
)
|
2609
|
+
for ity in deep_subclasses(ty)
|
2610
|
+
if abc.ABC not in ity.__bases__
|
2611
|
+
])
|
2612
|
+
|
2613
|
+
if issubclass(ty, enum.Enum):
|
2614
|
+
return EnumObjMarshaler(ty)
|
2615
|
+
|
2616
|
+
if dc.is_dataclass(ty):
|
2617
|
+
return DataclassObjMarshaler(
|
2618
|
+
ty,
|
2619
|
+
{f.name: rec(f.type) for f in dc.fields(ty)},
|
2620
|
+
nonstrict=nonstrict_dataclasses,
|
2621
|
+
)
|
2622
|
+
|
2623
|
+
if is_generic_alias(ty):
|
2624
|
+
try:
|
2625
|
+
mt = self._generic_mapping_types[ta.get_origin(ty)]
|
2626
|
+
except KeyError:
|
2627
|
+
pass
|
2628
|
+
else:
|
2629
|
+
k, v = ta.get_args(ty)
|
2630
|
+
return MappingObjMarshaler(mt, rec(k), rec(v))
|
2631
|
+
|
2632
|
+
try:
|
2633
|
+
st = self._generic_iterable_types[ta.get_origin(ty)]
|
2634
|
+
except KeyError:
|
2635
|
+
pass
|
2636
|
+
else:
|
2637
|
+
[e] = ta.get_args(ty)
|
2638
|
+
return IterableObjMarshaler(st, rec(e))
|
2639
|
+
|
2640
|
+
if is_union_alias(ty):
|
2641
|
+
return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
|
2642
|
+
|
2643
|
+
raise TypeError(ty)
|
2644
|
+
|
2645
|
+
#
|
2646
|
+
|
2647
|
+
def register_opj_marshaler(self, ty: ta.Any, m: ObjMarshaler) -> None:
|
2648
|
+
with self._lock:
|
2649
|
+
if ty in self._obj_marshalers:
|
2650
|
+
raise KeyError(ty)
|
2651
|
+
self._obj_marshalers[ty] = m
|
2652
|
+
|
2653
|
+
def get_obj_marshaler(
|
2654
|
+
self,
|
2655
|
+
ty: ta.Any,
|
2656
|
+
*,
|
2657
|
+
no_cache: bool = False,
|
2658
|
+
**kwargs: ta.Any,
|
2659
|
+
) -> ObjMarshaler:
|
2660
|
+
with self._lock:
|
2661
|
+
if not no_cache:
|
2662
|
+
try:
|
2663
|
+
return self._obj_marshalers[ty]
|
2664
|
+
except KeyError:
|
2665
|
+
pass
|
2666
|
+
|
2667
|
+
try:
|
2668
|
+
return self._proxies[ty]
|
2669
|
+
except KeyError:
|
2670
|
+
pass
|
2671
|
+
|
2672
|
+
rec = functools.partial(
|
2673
|
+
self.get_obj_marshaler,
|
2674
|
+
no_cache=no_cache,
|
2675
|
+
**kwargs,
|
2676
|
+
)
|
2677
|
+
|
2678
|
+
p = ProxyObjMarshaler()
|
2679
|
+
self._proxies[ty] = p
|
2680
|
+
try:
|
2681
|
+
m = self.make_obj_marshaler(ty, rec, **kwargs)
|
2682
|
+
finally:
|
2683
|
+
del self._proxies[ty]
|
2684
|
+
p.m = m
|
2685
|
+
|
2686
|
+
if not no_cache:
|
2687
|
+
self._obj_marshalers[ty] = m
|
2688
|
+
return m
|
2689
|
+
|
2690
|
+
#
|
2691
|
+
|
2692
|
+
def marshal_obj(
|
2693
|
+
self,
|
2694
|
+
o: ta.Any,
|
2695
|
+
ty: ta.Any = None,
|
2696
|
+
opts: ta.Optional[ObjMarshalOptions] = None,
|
2697
|
+
) -> ta.Any:
|
2698
|
+
m = self.get_obj_marshaler(ty if ty is not None else type(o))
|
2699
|
+
return m.marshal(o, opts or self._default_options)
|
2700
|
+
|
2701
|
+
def unmarshal_obj(
|
2702
|
+
self,
|
2703
|
+
o: ta.Any,
|
2704
|
+
ty: ta.Union[ta.Type[T], ta.Any],
|
2705
|
+
opts: ta.Optional[ObjMarshalOptions] = None,
|
2706
|
+
) -> T:
|
2707
|
+
m = self.get_obj_marshaler(ty)
|
2708
|
+
return m.unmarshal(o, opts or self._default_options)
|
2709
|
+
|
2710
|
+
def roundtrip_obj(
|
2711
|
+
self,
|
2712
|
+
o: ta.Any,
|
2713
|
+
ty: ta.Any = None,
|
2714
|
+
opts: ta.Optional[ObjMarshalOptions] = None,
|
2715
|
+
) -> ta.Any:
|
2716
|
+
if ty is None:
|
2717
|
+
ty = type(o)
|
2718
|
+
m: ta.Any = self.marshal_obj(o, ty, opts)
|
2719
|
+
u: ta.Any = self.unmarshal_obj(m, ty, opts)
|
2720
|
+
return u
|
2721
|
+
|
2722
|
+
|
2723
|
+
##
|
2724
|
+
|
2725
|
+
|
2726
|
+
OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
|
2727
|
+
|
2728
|
+
register_opj_marshaler = OBJ_MARSHALER_MANAGER.register_opj_marshaler
|
2729
|
+
get_obj_marshaler = OBJ_MARSHALER_MANAGER.get_obj_marshaler
|
2730
|
+
|
2731
|
+
marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
|
2732
|
+
unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj
|
2733
|
+
|
2734
|
+
|
2735
|
+
########################################
|
2736
|
+
# ../../../omlish/lite/runtime.py
|
2737
|
+
|
2738
|
+
|
2739
|
+
@cached_nullary
|
2740
|
+
def is_debugger_attached() -> bool:
|
2741
|
+
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
2742
|
+
|
2743
|
+
|
2744
|
+
REQUIRED_PYTHON_VERSION = (3, 8)
|
2745
|
+
|
2746
|
+
|
2747
|
+
def check_runtime_version() -> None:
|
2748
|
+
if sys.version_info < REQUIRED_PYTHON_VERSION:
|
2749
|
+
raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
|
1287
2750
|
|
1288
2751
|
|
1289
|
-
|
1290
|
-
|
1291
|
-
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
1292
|
-
*,
|
1293
|
-
nonstrict_dataclasses: bool = False,
|
1294
|
-
) -> ObjMarshaler:
|
1295
|
-
if isinstance(ty, type):
|
1296
|
-
if abc.ABC in ty.__bases__:
|
1297
|
-
return PolymorphicObjMarshaler.of([ # type: ignore
|
1298
|
-
PolymorphicObjMarshaler.Impl(
|
1299
|
-
ity,
|
1300
|
-
ity.__qualname__,
|
1301
|
-
rec(ity),
|
1302
|
-
)
|
1303
|
-
for ity in deep_subclasses(ty)
|
1304
|
-
if abc.ABC not in ity.__bases__
|
1305
|
-
])
|
1306
|
-
|
1307
|
-
if issubclass(ty, enum.Enum):
|
1308
|
-
return EnumObjMarshaler(ty)
|
1309
|
-
|
1310
|
-
if dc.is_dataclass(ty):
|
1311
|
-
return DataclassObjMarshaler(
|
1312
|
-
ty,
|
1313
|
-
{f.name: rec(f.type) for f in dc.fields(ty)},
|
1314
|
-
nonstrict=nonstrict_dataclasses,
|
1315
|
-
)
|
2752
|
+
########################################
|
2753
|
+
# ../bootstrap.py
|
1316
2754
|
|
1317
|
-
if is_generic_alias(ty):
|
1318
|
-
try:
|
1319
|
-
mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
|
1320
|
-
except KeyError:
|
1321
|
-
pass
|
1322
|
-
else:
|
1323
|
-
k, v = ta.get_args(ty)
|
1324
|
-
return MappingObjMarshaler(mt, rec(k), rec(v))
|
1325
2755
|
|
1326
|
-
|
1327
|
-
|
1328
|
-
|
1329
|
-
pass
|
1330
|
-
else:
|
1331
|
-
[e] = ta.get_args(ty)
|
1332
|
-
return IterableObjMarshaler(st, rec(e))
|
2756
|
+
@dc.dataclass(frozen=True)
|
2757
|
+
class MainBootstrap:
|
2758
|
+
main_config: MainConfig = MainConfig()
|
1333
2759
|
|
1334
|
-
|
1335
|
-
return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
|
2760
|
+
remote_config: RemoteConfig = RemoteConfig()
|
1336
2761
|
|
1337
|
-
raise TypeError(ty)
|
1338
2762
|
|
2763
|
+
########################################
|
2764
|
+
# ../commands/execution.py
|
1339
2765
|
|
1340
|
-
##
|
1341
2766
|
|
2767
|
+
CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
|
1342
2768
|
|
1343
|
-
_OBJ_MARSHALERS_LOCK = threading.RLock()
|
1344
2769
|
|
1345
|
-
|
2770
|
+
class CommandExecutionService(CommandExecutor):
|
2771
|
+
def __init__(
|
2772
|
+
self,
|
2773
|
+
*,
|
2774
|
+
command_executors: CommandExecutorMap,
|
2775
|
+
) -> None:
|
2776
|
+
super().__init__()
|
1346
2777
|
|
1347
|
-
|
2778
|
+
self._command_executors = command_executors
|
1348
2779
|
|
2780
|
+
def execute(self, cmd: Command) -> Command.Output:
|
2781
|
+
ce: CommandExecutor = self._command_executors[type(cmd)]
|
2782
|
+
return ce.execute(cmd)
|
1349
2783
|
|
1350
|
-
def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
|
1351
|
-
with _OBJ_MARSHALERS_LOCK:
|
1352
|
-
if ty in _OBJ_MARSHALERS:
|
1353
|
-
raise KeyError(ty)
|
1354
|
-
_OBJ_MARSHALERS[ty] = m
|
1355
2784
|
|
2785
|
+
########################################
|
2786
|
+
# ../commands/marshal.py
|
1356
2787
|
|
1357
|
-
def get_obj_marshaler(
|
1358
|
-
ty: ta.Any,
|
1359
|
-
*,
|
1360
|
-
no_cache: bool = False,
|
1361
|
-
**kwargs: ta.Any,
|
1362
|
-
) -> ObjMarshaler:
|
1363
|
-
with _OBJ_MARSHALERS_LOCK:
|
1364
|
-
if not no_cache:
|
1365
|
-
try:
|
1366
|
-
return _OBJ_MARSHALERS[ty]
|
1367
|
-
except KeyError:
|
1368
|
-
pass
|
1369
2788
|
|
1370
|
-
|
1371
|
-
|
1372
|
-
|
1373
|
-
|
1374
|
-
|
1375
|
-
|
1376
|
-
|
1377
|
-
|
1378
|
-
|
2789
|
+
def install_command_marshaling(
|
2790
|
+
cmds: CommandNameMap,
|
2791
|
+
msh: ObjMarshalerManager,
|
2792
|
+
) -> None:
|
2793
|
+
for fn in [
|
2794
|
+
lambda c: c,
|
2795
|
+
lambda c: c.Output,
|
2796
|
+
]:
|
2797
|
+
msh.register_opj_marshaler(
|
2798
|
+
fn(Command),
|
2799
|
+
PolymorphicObjMarshaler.of([
|
2800
|
+
PolymorphicObjMarshaler.Impl(
|
2801
|
+
fn(cmd),
|
2802
|
+
name,
|
2803
|
+
msh.get_obj_marshaler(fn(cmd)),
|
2804
|
+
)
|
2805
|
+
for name, cmd in cmds.items()
|
2806
|
+
]),
|
1379
2807
|
)
|
1380
2808
|
|
1381
|
-
p = ProxyObjMarshaler()
|
1382
|
-
_OBJ_MARSHALER_PROXIES[ty] = p
|
1383
|
-
try:
|
1384
|
-
m = _make_obj_marshaler(ty, rec, **kwargs)
|
1385
|
-
finally:
|
1386
|
-
del _OBJ_MARSHALER_PROXIES[ty]
|
1387
|
-
p.m = m
|
1388
2809
|
|
1389
|
-
|
1390
|
-
|
1391
|
-
return m
|
2810
|
+
########################################
|
2811
|
+
# ../marshal.py
|
1392
2812
|
|
1393
2813
|
|
1394
|
-
|
2814
|
+
@dc.dataclass(frozen=True)
|
2815
|
+
class ObjMarshalerInstaller:
|
2816
|
+
fn: ta.Callable[[ObjMarshalerManager], None]
|
1395
2817
|
|
1396
2818
|
|
1397
|
-
|
1398
|
-
return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
|
2819
|
+
ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMarshalerInstaller])
|
1399
2820
|
|
1400
2821
|
|
1401
|
-
|
1402
|
-
|
2822
|
+
########################################
|
2823
|
+
# ../remote/channel.py
|
1403
2824
|
|
1404
2825
|
|
1405
|
-
|
1406
|
-
|
2826
|
+
class RemoteChannel:
|
2827
|
+
def __init__(
|
2828
|
+
self,
|
2829
|
+
input: ta.IO, # noqa
|
2830
|
+
output: ta.IO,
|
2831
|
+
*,
|
2832
|
+
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
2833
|
+
) -> None:
|
2834
|
+
super().__init__()
|
1407
2835
|
|
2836
|
+
self._input = input
|
2837
|
+
self._output = output
|
2838
|
+
self._msh = msh
|
1408
2839
|
|
1409
|
-
|
1410
|
-
|
1411
|
-
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
2840
|
+
def set_marshaler(self, msh: ObjMarshalerManager) -> None:
|
2841
|
+
self._msh = msh
|
1412
2842
|
|
2843
|
+
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
2844
|
+
j = json_dumps_compact(self._msh.marshal_obj(o, ty))
|
2845
|
+
d = j.encode('utf-8')
|
1413
2846
|
|
1414
|
-
|
2847
|
+
self._output.write(struct.pack('<I', len(d)))
|
2848
|
+
self._output.write(d)
|
2849
|
+
self._output.flush()
|
1415
2850
|
|
2851
|
+
def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
2852
|
+
d = self._input.read(4)
|
2853
|
+
if not d:
|
2854
|
+
return None
|
2855
|
+
if len(d) != 4:
|
2856
|
+
raise EOFError
|
1416
2857
|
|
1417
|
-
|
1418
|
-
|
1419
|
-
|
2858
|
+
sz = struct.unpack('<I', d)[0]
|
2859
|
+
d = self._input.read(sz)
|
2860
|
+
if len(d) != sz:
|
2861
|
+
raise EOFError
|
2862
|
+
|
2863
|
+
j = json.loads(d.decode('utf-8'))
|
2864
|
+
return self._msh.unmarshal_obj(j, ty)
|
1420
2865
|
|
1421
2866
|
|
1422
2867
|
########################################
|
@@ -1426,6 +2871,16 @@ def check_runtime_version() -> None:
|
|
1426
2871
|
##
|
1427
2872
|
|
1428
2873
|
|
2874
|
+
SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
|
2875
|
+
'pipe': subprocess.PIPE,
|
2876
|
+
'stdout': subprocess.STDOUT,
|
2877
|
+
'devnull': subprocess.DEVNULL,
|
2878
|
+
}
|
2879
|
+
|
2880
|
+
|
2881
|
+
##
|
2882
|
+
|
2883
|
+
|
1429
2884
|
_SUBPROCESS_SHELL_WRAP_EXECS = False
|
1430
2885
|
|
1431
2886
|
|
@@ -1552,21 +3007,21 @@ def subprocess_close(
|
|
1552
3007
|
|
1553
3008
|
@dc.dataclass(frozen=True)
|
1554
3009
|
class SubprocessCommand(Command['SubprocessCommand.Output']):
|
1555
|
-
|
3010
|
+
cmd: ta.Sequence[str]
|
1556
3011
|
|
1557
3012
|
shell: bool = False
|
1558
3013
|
cwd: ta.Optional[str] = None
|
1559
3014
|
env: ta.Optional[ta.Mapping[str, str]] = None
|
1560
3015
|
|
1561
|
-
|
1562
|
-
|
3016
|
+
stdout: str = 'pipe' # SubprocessChannelOption
|
3017
|
+
stderr: str = 'pipe' # SubprocessChannelOption
|
1563
3018
|
|
1564
3019
|
input: ta.Optional[bytes] = None
|
1565
3020
|
timeout: ta.Optional[float] = None
|
1566
3021
|
|
1567
3022
|
def __post_init__(self) -> None:
|
1568
|
-
if isinstance(self.
|
1569
|
-
raise TypeError(self.
|
3023
|
+
if isinstance(self.cmd, str):
|
3024
|
+
raise TypeError(self.cmd)
|
1570
3025
|
|
1571
3026
|
@dc.dataclass(frozen=True)
|
1572
3027
|
class Output(Command.Output):
|
@@ -1584,24 +3039,23 @@ class SubprocessCommand(Command['SubprocessCommand.Output']):
|
|
1584
3039
|
|
1585
3040
|
class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
|
1586
3041
|
def execute(self, inp: SubprocessCommand) -> SubprocessCommand.Output:
|
1587
|
-
|
1588
|
-
subprocess_maybe_shell_wrap_exec(*inp.
|
3042
|
+
with subprocess.Popen(
|
3043
|
+
subprocess_maybe_shell_wrap_exec(*inp.cmd),
|
1589
3044
|
|
1590
3045
|
shell=inp.shell,
|
1591
3046
|
cwd=inp.cwd,
|
1592
3047
|
env={**os.environ, **(inp.env or {})},
|
1593
3048
|
|
1594
3049
|
stdin=subprocess.PIPE if inp.input is not None else None,
|
1595
|
-
stdout=
|
1596
|
-
stderr=
|
1597
|
-
)
|
1598
|
-
|
1599
|
-
|
1600
|
-
|
1601
|
-
|
1602
|
-
|
1603
|
-
|
1604
|
-
end_time = time.time()
|
3050
|
+
stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stdout)],
|
3051
|
+
stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stderr)],
|
3052
|
+
) as proc:
|
3053
|
+
start_time = time.time()
|
3054
|
+
stdout, stderr = proc.communicate(
|
3055
|
+
input=inp.input,
|
3056
|
+
timeout=inp.timeout,
|
3057
|
+
)
|
3058
|
+
end_time = time.time()
|
1605
3059
|
|
1606
3060
|
return SubprocessCommand.Output(
|
1607
3061
|
rc=proc.returncode,
|
@@ -1615,28 +3069,19 @@ class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCom
|
|
1615
3069
|
|
1616
3070
|
|
1617
3071
|
########################################
|
1618
|
-
# ../spawning.py
|
3072
|
+
# ../remote/spawning.py
|
1619
3073
|
|
1620
3074
|
|
1621
|
-
class
|
1622
|
-
|
3075
|
+
class RemoteSpawning:
|
3076
|
+
@dc.dataclass(frozen=True)
|
3077
|
+
class Target:
|
3078
|
+
shell: ta.Optional[str] = None
|
3079
|
+
shell_quote: bool = False
|
1623
3080
|
|
1624
|
-
|
1625
|
-
|
1626
|
-
src: str,
|
1627
|
-
*,
|
1628
|
-
shell: ta.Optional[str] = None,
|
1629
|
-
shell_quote: bool = False,
|
1630
|
-
python: str = DEFAULT_PYTHON,
|
1631
|
-
stderr: ta.Optional[ta.Literal['pipe', 'stdout', 'devnull']] = None,
|
1632
|
-
) -> None:
|
1633
|
-
super().__init__()
|
3081
|
+
DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
|
3082
|
+
python: str = DEFAULT_PYTHON
|
1634
3083
|
|
1635
|
-
|
1636
|
-
self._shell = shell
|
1637
|
-
self._shell_quote = shell_quote
|
1638
|
-
self._python = python
|
1639
|
-
self._stderr = stderr
|
3084
|
+
stderr: ta.Optional[str] = None # SubprocessChannelOption
|
1640
3085
|
|
1641
3086
|
#
|
1642
3087
|
|
@@ -1644,31 +3089,29 @@ class PySpawner:
|
|
1644
3089
|
cmd: ta.Sequence[str]
|
1645
3090
|
shell: bool
|
1646
3091
|
|
1647
|
-
def _prepare_cmd(
|
1648
|
-
|
1649
|
-
|
1650
|
-
|
3092
|
+
def _prepare_cmd(
|
3093
|
+
self,
|
3094
|
+
tgt: Target,
|
3095
|
+
src: str,
|
3096
|
+
) -> _PreparedCmd:
|
3097
|
+
if tgt.shell is not None:
|
3098
|
+
sh_src = f'{tgt.python} -c {shlex.quote(src)}'
|
3099
|
+
if tgt.shell_quote:
|
1651
3100
|
sh_src = shlex.quote(sh_src)
|
1652
|
-
sh_cmd = f'{
|
1653
|
-
return
|
3101
|
+
sh_cmd = f'{tgt.shell} {sh_src}'
|
3102
|
+
return RemoteSpawning._PreparedCmd(
|
1654
3103
|
cmd=[sh_cmd],
|
1655
3104
|
shell=True,
|
1656
3105
|
)
|
1657
3106
|
|
1658
3107
|
else:
|
1659
|
-
return
|
1660
|
-
cmd=[
|
3108
|
+
return RemoteSpawning._PreparedCmd(
|
3109
|
+
cmd=[tgt.python, '-c', src],
|
1661
3110
|
shell=False,
|
1662
3111
|
)
|
1663
3112
|
|
1664
3113
|
#
|
1665
3114
|
|
1666
|
-
_STDERR_KWARG_MAP: ta.Mapping[str, int] = {
|
1667
|
-
'pipe': subprocess.PIPE,
|
1668
|
-
'stdout': subprocess.STDOUT,
|
1669
|
-
'devnull': subprocess.DEVNULL,
|
1670
|
-
}
|
1671
|
-
|
1672
3115
|
@dc.dataclass(frozen=True)
|
1673
3116
|
class Spawned:
|
1674
3117
|
stdin: ta.IO
|
@@ -1678,23 +3121,28 @@ class PySpawner:
|
|
1678
3121
|
@contextlib.contextmanager
|
1679
3122
|
def spawn(
|
1680
3123
|
self,
|
3124
|
+
tgt: Target,
|
3125
|
+
src: str,
|
1681
3126
|
*,
|
1682
3127
|
timeout: ta.Optional[float] = None,
|
1683
3128
|
) -> ta.Generator[Spawned, None, None]:
|
1684
|
-
pc = self._prepare_cmd()
|
3129
|
+
pc = self._prepare_cmd(tgt, src)
|
1685
3130
|
|
1686
3131
|
with subprocess.Popen(
|
1687
3132
|
subprocess_maybe_shell_wrap_exec(*pc.cmd),
|
1688
3133
|
shell=pc.shell,
|
1689
3134
|
stdin=subprocess.PIPE,
|
1690
3135
|
stdout=subprocess.PIPE,
|
1691
|
-
stderr=
|
3136
|
+
stderr=(
|
3137
|
+
SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, tgt.stderr)]
|
3138
|
+
if tgt.stderr is not None else None
|
3139
|
+
),
|
1692
3140
|
) as proc:
|
1693
3141
|
stdin = check_not_none(proc.stdin)
|
1694
3142
|
stdout = check_not_none(proc.stdout)
|
1695
3143
|
|
1696
3144
|
try:
|
1697
|
-
yield
|
3145
|
+
yield RemoteSpawning.Spawned(
|
1698
3146
|
stdin=stdin,
|
1699
3147
|
stdout=stdout,
|
1700
3148
|
stderr=proc.stderr,
|
@@ -1710,81 +3158,356 @@ class PySpawner:
|
|
1710
3158
|
|
1711
3159
|
|
1712
3160
|
########################################
|
1713
|
-
#
|
3161
|
+
# ../commands/inject.py
|
1714
3162
|
|
1715
3163
|
|
1716
3164
|
##
|
1717
3165
|
|
1718
3166
|
|
1719
|
-
|
1720
|
-
|
1721
|
-
|
3167
|
+
def bind_command(
|
3168
|
+
command_cls: ta.Type[Command],
|
3169
|
+
executor_cls: ta.Optional[ta.Type[CommandExecutor]],
|
3170
|
+
) -> InjectorBindings:
|
3171
|
+
lst: ta.List[InjectorBindingOrBindings] = [
|
3172
|
+
inj.bind(CommandRegistration(command_cls), array=True),
|
3173
|
+
]
|
1722
3174
|
|
3175
|
+
if executor_cls is not None:
|
3176
|
+
lst.extend([
|
3177
|
+
inj.bind(executor_cls, singleton=True),
|
3178
|
+
inj.bind(CommandExecutorRegistration(command_cls, executor_cls), array=True),
|
3179
|
+
])
|
1723
3180
|
|
1724
|
-
|
1725
|
-
def _register_command_marshaling() -> None:
|
1726
|
-
for fn in [
|
1727
|
-
lambda c: c,
|
1728
|
-
lambda c: c.Output,
|
1729
|
-
]:
|
1730
|
-
register_opj_marshaler(
|
1731
|
-
fn(Command),
|
1732
|
-
PolymorphicObjMarshaler.of([
|
1733
|
-
PolymorphicObjMarshaler.Impl(
|
1734
|
-
fn(cty),
|
1735
|
-
k,
|
1736
|
-
get_obj_marshaler(fn(cty)),
|
1737
|
-
)
|
1738
|
-
for k, cty in _COMMAND_TYPES.items()
|
1739
|
-
]),
|
1740
|
-
)
|
3181
|
+
return inj.as_bindings(*lst)
|
1741
3182
|
|
1742
3183
|
|
1743
3184
|
##
|
1744
3185
|
|
1745
3186
|
|
1746
|
-
|
1747
|
-
|
1748
|
-
|
3187
|
+
@dc.dataclass(frozen=True)
|
3188
|
+
class _FactoryCommandExecutor(CommandExecutor):
|
3189
|
+
factory: ta.Callable[[], CommandExecutor]
|
3190
|
+
|
3191
|
+
def execute(self, i: Command) -> Command.Output:
|
3192
|
+
return self.factory().execute(i)
|
3193
|
+
|
1749
3194
|
|
1750
|
-
|
1751
|
-
f.write(d)
|
1752
|
-
f.flush()
|
3195
|
+
##
|
1753
3196
|
|
1754
3197
|
|
1755
|
-
def
|
1756
|
-
|
1757
|
-
|
1758
|
-
|
1759
|
-
|
1760
|
-
|
3198
|
+
def bind_commands(
|
3199
|
+
*,
|
3200
|
+
main_config: MainConfig,
|
3201
|
+
) -> InjectorBindings:
|
3202
|
+
lst: ta.List[InjectorBindingOrBindings] = [
|
3203
|
+
inj.bind_array(CommandRegistration),
|
3204
|
+
inj.bind_array_type(CommandRegistration, CommandRegistrations),
|
3205
|
+
|
3206
|
+
inj.bind_array(CommandExecutorRegistration),
|
3207
|
+
inj.bind_array_type(CommandExecutorRegistration, CommandExecutorRegistrations),
|
3208
|
+
|
3209
|
+
inj.bind(build_command_name_map, singleton=True),
|
3210
|
+
]
|
3211
|
+
|
3212
|
+
#
|
3213
|
+
|
3214
|
+
def provide_obj_marshaler_installer(cmds: CommandNameMap) -> ObjMarshalerInstaller:
|
3215
|
+
return ObjMarshalerInstaller(functools.partial(install_command_marshaling, cmds))
|
3216
|
+
|
3217
|
+
lst.append(inj.bind(provide_obj_marshaler_installer, array=True))
|
3218
|
+
|
3219
|
+
#
|
3220
|
+
|
3221
|
+
def provide_command_executor_map(
|
3222
|
+
injector: Injector,
|
3223
|
+
crs: CommandExecutorRegistrations,
|
3224
|
+
) -> CommandExecutorMap:
|
3225
|
+
dct: ta.Dict[ta.Type[Command], CommandExecutor] = {}
|
3226
|
+
|
3227
|
+
cr: CommandExecutorRegistration
|
3228
|
+
for cr in crs:
|
3229
|
+
if cr.command_cls in dct:
|
3230
|
+
raise KeyError(cr.command_cls)
|
3231
|
+
|
3232
|
+
factory = functools.partial(injector.provide, cr.executor_cls)
|
3233
|
+
if main_config.debug:
|
3234
|
+
ce = factory()
|
3235
|
+
else:
|
3236
|
+
ce = _FactoryCommandExecutor(factory)
|
3237
|
+
|
3238
|
+
dct[cr.command_cls] = ce
|
3239
|
+
|
3240
|
+
return CommandExecutorMap(dct)
|
1761
3241
|
|
1762
|
-
|
1763
|
-
|
1764
|
-
if len(d) != sz:
|
1765
|
-
raise EOFError
|
3242
|
+
lst.extend([
|
3243
|
+
inj.bind(provide_command_executor_map, singleton=True),
|
1766
3244
|
|
1767
|
-
|
1768
|
-
|
3245
|
+
inj.bind(CommandExecutionService, singleton=True, eager=main_config.debug),
|
3246
|
+
inj.bind(CommandExecutor, to_key=CommandExecutionService),
|
3247
|
+
])
|
3248
|
+
|
3249
|
+
#
|
3250
|
+
|
3251
|
+
for command_cls, executor_cls in [
|
3252
|
+
(SubprocessCommand, SubprocessCommandExecutor),
|
3253
|
+
]:
|
3254
|
+
lst.append(bind_command(command_cls, executor_cls))
|
3255
|
+
|
3256
|
+
#
|
3257
|
+
|
3258
|
+
return inj.as_bindings(*lst)
|
3259
|
+
|
3260
|
+
|
3261
|
+
########################################
|
3262
|
+
# ../remote/execution.py
|
1769
3263
|
|
1770
3264
|
|
1771
3265
|
##
|
1772
3266
|
|
1773
3267
|
|
1774
|
-
def
|
3268
|
+
def _remote_execution_main() -> None:
|
1775
3269
|
rt = pyremote_bootstrap_finalize() # noqa
|
1776
3270
|
|
3271
|
+
chan = RemoteChannel(
|
3272
|
+
rt.input,
|
3273
|
+
rt.output,
|
3274
|
+
)
|
3275
|
+
|
3276
|
+
bs = check_not_none(chan.recv_obj(MainBootstrap))
|
3277
|
+
|
3278
|
+
if (prd := bs.remote_config.pycharm_remote_debug) is not None:
|
3279
|
+
pycharm_debug_connect(prd)
|
3280
|
+
|
3281
|
+
injector = main_bootstrap(bs)
|
3282
|
+
|
3283
|
+
chan.set_marshaler(injector[ObjMarshalerManager])
|
3284
|
+
|
3285
|
+
ce = injector[CommandExecutor]
|
3286
|
+
|
1777
3287
|
while True:
|
1778
|
-
i =
|
3288
|
+
i = chan.recv_obj(Command)
|
1779
3289
|
if i is None:
|
1780
3290
|
break
|
1781
3291
|
|
1782
|
-
|
1783
|
-
|
3292
|
+
r = ce.try_execute(
|
3293
|
+
i,
|
3294
|
+
log=log,
|
3295
|
+
omit_exc_object=True,
|
3296
|
+
)
|
3297
|
+
|
3298
|
+
chan.send_obj(r)
|
3299
|
+
|
3300
|
+
|
3301
|
+
##
|
3302
|
+
|
3303
|
+
|
3304
|
+
@dc.dataclass()
|
3305
|
+
class RemoteCommandError(Exception):
|
3306
|
+
e: CommandException
|
3307
|
+
|
3308
|
+
|
3309
|
+
class RemoteCommandExecutor(CommandExecutor):
|
3310
|
+
def __init__(self, chan: RemoteChannel) -> None:
|
3311
|
+
super().__init__()
|
3312
|
+
|
3313
|
+
self._chan = chan
|
3314
|
+
|
3315
|
+
def _remote_execute(self, cmd: Command) -> CommandOutputOrException:
|
3316
|
+
self._chan.send_obj(cmd, Command)
|
3317
|
+
|
3318
|
+
if (r := self._chan.recv_obj(CommandOutputOrExceptionData)) is None:
|
3319
|
+
raise EOFError
|
3320
|
+
|
3321
|
+
return r
|
3322
|
+
|
3323
|
+
# @ta.override
|
3324
|
+
def execute(self, cmd: Command) -> Command.Output:
|
3325
|
+
r = self._remote_execute(cmd)
|
3326
|
+
if (e := r.exception) is not None:
|
3327
|
+
raise RemoteCommandError(e)
|
3328
|
+
else:
|
3329
|
+
return check_not_none(r.output)
|
3330
|
+
|
3331
|
+
# @ta.override
|
3332
|
+
def try_execute(
|
3333
|
+
self,
|
3334
|
+
cmd: Command,
|
3335
|
+
*,
|
3336
|
+
log: ta.Optional[logging.Logger] = None,
|
3337
|
+
omit_exc_object: bool = False,
|
3338
|
+
) -> CommandOutputOrException:
|
3339
|
+
try:
|
3340
|
+
r = self._remote_execute(cmd)
|
3341
|
+
|
3342
|
+
except Exception as e: # noqa
|
3343
|
+
if log is not None:
|
3344
|
+
log.exception('Exception executing remote command: %r', type(cmd))
|
3345
|
+
|
3346
|
+
return CommandOutputOrExceptionData(exception=CommandException.of(
|
3347
|
+
e,
|
3348
|
+
omit_exc_object=omit_exc_object,
|
3349
|
+
cmd=cmd,
|
3350
|
+
))
|
3351
|
+
|
1784
3352
|
else:
|
1785
|
-
|
3353
|
+
return r
|
3354
|
+
|
3355
|
+
|
3356
|
+
##
|
3357
|
+
|
3358
|
+
|
3359
|
+
class RemoteExecution:
|
3360
|
+
def __init__(
|
3361
|
+
self,
|
3362
|
+
*,
|
3363
|
+
spawning: RemoteSpawning,
|
3364
|
+
msh: ObjMarshalerManager,
|
3365
|
+
payload_file: ta.Optional[RemoteExecutionPayloadFile] = None,
|
3366
|
+
) -> None:
|
3367
|
+
super().__init__()
|
3368
|
+
|
3369
|
+
self._spawning = spawning
|
3370
|
+
self._msh = msh
|
3371
|
+
self._payload_file = payload_file
|
3372
|
+
|
3373
|
+
#
|
3374
|
+
|
3375
|
+
@cached_nullary
|
3376
|
+
def _payload_src(self) -> str:
|
3377
|
+
return get_remote_payload_src(file=self._payload_file)
|
3378
|
+
|
3379
|
+
@cached_nullary
|
3380
|
+
def _remote_src(self) -> ta.Sequence[str]:
|
3381
|
+
return [
|
3382
|
+
self._payload_src(),
|
3383
|
+
'_remote_execution_main()',
|
3384
|
+
]
|
3385
|
+
|
3386
|
+
@cached_nullary
|
3387
|
+
def _spawn_src(self) -> str:
|
3388
|
+
return pyremote_build_bootstrap_cmd(__package__ or 'manage')
|
3389
|
+
|
3390
|
+
#
|
3391
|
+
|
3392
|
+
@contextlib.contextmanager
|
3393
|
+
def connect(
|
3394
|
+
self,
|
3395
|
+
tgt: RemoteSpawning.Target,
|
3396
|
+
bs: MainBootstrap,
|
3397
|
+
) -> ta.Generator[RemoteCommandExecutor, None, None]:
|
3398
|
+
spawn_src = self._spawn_src()
|
3399
|
+
remote_src = self._remote_src()
|
3400
|
+
|
3401
|
+
with self._spawning.spawn(
|
3402
|
+
tgt,
|
3403
|
+
spawn_src,
|
3404
|
+
) as proc:
|
3405
|
+
res = PyremoteBootstrapDriver( # noqa
|
3406
|
+
remote_src,
|
3407
|
+
PyremoteBootstrapOptions(
|
3408
|
+
debug=bs.main_config.debug,
|
3409
|
+
),
|
3410
|
+
).run(
|
3411
|
+
proc.stdout,
|
3412
|
+
proc.stdin,
|
3413
|
+
)
|
3414
|
+
|
3415
|
+
chan = RemoteChannel(
|
3416
|
+
proc.stdout,
|
3417
|
+
proc.stdin,
|
3418
|
+
msh=self._msh,
|
3419
|
+
)
|
3420
|
+
|
3421
|
+
chan.send_obj(bs)
|
3422
|
+
|
3423
|
+
yield RemoteCommandExecutor(chan)
|
3424
|
+
|
3425
|
+
|
3426
|
+
########################################
|
3427
|
+
# ../remote/inject.py
|
3428
|
+
|
3429
|
+
|
3430
|
+
def bind_remote(
|
3431
|
+
*,
|
3432
|
+
remote_config: RemoteConfig,
|
3433
|
+
) -> InjectorBindings:
|
3434
|
+
lst: ta.List[InjectorBindingOrBindings] = [
|
3435
|
+
inj.bind(remote_config),
|
3436
|
+
|
3437
|
+
inj.bind(RemoteSpawning, singleton=True),
|
3438
|
+
|
3439
|
+
inj.bind(RemoteExecution, singleton=True),
|
3440
|
+
]
|
3441
|
+
|
3442
|
+
if (pf := remote_config.payload_file) is not None:
|
3443
|
+
lst.append(inj.bind(pf, to_key=RemoteExecutionPayloadFile))
|
3444
|
+
|
3445
|
+
return inj.as_bindings(*lst)
|
3446
|
+
|
3447
|
+
|
3448
|
+
########################################
|
3449
|
+
# ../inject.py
|
3450
|
+
|
3451
|
+
|
3452
|
+
##
|
3453
|
+
|
3454
|
+
|
3455
|
+
def bind_main(
|
3456
|
+
*,
|
3457
|
+
main_config: MainConfig,
|
3458
|
+
remote_config: RemoteConfig,
|
3459
|
+
) -> InjectorBindings:
|
3460
|
+
lst: ta.List[InjectorBindingOrBindings] = [
|
3461
|
+
inj.bind(main_config),
|
3462
|
+
|
3463
|
+
bind_commands(
|
3464
|
+
main_config=main_config,
|
3465
|
+
),
|
3466
|
+
|
3467
|
+
bind_remote(
|
3468
|
+
remote_config=remote_config,
|
3469
|
+
),
|
3470
|
+
]
|
3471
|
+
|
3472
|
+
#
|
3473
|
+
|
3474
|
+
def build_obj_marshaler_manager(insts: ObjMarshalerInstallers) -> ObjMarshalerManager:
|
3475
|
+
msh = ObjMarshalerManager()
|
3476
|
+
inst: ObjMarshalerInstaller
|
3477
|
+
for inst in insts:
|
3478
|
+
inst.fn(msh)
|
3479
|
+
return msh
|
3480
|
+
|
3481
|
+
lst.extend([
|
3482
|
+
inj.bind(build_obj_marshaler_manager, singleton=True),
|
3483
|
+
|
3484
|
+
inj.bind_array(ObjMarshalerInstaller),
|
3485
|
+
inj.bind_array_type(ObjMarshalerInstaller, ObjMarshalerInstallers),
|
3486
|
+
])
|
3487
|
+
|
3488
|
+
#
|
3489
|
+
|
3490
|
+
return inj.as_bindings(*lst)
|
3491
|
+
|
3492
|
+
|
3493
|
+
########################################
|
3494
|
+
# ../bootstrap_.py
|
3495
|
+
|
1786
3496
|
|
1787
|
-
|
3497
|
+
def main_bootstrap(bs: MainBootstrap) -> Injector:
|
3498
|
+
if (log_level := bs.main_config.log_level) is not None:
|
3499
|
+
configure_standard_logging(log_level)
|
3500
|
+
|
3501
|
+
injector = inj.create_injector(bind_main( # noqa
|
3502
|
+
main_config=bs.main_config,
|
3503
|
+
remote_config=bs.remote_config,
|
3504
|
+
))
|
3505
|
+
|
3506
|
+
return injector
|
3507
|
+
|
3508
|
+
|
3509
|
+
########################################
|
3510
|
+
# main.py
|
1788
3511
|
|
1789
3512
|
|
1790
3513
|
##
|
@@ -1795,65 +3518,70 @@ def _main() -> None:
|
|
1795
3518
|
|
1796
3519
|
parser = argparse.ArgumentParser()
|
1797
3520
|
|
3521
|
+
parser.add_argument('--_payload-file')
|
3522
|
+
|
1798
3523
|
parser.add_argument('-s', '--shell')
|
1799
3524
|
parser.add_argument('-q', '--shell-quote', action='store_true')
|
1800
3525
|
parser.add_argument('--python', default='python3')
|
3526
|
+
|
3527
|
+
parser.add_argument('--pycharm-debug-port', type=int)
|
3528
|
+
parser.add_argument('--pycharm-debug-host')
|
3529
|
+
parser.add_argument('--pycharm-debug-version')
|
3530
|
+
|
1801
3531
|
parser.add_argument('--debug', action='store_true')
|
1802
|
-
|
3532
|
+
|
3533
|
+
parser.add_argument('command', nargs='+')
|
1803
3534
|
|
1804
3535
|
args = parser.parse_args()
|
1805
3536
|
|
1806
3537
|
#
|
1807
3538
|
|
1808
|
-
|
3539
|
+
bs = MainBootstrap(
|
3540
|
+
main_config=MainConfig(
|
3541
|
+
log_level='DEBUG' if args.debug else 'INFO',
|
1809
3542
|
|
1810
|
-
|
3543
|
+
debug=bool(args.debug),
|
3544
|
+
),
|
1811
3545
|
|
1812
|
-
|
1813
|
-
|
1814
|
-
|
1815
|
-
|
1816
|
-
|
3546
|
+
remote_config=RemoteConfig(
|
3547
|
+
payload_file=args._payload_file, # noqa
|
3548
|
+
|
3549
|
+
pycharm_remote_debug=PycharmRemoteDebug(
|
3550
|
+
port=args.pycharm_debug_port,
|
3551
|
+
host=args.pycharm_debug_host,
|
3552
|
+
install_version=args.pycharm_debug_version,
|
3553
|
+
) if args.pycharm_debug_port is not None else None,
|
3554
|
+
),
|
3555
|
+
)
|
3556
|
+
|
3557
|
+
injector = main_bootstrap(
|
3558
|
+
bs,
|
3559
|
+
)
|
1817
3560
|
|
1818
3561
|
#
|
1819
3562
|
|
1820
|
-
|
3563
|
+
cmds = [
|
3564
|
+
SubprocessCommand([c])
|
3565
|
+
for c in args.command
|
3566
|
+
]
|
1821
3567
|
|
1822
3568
|
#
|
1823
3569
|
|
1824
|
-
|
1825
|
-
bs_src,
|
3570
|
+
tgt = RemoteSpawning.Target(
|
1826
3571
|
shell=args.shell,
|
1827
3572
|
shell_quote=args.shell_quote,
|
1828
3573
|
python=args.python,
|
1829
3574
|
)
|
1830
|
-
with spawner.spawn() as proc:
|
1831
|
-
res = PyremoteBootstrapDriver( # noqa
|
1832
|
-
remote_src,
|
1833
|
-
PyremoteBootstrapOptions(
|
1834
|
-
debug=args.debug,
|
1835
|
-
),
|
1836
|
-
).run(proc.stdin, proc.stdout)
|
1837
|
-
# print(res)
|
1838
3575
|
|
1839
|
-
|
3576
|
+
with injector[RemoteExecution].connect(tgt, bs) as rce:
|
3577
|
+
for cmd in cmds:
|
3578
|
+
r = rce.try_execute(cmd)
|
1840
3579
|
|
1841
|
-
|
1842
|
-
SubprocessCommand(
|
1843
|
-
args=['python3', '-'],
|
1844
|
-
input=b'print(1)\n',
|
1845
|
-
capture_stdout=True,
|
1846
|
-
),
|
1847
|
-
SubprocessCommand(
|
1848
|
-
args=['uname'],
|
1849
|
-
capture_stdout=True,
|
1850
|
-
),
|
1851
|
-
]:
|
1852
|
-
_send_obj(proc.stdin, ci, Command)
|
3580
|
+
print(injector[ObjMarshalerManager].marshal_obj(r, opts=ObjMarshalOptions(raw_bytes=True)))
|
1853
3581
|
|
1854
|
-
|
3582
|
+
#
|
1855
3583
|
|
1856
|
-
|
3584
|
+
print('Success')
|
1857
3585
|
|
1858
3586
|
|
1859
3587
|
if __name__ == '__main__':
|