ominfra 0.0.0.dev157__py3-none-any.whl → 0.0.0.dev158__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/main.py +1 -1
- ominfra/journald/tailer.py +2 -2
- ominfra/manage/bootstrap_.py +1 -1
- ominfra/manage/commands/subprocess.py +4 -4
- ominfra/manage/deploy/apps.py +14 -15
- ominfra/manage/deploy/config.py +3 -0
- ominfra/manage/deploy/git.py +11 -27
- ominfra/manage/deploy/paths.py +48 -48
- ominfra/manage/deploy/specs.py +32 -0
- ominfra/manage/deploy/venvs.py +10 -5
- ominfra/manage/remote/spawning.py +3 -3
- ominfra/manage/system/packages.py +1 -1
- ominfra/pyremote.py +26 -26
- ominfra/scripts/journald2aws.py +461 -350
- ominfra/scripts/manage.py +1783 -1693
- ominfra/scripts/supervisor.py +353 -332
- ominfra/supervisor/http.py +1 -1
- ominfra/supervisor/main.py +2 -2
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/RECORD +24 -23
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -100,7 +100,7 @@ CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
|
|
100
100
|
|
101
101
|
# deploy/paths.py
|
102
102
|
DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
|
103
|
-
|
103
|
+
DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
|
104
104
|
|
105
105
|
# ../../omlish/argparse/cli.py
|
106
106
|
ArgparseCommandFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
|
@@ -118,7 +118,7 @@ InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
|
|
118
118
|
# ../configs.py
|
119
119
|
ConfigMapping = ta.Mapping[str, ta.Any]
|
120
120
|
|
121
|
-
# ../../omlish/
|
121
|
+
# ../../omlish/subprocesses.py
|
122
122
|
SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
|
123
123
|
|
124
124
|
# system/packages.py
|
@@ -1365,6 +1365,9 @@ class MainConfig:
|
|
1365
1365
|
# ../deploy/config.py
|
1366
1366
|
|
1367
1367
|
|
1368
|
+
##
|
1369
|
+
|
1370
|
+
|
1368
1371
|
@dc.dataclass(frozen=True)
|
1369
1372
|
class DeployConfig:
|
1370
1373
|
deploy_home: ta.Optional[str] = None
|
@@ -1539,7 +1542,7 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
1539
1542
|
# Get pid
|
1540
1543
|
pid = os.getpid()
|
1541
1544
|
|
1542
|
-
# Two copies of
|
1545
|
+
# Two copies of payload src to be sent to parent
|
1543
1546
|
r0, w0 = os.pipe()
|
1544
1547
|
r1, w1 = os.pipe()
|
1545
1548
|
|
@@ -1578,17 +1581,17 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
1578
1581
|
# Write pid
|
1579
1582
|
os.write(1, struct.pack('<Q', pid))
|
1580
1583
|
|
1581
|
-
# Read
|
1582
|
-
|
1583
|
-
if len(
|
1584
|
+
# Read payload src from stdin
|
1585
|
+
payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
|
1586
|
+
if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
|
1584
1587
|
raise EOFError
|
1585
|
-
|
1588
|
+
payload_src = zlib.decompress(payload_z)
|
1586
1589
|
|
1587
|
-
# Write both copies of
|
1588
|
-
# and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
1590
|
+
# Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
|
1591
|
+
# fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
1589
1592
|
for w in [w0, w1]:
|
1590
1593
|
fp = os.fdopen(w, 'wb', 0)
|
1591
|
-
fp.write(
|
1594
|
+
fp.write(payload_src)
|
1592
1595
|
fp.close()
|
1593
1596
|
|
1594
1597
|
# Write second ack
|
@@ -1652,7 +1655,7 @@ class PyremotePayloadRuntime:
|
|
1652
1655
|
input: ta.BinaryIO
|
1653
1656
|
output: ta.BinaryIO
|
1654
1657
|
context_name: str
|
1655
|
-
|
1658
|
+
payload_src: str
|
1656
1659
|
options: PyremoteBootstrapOptions
|
1657
1660
|
env_info: PyremoteEnvInfo
|
1658
1661
|
|
@@ -1660,9 +1663,9 @@ class PyremotePayloadRuntime:
|
|
1660
1663
|
def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
1661
1664
|
# If src file var is not present we need to do initial finalization
|
1662
1665
|
if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
|
1663
|
-
# Read second copy of
|
1666
|
+
# Read second copy of payload src
|
1664
1667
|
r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
|
1665
|
-
|
1668
|
+
payload_src = r1.read().decode('utf-8')
|
1666
1669
|
r1.close()
|
1667
1670
|
|
1668
1671
|
# Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
|
@@ -1680,7 +1683,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
1680
1683
|
# Write temp source file
|
1681
1684
|
import tempfile
|
1682
1685
|
tfd, tfn = tempfile.mkstemp('-pyremote.py')
|
1683
|
-
os.write(tfd,
|
1686
|
+
os.write(tfd, payload_src.encode('utf-8'))
|
1684
1687
|
os.close(tfd)
|
1685
1688
|
|
1686
1689
|
# Set vars
|
@@ -1699,7 +1702,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
1699
1702
|
|
1700
1703
|
# Read temp source file
|
1701
1704
|
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
1702
|
-
|
1705
|
+
payload_src = sf.read()
|
1703
1706
|
|
1704
1707
|
# Restore vars
|
1705
1708
|
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
@@ -1732,7 +1735,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
1732
1735
|
input=input,
|
1733
1736
|
output=output,
|
1734
1737
|
context_name=context_name,
|
1735
|
-
|
1738
|
+
payload_src=payload_src,
|
1736
1739
|
options=options,
|
1737
1740
|
env_info=env_info,
|
1738
1741
|
)
|
@@ -1744,31 +1747,31 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
1744
1747
|
class PyremoteBootstrapDriver:
|
1745
1748
|
def __init__(
|
1746
1749
|
self,
|
1747
|
-
|
1750
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
1748
1751
|
options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
|
1749
1752
|
) -> None:
|
1750
1753
|
super().__init__()
|
1751
1754
|
|
1752
|
-
self.
|
1755
|
+
self._payload_src = payload_src
|
1753
1756
|
self._options = options
|
1754
1757
|
|
1755
|
-
self.
|
1756
|
-
self.
|
1758
|
+
self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
|
1759
|
+
self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
|
1757
1760
|
|
1758
1761
|
self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
|
1759
1762
|
#
|
1760
1763
|
|
1761
1764
|
@classmethod
|
1762
|
-
def
|
1765
|
+
def _prepare_payload_src(
|
1763
1766
|
cls,
|
1764
|
-
|
1767
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
1765
1768
|
options: PyremoteBootstrapOptions,
|
1766
1769
|
) -> str:
|
1767
1770
|
parts: ta.List[str]
|
1768
|
-
if isinstance(
|
1769
|
-
parts = [
|
1771
|
+
if isinstance(payload_src, str):
|
1772
|
+
parts = [payload_src]
|
1770
1773
|
else:
|
1771
|
-
parts = list(
|
1774
|
+
parts = list(payload_src)
|
1772
1775
|
|
1773
1776
|
if (mn := options.main_name_override) is not None:
|
1774
1777
|
parts.insert(0, f'__name__ = {mn!r}')
|
@@ -1804,9 +1807,9 @@ class PyremoteBootstrapDriver:
|
|
1804
1807
|
d = yield from self._read(8)
|
1805
1808
|
pid = struct.unpack('<Q', d)[0]
|
1806
1809
|
|
1807
|
-
# Write
|
1808
|
-
yield from self._write(struct.pack('<I', len(self.
|
1809
|
-
yield from self._write(self.
|
1810
|
+
# Write payload src
|
1811
|
+
yield from self._write(struct.pack('<I', len(self._payload_z)))
|
1812
|
+
yield from self._write(self._payload_z)
|
1810
1813
|
|
1811
1814
|
# Read second ack (after writing src copies)
|
1812
1815
|
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
|
@@ -2540,6 +2543,13 @@ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON
|
|
2540
2543
|
json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
|
2541
2544
|
|
2542
2545
|
|
2546
|
+
########################################
|
2547
|
+
# ../../../omlish/lite/logs.py
|
2548
|
+
|
2549
|
+
|
2550
|
+
log = logging.getLogger(__name__)
|
2551
|
+
|
2552
|
+
|
2543
2553
|
########################################
|
2544
2554
|
# ../../../omlish/lite/maybes.py
|
2545
2555
|
|
@@ -2754,6 +2764,116 @@ def format_num_bytes(num_bytes: int) -> str:
|
|
2754
2764
|
return f'{num_bytes / 1024 ** (len(FORMAT_NUM_BYTES_SUFFIXES) - 1):.2f}{FORMAT_NUM_BYTES_SUFFIXES[-1]}'
|
2755
2765
|
|
2756
2766
|
|
2767
|
+
########################################
|
2768
|
+
# ../../../omlish/logs/filters.py
|
2769
|
+
|
2770
|
+
|
2771
|
+
class TidLogFilter(logging.Filter):
|
2772
|
+
def filter(self, record):
|
2773
|
+
record.tid = threading.get_native_id()
|
2774
|
+
return True
|
2775
|
+
|
2776
|
+
|
2777
|
+
########################################
|
2778
|
+
# ../../../omlish/logs/proxy.py
|
2779
|
+
|
2780
|
+
|
2781
|
+
class ProxyLogFilterer(logging.Filterer):
|
2782
|
+
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
2783
|
+
self._underlying = underlying
|
2784
|
+
|
2785
|
+
@property
|
2786
|
+
def underlying(self) -> logging.Filterer:
|
2787
|
+
return self._underlying
|
2788
|
+
|
2789
|
+
@property
|
2790
|
+
def filters(self):
|
2791
|
+
return self._underlying.filters
|
2792
|
+
|
2793
|
+
@filters.setter
|
2794
|
+
def filters(self, filters):
|
2795
|
+
self._underlying.filters = filters
|
2796
|
+
|
2797
|
+
def addFilter(self, filter): # noqa
|
2798
|
+
self._underlying.addFilter(filter)
|
2799
|
+
|
2800
|
+
def removeFilter(self, filter): # noqa
|
2801
|
+
self._underlying.removeFilter(filter)
|
2802
|
+
|
2803
|
+
def filter(self, record):
|
2804
|
+
return self._underlying.filter(record)
|
2805
|
+
|
2806
|
+
|
2807
|
+
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
2808
|
+
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
2809
|
+
ProxyLogFilterer.__init__(self, underlying)
|
2810
|
+
|
2811
|
+
_underlying: logging.Handler
|
2812
|
+
|
2813
|
+
@property
|
2814
|
+
def underlying(self) -> logging.Handler:
|
2815
|
+
return self._underlying
|
2816
|
+
|
2817
|
+
def get_name(self):
|
2818
|
+
return self._underlying.get_name()
|
2819
|
+
|
2820
|
+
def set_name(self, name):
|
2821
|
+
self._underlying.set_name(name)
|
2822
|
+
|
2823
|
+
@property
|
2824
|
+
def name(self):
|
2825
|
+
return self._underlying.name
|
2826
|
+
|
2827
|
+
@property
|
2828
|
+
def level(self):
|
2829
|
+
return self._underlying.level
|
2830
|
+
|
2831
|
+
@level.setter
|
2832
|
+
def level(self, level):
|
2833
|
+
self._underlying.level = level
|
2834
|
+
|
2835
|
+
@property
|
2836
|
+
def formatter(self):
|
2837
|
+
return self._underlying.formatter
|
2838
|
+
|
2839
|
+
@formatter.setter
|
2840
|
+
def formatter(self, formatter):
|
2841
|
+
self._underlying.formatter = formatter
|
2842
|
+
|
2843
|
+
def createLock(self):
|
2844
|
+
self._underlying.createLock()
|
2845
|
+
|
2846
|
+
def acquire(self):
|
2847
|
+
self._underlying.acquire()
|
2848
|
+
|
2849
|
+
def release(self):
|
2850
|
+
self._underlying.release()
|
2851
|
+
|
2852
|
+
def setLevel(self, level):
|
2853
|
+
self._underlying.setLevel(level)
|
2854
|
+
|
2855
|
+
def format(self, record):
|
2856
|
+
return self._underlying.format(record)
|
2857
|
+
|
2858
|
+
def emit(self, record):
|
2859
|
+
self._underlying.emit(record)
|
2860
|
+
|
2861
|
+
def handle(self, record):
|
2862
|
+
return self._underlying.handle(record)
|
2863
|
+
|
2864
|
+
def setFormatter(self, fmt):
|
2865
|
+
self._underlying.setFormatter(fmt)
|
2866
|
+
|
2867
|
+
def flush(self):
|
2868
|
+
self._underlying.flush()
|
2869
|
+
|
2870
|
+
def close(self):
|
2871
|
+
self._underlying.close()
|
2872
|
+
|
2873
|
+
def handleError(self, record):
|
2874
|
+
self._underlying.handleError(record)
|
2875
|
+
|
2876
|
+
|
2757
2877
|
########################################
|
2758
2878
|
# ../../../omlish/os/deathsig.py
|
2759
2879
|
|
@@ -3939,22 +4059,22 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
|
3939
4059
|
~deploy
|
3940
4060
|
deploy.pid (flock)
|
3941
4061
|
/app
|
3942
|
-
/<
|
4062
|
+
/<appplaceholder> - shallow clone
|
3943
4063
|
/conf
|
3944
4064
|
/env
|
3945
|
-
<
|
4065
|
+
<appplaceholder>.env
|
3946
4066
|
/nginx
|
3947
|
-
<
|
4067
|
+
<appplaceholder>.conf
|
3948
4068
|
/supervisor
|
3949
|
-
<
|
4069
|
+
<appplaceholder>.conf
|
3950
4070
|
/venv
|
3951
|
-
/<
|
4071
|
+
/<appplaceholder>
|
3952
4072
|
|
3953
4073
|
?
|
3954
4074
|
/logs
|
3955
|
-
/wrmsr--omlish--<
|
4075
|
+
/wrmsr--omlish--<placeholder>
|
3956
4076
|
|
3957
|
-
|
4077
|
+
placeholder = <name>--<rev>--<when>
|
3958
4078
|
|
3959
4079
|
==
|
3960
4080
|
|
@@ -3975,10 +4095,10 @@ for dn in [
|
|
3975
4095
|
##
|
3976
4096
|
|
3977
4097
|
|
3978
|
-
|
3979
|
-
|
4098
|
+
DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER = '@'
|
4099
|
+
DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
|
3980
4100
|
|
3981
|
-
|
4101
|
+
DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
|
3982
4102
|
'app',
|
3983
4103
|
'tag', # <rev>-<dt>
|
3984
4104
|
])
|
@@ -3996,7 +4116,7 @@ class DeployPathPart(abc.ABC): # noqa
|
|
3996
4116
|
raise NotImplementedError
|
3997
4117
|
|
3998
4118
|
@abc.abstractmethod
|
3999
|
-
def render(self,
|
4119
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4000
4120
|
raise NotImplementedError
|
4001
4121
|
|
4002
4122
|
|
@@ -4010,9 +4130,9 @@ class DirDeployPathPart(DeployPathPart, abc.ABC):
|
|
4010
4130
|
|
4011
4131
|
@classmethod
|
4012
4132
|
def parse(cls, s: str) -> 'DirDeployPathPart':
|
4013
|
-
if
|
4014
|
-
check.equal(s[0],
|
4015
|
-
return
|
4133
|
+
if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
|
4134
|
+
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
4135
|
+
return PlaceholderDirDeployPathPart(s[1:])
|
4016
4136
|
else:
|
4017
4137
|
return ConstDirDeployPathPart(s)
|
4018
4138
|
|
@@ -4024,13 +4144,13 @@ class FileDeployPathPart(DeployPathPart, abc.ABC):
|
|
4024
4144
|
|
4025
4145
|
@classmethod
|
4026
4146
|
def parse(cls, s: str) -> 'FileDeployPathPart':
|
4027
|
-
if
|
4028
|
-
check.equal(s[0],
|
4029
|
-
if not any(c in s for c in
|
4030
|
-
return
|
4147
|
+
if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
|
4148
|
+
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
4149
|
+
if not any(c in s for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS):
|
4150
|
+
return PlaceholderFileDeployPathPart(s[1:], '')
|
4031
4151
|
else:
|
4032
|
-
p = min(f for c in
|
4033
|
-
return
|
4152
|
+
p = min(f for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS if (f := s.find(c)) > 0)
|
4153
|
+
return PlaceholderFileDeployPathPart(s[1:p], s[p:])
|
4034
4154
|
else:
|
4035
4155
|
return ConstFileDeployPathPart(s)
|
4036
4156
|
|
@@ -4045,9 +4165,9 @@ class ConstDeployPathPart(DeployPathPart, abc.ABC):
|
|
4045
4165
|
def __post_init__(self) -> None:
|
4046
4166
|
check.non_empty_str(self.name)
|
4047
4167
|
check.not_in('/', self.name)
|
4048
|
-
check.not_in(
|
4168
|
+
check.not_in(DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, self.name)
|
4049
4169
|
|
4050
|
-
def render(self,
|
4170
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4051
4171
|
return self.name
|
4052
4172
|
|
4053
4173
|
|
@@ -4063,40 +4183,40 @@ class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
|
|
4063
4183
|
|
4064
4184
|
|
4065
4185
|
@dc.dataclass(frozen=True)
|
4066
|
-
class
|
4067
|
-
|
4186
|
+
class PlaceholderDeployPathPart(DeployPathPart, abc.ABC):
|
4187
|
+
placeholder: str # DeployPathPlaceholder
|
4068
4188
|
|
4069
4189
|
def __post_init__(self) -> None:
|
4070
|
-
check.non_empty_str(self.
|
4071
|
-
for c in [*
|
4072
|
-
check.not_in(c, self.
|
4073
|
-
check.in_(self.
|
4074
|
-
|
4075
|
-
def
|
4076
|
-
if
|
4077
|
-
return
|
4190
|
+
check.non_empty_str(self.placeholder)
|
4191
|
+
for c in [*DEPLOY_PATH_PLACEHOLDER_SEPARATORS, DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
|
4192
|
+
check.not_in(c, self.placeholder)
|
4193
|
+
check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
|
4194
|
+
|
4195
|
+
def _render_placeholder(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4196
|
+
if placeholders is not None:
|
4197
|
+
return placeholders[self.placeholder] # type: ignore
|
4078
4198
|
else:
|
4079
|
-
return
|
4199
|
+
return DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER + self.placeholder
|
4080
4200
|
|
4081
4201
|
|
4082
4202
|
@dc.dataclass(frozen=True)
|
4083
|
-
class
|
4084
|
-
def render(self,
|
4085
|
-
return self.
|
4203
|
+
class PlaceholderDirDeployPathPart(PlaceholderDeployPathPart, DirDeployPathPart):
|
4204
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4205
|
+
return self._render_placeholder(placeholders)
|
4086
4206
|
|
4087
4207
|
|
4088
4208
|
@dc.dataclass(frozen=True)
|
4089
|
-
class
|
4209
|
+
class PlaceholderFileDeployPathPart(PlaceholderDeployPathPart, FileDeployPathPart):
|
4090
4210
|
suffix: str
|
4091
4211
|
|
4092
4212
|
def __post_init__(self) -> None:
|
4093
4213
|
super().__post_init__()
|
4094
4214
|
if self.suffix:
|
4095
|
-
for c in [
|
4215
|
+
for c in [DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
|
4096
4216
|
check.not_in(c, self.suffix)
|
4097
4217
|
|
4098
|
-
def render(self,
|
4099
|
-
return self.
|
4218
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4219
|
+
return self._render_placeholder(placeholders) + self.suffix
|
4100
4220
|
|
4101
4221
|
|
4102
4222
|
##
|
@@ -4113,22 +4233,22 @@ class DeployPath:
|
|
4113
4233
|
|
4114
4234
|
pd = {}
|
4115
4235
|
for i, p in enumerate(self.parts):
|
4116
|
-
if isinstance(p,
|
4117
|
-
if p.
|
4118
|
-
raise DeployPathError('Duplicate
|
4119
|
-
pd[p.
|
4236
|
+
if isinstance(p, PlaceholderDeployPathPart):
|
4237
|
+
if p.placeholder in pd:
|
4238
|
+
raise DeployPathError('Duplicate placeholders in path', self)
|
4239
|
+
pd[p.placeholder] = i
|
4120
4240
|
|
4121
4241
|
if 'tag' in pd:
|
4122
4242
|
if 'app' not in pd or pd['app'] >= pd['tag']:
|
4123
|
-
raise DeployPathError('Tag
|
4243
|
+
raise DeployPathError('Tag placeholder in path without preceding app', self)
|
4124
4244
|
|
4125
4245
|
@property
|
4126
4246
|
def kind(self) -> ta.Literal['file', 'dir']:
|
4127
4247
|
return self.parts[-1].kind
|
4128
4248
|
|
4129
|
-
def render(self,
|
4249
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4130
4250
|
return os.path.join( # noqa
|
4131
|
-
*[p.render(
|
4251
|
+
*[p.render(placeholders) for p in self.parts],
|
4132
4252
|
*([''] if self.kind == 'dir' else []),
|
4133
4253
|
)
|
4134
4254
|
|
@@ -4156,6 +4276,34 @@ class DeployPathOwner(abc.ABC):
|
|
4156
4276
|
raise NotImplementedError
|
4157
4277
|
|
4158
4278
|
|
4279
|
+
########################################
|
4280
|
+
# ../deploy/specs.py
|
4281
|
+
|
4282
|
+
|
4283
|
+
##
|
4284
|
+
|
4285
|
+
|
4286
|
+
@dc.dataclass(frozen=True)
|
4287
|
+
class DeployGitRepo:
|
4288
|
+
host: ta.Optional[str] = None
|
4289
|
+
username: ta.Optional[str] = None
|
4290
|
+
path: ta.Optional[str] = None
|
4291
|
+
|
4292
|
+
def __post_init__(self) -> None:
|
4293
|
+
check.not_in('..', check.non_empty_str(self.host))
|
4294
|
+
check.not_in('.', check.non_empty_str(self.path))
|
4295
|
+
|
4296
|
+
|
4297
|
+
##
|
4298
|
+
|
4299
|
+
|
4300
|
+
@dc.dataclass(frozen=True)
|
4301
|
+
class DeploySpec:
|
4302
|
+
app: DeployApp
|
4303
|
+
repo: DeployGitRepo
|
4304
|
+
rev: DeployRev
|
4305
|
+
|
4306
|
+
|
4159
4307
|
########################################
|
4160
4308
|
# ../remote/config.py
|
4161
4309
|
|
@@ -4216,6 +4364,75 @@ def get_remote_payload_src(
|
|
4216
4364
|
return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
|
4217
4365
|
|
4218
4366
|
|
4367
|
+
########################################
|
4368
|
+
# ../system/platforms.py
|
4369
|
+
|
4370
|
+
|
4371
|
+
##
|
4372
|
+
|
4373
|
+
|
4374
|
+
@dc.dataclass(frozen=True)
|
4375
|
+
class Platform(abc.ABC): # noqa
|
4376
|
+
pass
|
4377
|
+
|
4378
|
+
|
4379
|
+
class LinuxPlatform(Platform, abc.ABC):
|
4380
|
+
pass
|
4381
|
+
|
4382
|
+
|
4383
|
+
class UbuntuPlatform(LinuxPlatform):
|
4384
|
+
pass
|
4385
|
+
|
4386
|
+
|
4387
|
+
class AmazonLinuxPlatform(LinuxPlatform):
|
4388
|
+
pass
|
4389
|
+
|
4390
|
+
|
4391
|
+
class GenericLinuxPlatform(LinuxPlatform):
|
4392
|
+
pass
|
4393
|
+
|
4394
|
+
|
4395
|
+
class DarwinPlatform(Platform):
|
4396
|
+
pass
|
4397
|
+
|
4398
|
+
|
4399
|
+
class UnknownPlatform(Platform):
|
4400
|
+
pass
|
4401
|
+
|
4402
|
+
|
4403
|
+
##
|
4404
|
+
|
4405
|
+
|
4406
|
+
def _detect_system_platform() -> Platform:
|
4407
|
+
plat = sys.platform
|
4408
|
+
|
4409
|
+
if plat == 'linux':
|
4410
|
+
if (osr := LinuxOsRelease.read()) is None:
|
4411
|
+
return GenericLinuxPlatform()
|
4412
|
+
|
4413
|
+
if osr.id == 'amzn':
|
4414
|
+
return AmazonLinuxPlatform()
|
4415
|
+
|
4416
|
+
elif osr.id == 'ubuntu':
|
4417
|
+
return UbuntuPlatform()
|
4418
|
+
|
4419
|
+
else:
|
4420
|
+
return GenericLinuxPlatform()
|
4421
|
+
|
4422
|
+
elif plat == 'darwin':
|
4423
|
+
return DarwinPlatform()
|
4424
|
+
|
4425
|
+
else:
|
4426
|
+
return UnknownPlatform()
|
4427
|
+
|
4428
|
+
|
4429
|
+
@cached_nullary
|
4430
|
+
def detect_system_platform() -> Platform:
|
4431
|
+
platform = _detect_system_platform()
|
4432
|
+
log.info('Detected platform: %r', platform)
|
4433
|
+
return platform
|
4434
|
+
|
4435
|
+
|
4219
4436
|
########################################
|
4220
4437
|
# ../targets/targets.py
|
4221
4438
|
"""
|
@@ -5540,1150 +5757,994 @@ inj = Injection
|
|
5540
5757
|
|
5541
5758
|
|
5542
5759
|
########################################
|
5543
|
-
# ../../../omlish/lite/
|
5760
|
+
# ../../../omlish/lite/marshal.py
|
5544
5761
|
"""
|
5545
5762
|
TODO:
|
5546
|
-
-
|
5547
|
-
-
|
5763
|
+
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
5764
|
+
- namedtuple
|
5765
|
+
- literals
|
5766
|
+
- newtypes?
|
5548
5767
|
"""
|
5549
5768
|
|
5550
5769
|
|
5551
|
-
|
5770
|
+
##
|
5552
5771
|
|
5553
5772
|
|
5554
|
-
|
5773
|
+
@dc.dataclass(frozen=True)
|
5774
|
+
class ObjMarshalOptions:
|
5775
|
+
raw_bytes: bool = False
|
5776
|
+
nonstrict_dataclasses: bool = False
|
5555
5777
|
|
5556
5778
|
|
5557
|
-
class
|
5779
|
+
class ObjMarshaler(abc.ABC):
|
5780
|
+
@abc.abstractmethod
|
5781
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5782
|
+
raise NotImplementedError
|
5558
5783
|
|
5559
|
-
|
5560
|
-
|
5561
|
-
|
5784
|
+
@abc.abstractmethod
|
5785
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5786
|
+
raise NotImplementedError
|
5562
5787
|
|
5563
5788
|
|
5564
|
-
|
5789
|
+
class NopObjMarshaler(ObjMarshaler):
|
5790
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5791
|
+
return o
|
5565
5792
|
|
5793
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5794
|
+
return o
|
5566
5795
|
|
5567
|
-
class JsonLogFormatter(logging.Formatter):
|
5568
5796
|
|
5569
|
-
|
5570
|
-
|
5571
|
-
|
5572
|
-
'args': False,
|
5573
|
-
'levelname': False,
|
5574
|
-
'levelno': False,
|
5575
|
-
'pathname': False,
|
5576
|
-
'filename': False,
|
5577
|
-
'module': False,
|
5578
|
-
'exc_info': True,
|
5579
|
-
'exc_text': True,
|
5580
|
-
'stack_info': True,
|
5581
|
-
'lineno': False,
|
5582
|
-
'funcName': False,
|
5583
|
-
'created': False,
|
5584
|
-
'msecs': False,
|
5585
|
-
'relativeCreated': False,
|
5586
|
-
'thread': False,
|
5587
|
-
'threadName': False,
|
5588
|
-
'processName': False,
|
5589
|
-
'process': False,
|
5590
|
-
}
|
5797
|
+
@dc.dataclass()
|
5798
|
+
class ProxyObjMarshaler(ObjMarshaler):
|
5799
|
+
m: ta.Optional[ObjMarshaler] = None
|
5591
5800
|
|
5592
|
-
def
|
5593
|
-
|
5594
|
-
k: v
|
5595
|
-
for k, o in self.KEYS.items()
|
5596
|
-
for v in [getattr(record, k)]
|
5597
|
-
if not (o and v is None)
|
5598
|
-
}
|
5599
|
-
return json_dumps_compact(dct)
|
5801
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5802
|
+
return check.not_none(self.m).marshal(o, ctx)
|
5600
5803
|
|
5804
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5805
|
+
return check.not_none(self.m).unmarshal(o, ctx)
|
5601
5806
|
|
5602
|
-
##
|
5603
5807
|
|
5808
|
+
@dc.dataclass(frozen=True)
|
5809
|
+
class CastObjMarshaler(ObjMarshaler):
|
5810
|
+
ty: type
|
5604
5811
|
|
5605
|
-
|
5606
|
-
|
5607
|
-
('process', 'pid=%(process)-6s'),
|
5608
|
-
('thread', 'tid=%(thread)x'),
|
5609
|
-
('levelname', '%(levelname)s'),
|
5610
|
-
('name', '%(name)s'),
|
5611
|
-
('separator', '::'),
|
5612
|
-
('message', '%(message)s'),
|
5613
|
-
]
|
5812
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5813
|
+
return o
|
5614
5814
|
|
5815
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5816
|
+
return self.ty(o)
|
5615
5817
|
|
5616
|
-
class StandardLogFormatter(logging.Formatter):
|
5617
5818
|
|
5618
|
-
|
5619
|
-
def
|
5620
|
-
return
|
5819
|
+
class DynamicObjMarshaler(ObjMarshaler):
|
5820
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5821
|
+
return ctx.manager.marshal_obj(o, opts=ctx.options)
|
5621
5822
|
|
5622
|
-
|
5823
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5824
|
+
return o
|
5623
5825
|
|
5624
|
-
def formatTime(self, record, datefmt=None):
|
5625
|
-
ct = self.converter(record.created) # type: ignore
|
5626
|
-
if datefmt:
|
5627
|
-
return ct.strftime(datefmt) # noqa
|
5628
|
-
else:
|
5629
|
-
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
5630
|
-
return '%s.%03d' % (t, record.msecs) # noqa
|
5631
5826
|
|
5827
|
+
@dc.dataclass(frozen=True)
|
5828
|
+
class Base64ObjMarshaler(ObjMarshaler):
|
5829
|
+
ty: type
|
5632
5830
|
|
5633
|
-
|
5831
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5832
|
+
return base64.b64encode(o).decode('ascii')
|
5634
5833
|
|
5834
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5835
|
+
return self.ty(base64.b64decode(o))
|
5635
5836
|
|
5636
|
-
class ProxyLogFilterer(logging.Filterer):
|
5637
|
-
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
5638
|
-
self._underlying = underlying
|
5639
5837
|
|
5640
|
-
|
5641
|
-
|
5642
|
-
|
5838
|
+
@dc.dataclass(frozen=True)
|
5839
|
+
class BytesSwitchedObjMarshaler(ObjMarshaler):
|
5840
|
+
m: ObjMarshaler
|
5643
5841
|
|
5644
|
-
|
5645
|
-
|
5646
|
-
|
5842
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5843
|
+
if ctx.options.raw_bytes:
|
5844
|
+
return o
|
5845
|
+
return self.m.marshal(o, ctx)
|
5647
5846
|
|
5648
|
-
|
5649
|
-
|
5650
|
-
|
5847
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5848
|
+
if ctx.options.raw_bytes:
|
5849
|
+
return o
|
5850
|
+
return self.m.unmarshal(o, ctx)
|
5651
5851
|
|
5652
|
-
def addFilter(self, filter): # noqa
|
5653
|
-
self._underlying.addFilter(filter)
|
5654
5852
|
|
5655
|
-
|
5656
|
-
|
5853
|
+
@dc.dataclass(frozen=True)
|
5854
|
+
class EnumObjMarshaler(ObjMarshaler):
|
5855
|
+
ty: type
|
5657
5856
|
|
5658
|
-
def
|
5659
|
-
return
|
5857
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5858
|
+
return o.name
|
5660
5859
|
|
5860
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5861
|
+
return self.ty.__members__[o] # type: ignore
|
5661
5862
|
|
5662
|
-
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
5663
|
-
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
5664
|
-
ProxyLogFilterer.__init__(self, underlying)
|
5665
5863
|
|
5666
|
-
|
5864
|
+
@dc.dataclass(frozen=True)
|
5865
|
+
class OptionalObjMarshaler(ObjMarshaler):
|
5866
|
+
item: ObjMarshaler
|
5667
5867
|
|
5668
|
-
|
5669
|
-
|
5670
|
-
|
5868
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5869
|
+
if o is None:
|
5870
|
+
return None
|
5871
|
+
return self.item.marshal(o, ctx)
|
5671
5872
|
|
5672
|
-
def
|
5673
|
-
|
5873
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5874
|
+
if o is None:
|
5875
|
+
return None
|
5876
|
+
return self.item.unmarshal(o, ctx)
|
5674
5877
|
|
5675
|
-
def set_name(self, name):
|
5676
|
-
self._underlying.set_name(name)
|
5677
5878
|
|
5678
|
-
|
5679
|
-
|
5680
|
-
|
5879
|
+
@dc.dataclass(frozen=True)
|
5880
|
+
class MappingObjMarshaler(ObjMarshaler):
|
5881
|
+
ty: type
|
5882
|
+
km: ObjMarshaler
|
5883
|
+
vm: ObjMarshaler
|
5681
5884
|
|
5682
|
-
|
5683
|
-
|
5684
|
-
return self._underlying.level
|
5885
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5886
|
+
return {self.km.marshal(k, ctx): self.vm.marshal(v, ctx) for k, v in o.items()}
|
5685
5887
|
|
5686
|
-
|
5687
|
-
|
5688
|
-
self._underlying.level = level
|
5888
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5889
|
+
return self.ty((self.km.unmarshal(k, ctx), self.vm.unmarshal(v, ctx)) for k, v in o.items())
|
5689
5890
|
|
5690
|
-
@property
|
5691
|
-
def formatter(self):
|
5692
|
-
return self._underlying.formatter
|
5693
5891
|
|
5694
|
-
|
5695
|
-
|
5696
|
-
|
5892
|
+
@dc.dataclass(frozen=True)
|
5893
|
+
class IterableObjMarshaler(ObjMarshaler):
|
5894
|
+
ty: type
|
5895
|
+
item: ObjMarshaler
|
5697
5896
|
|
5698
|
-
def
|
5699
|
-
self.
|
5897
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5898
|
+
return [self.item.marshal(e, ctx) for e in o]
|
5700
5899
|
|
5701
|
-
def
|
5702
|
-
self.
|
5900
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5901
|
+
return self.ty(self.item.unmarshal(e, ctx) for e in o)
|
5703
5902
|
|
5704
|
-
def release(self):
|
5705
|
-
self._underlying.release()
|
5706
5903
|
|
5707
|
-
|
5708
|
-
|
5904
|
+
@dc.dataclass(frozen=True)
|
5905
|
+
class DataclassObjMarshaler(ObjMarshaler):
|
5906
|
+
ty: type
|
5907
|
+
fs: ta.Mapping[str, ObjMarshaler]
|
5908
|
+
nonstrict: bool = False
|
5709
5909
|
|
5710
|
-
def
|
5711
|
-
return
|
5910
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5911
|
+
return {
|
5912
|
+
k: m.marshal(getattr(o, k), ctx)
|
5913
|
+
for k, m in self.fs.items()
|
5914
|
+
}
|
5712
5915
|
|
5713
|
-
def
|
5714
|
-
self.
|
5916
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5917
|
+
return self.ty(**{
|
5918
|
+
k: self.fs[k].unmarshal(v, ctx)
|
5919
|
+
for k, v in o.items()
|
5920
|
+
if not (self.nonstrict or ctx.options.nonstrict_dataclasses) or k in self.fs
|
5921
|
+
})
|
5715
5922
|
|
5716
|
-
def handle(self, record):
|
5717
|
-
return self._underlying.handle(record)
|
5718
5923
|
|
5719
|
-
|
5720
|
-
|
5924
|
+
@dc.dataclass(frozen=True)
|
5925
|
+
class PolymorphicObjMarshaler(ObjMarshaler):
|
5926
|
+
class Impl(ta.NamedTuple):
|
5927
|
+
ty: type
|
5928
|
+
tag: str
|
5929
|
+
m: ObjMarshaler
|
5721
5930
|
|
5722
|
-
|
5723
|
-
|
5931
|
+
impls_by_ty: ta.Mapping[type, Impl]
|
5932
|
+
impls_by_tag: ta.Mapping[str, Impl]
|
5724
5933
|
|
5725
|
-
|
5726
|
-
|
5934
|
+
@classmethod
|
5935
|
+
def of(cls, impls: ta.Iterable[Impl]) -> 'PolymorphicObjMarshaler':
|
5936
|
+
return cls(
|
5937
|
+
{i.ty: i for i in impls},
|
5938
|
+
{i.tag: i for i in impls},
|
5939
|
+
)
|
5727
5940
|
|
5728
|
-
def
|
5729
|
-
self.
|
5941
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5942
|
+
impl = self.impls_by_ty[type(o)]
|
5943
|
+
return {impl.tag: impl.m.marshal(o, ctx)}
|
5730
5944
|
|
5945
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5946
|
+
[(t, v)] = o.items()
|
5947
|
+
impl = self.impls_by_tag[t]
|
5948
|
+
return impl.m.unmarshal(v, ctx)
|
5731
5949
|
|
5732
|
-
##
|
5733
5950
|
|
5951
|
+
@dc.dataclass(frozen=True)
|
5952
|
+
class DatetimeObjMarshaler(ObjMarshaler):
|
5953
|
+
ty: type
|
5734
5954
|
|
5735
|
-
|
5736
|
-
|
5955
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5956
|
+
return o.isoformat()
|
5737
5957
|
|
5958
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5959
|
+
return self.ty.fromisoformat(o) # type: ignore
|
5738
5960
|
|
5739
|
-
##
|
5740
5961
|
|
5962
|
+
class DecimalObjMarshaler(ObjMarshaler):
|
5963
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5964
|
+
return str(check.isinstance(o, decimal.Decimal))
|
5741
5965
|
|
5742
|
-
|
5743
|
-
|
5744
|
-
if hasattr(logging, '_acquireLock'):
|
5745
|
-
logging._acquireLock() # noqa
|
5746
|
-
try:
|
5747
|
-
yield
|
5748
|
-
finally:
|
5749
|
-
logging._releaseLock() # type: ignore # noqa
|
5966
|
+
def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5967
|
+
return decimal.Decimal(check.isinstance(v, str))
|
5750
5968
|
|
5751
|
-
elif hasattr(logging, '_lock'):
|
5752
|
-
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
5753
|
-
with logging._lock: # noqa
|
5754
|
-
yield
|
5755
5969
|
|
5756
|
-
|
5757
|
-
|
5970
|
+
class FractionObjMarshaler(ObjMarshaler):
|
5971
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5972
|
+
fr = check.isinstance(o, fractions.Fraction)
|
5973
|
+
return [fr.numerator, fr.denominator]
|
5758
5974
|
|
5975
|
+
def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5976
|
+
num, denom = check.isinstance(v, list)
|
5977
|
+
return fractions.Fraction(num, denom)
|
5759
5978
|
|
5760
|
-
def configure_standard_logging(
|
5761
|
-
level: ta.Union[int, str] = logging.INFO,
|
5762
|
-
*,
|
5763
|
-
json: bool = False,
|
5764
|
-
target: ta.Optional[logging.Logger] = None,
|
5765
|
-
force: bool = False,
|
5766
|
-
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
5767
|
-
) -> ta.Optional[StandardLogHandler]:
|
5768
|
-
with _locking_logging_module_lock():
|
5769
|
-
if target is None:
|
5770
|
-
target = logging.root
|
5771
5979
|
|
5772
|
-
|
5980
|
+
class UuidObjMarshaler(ObjMarshaler):
|
5981
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5982
|
+
return str(o)
|
5773
5983
|
|
5774
|
-
|
5775
|
-
|
5776
|
-
return None
|
5984
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5985
|
+
return uuid.UUID(o)
|
5777
5986
|
|
5778
|
-
#
|
5779
5987
|
|
5780
|
-
|
5781
|
-
handler = handler_factory()
|
5782
|
-
else:
|
5783
|
-
handler = logging.StreamHandler()
|
5988
|
+
##
|
5784
5989
|
|
5785
|
-
#
|
5786
5990
|
|
5787
|
-
|
5788
|
-
|
5789
|
-
|
5790
|
-
|
5791
|
-
|
5792
|
-
|
5991
|
+
_DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
|
5992
|
+
**{t: NopObjMarshaler() for t in (type(None),)},
|
5993
|
+
**{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
|
5994
|
+
**{t: BytesSwitchedObjMarshaler(Base64ObjMarshaler(t)) for t in (bytes, bytearray)},
|
5995
|
+
**{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
|
5996
|
+
**{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
|
5793
5997
|
|
5794
|
-
|
5998
|
+
ta.Any: DynamicObjMarshaler(),
|
5795
5999
|
|
5796
|
-
|
6000
|
+
**{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
|
6001
|
+
decimal.Decimal: DecimalObjMarshaler(),
|
6002
|
+
fractions.Fraction: FractionObjMarshaler(),
|
6003
|
+
uuid.UUID: UuidObjMarshaler(),
|
6004
|
+
}
|
5797
6005
|
|
5798
|
-
|
6006
|
+
_OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
|
6007
|
+
**{t: t for t in (dict,)},
|
6008
|
+
**{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
|
6009
|
+
}
|
5799
6010
|
|
5800
|
-
|
6011
|
+
_OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
|
6012
|
+
**{t: t for t in (list, tuple, set, frozenset)},
|
6013
|
+
collections.abc.Set: frozenset,
|
6014
|
+
collections.abc.MutableSet: set,
|
6015
|
+
collections.abc.Sequence: tuple,
|
6016
|
+
collections.abc.MutableSequence: list,
|
6017
|
+
}
|
5801
6018
|
|
5802
|
-
#
|
5803
6019
|
|
5804
|
-
|
5805
|
-
target.setLevel(level)
|
6020
|
+
##
|
5806
6021
|
|
5807
|
-
#
|
5808
6022
|
|
5809
|
-
|
6023
|
+
class ObjMarshalerManager:
|
6024
|
+
def __init__(
|
6025
|
+
self,
|
6026
|
+
*,
|
6027
|
+
default_options: ObjMarshalOptions = ObjMarshalOptions(),
|
5810
6028
|
|
6029
|
+
default_obj_marshalers: ta.Dict[ta.Any, ObjMarshaler] = _DEFAULT_OBJ_MARSHALERS, # noqa
|
6030
|
+
generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES, # noqa
|
6031
|
+
generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES, # noqa
|
6032
|
+
) -> None:
|
6033
|
+
super().__init__()
|
5811
6034
|
|
5812
|
-
|
5813
|
-
# ../../../omlish/lite/marshal.py
|
5814
|
-
"""
|
5815
|
-
TODO:
|
5816
|
-
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
5817
|
-
- namedtuple
|
5818
|
-
- literals
|
5819
|
-
- newtypes?
|
5820
|
-
"""
|
6035
|
+
self._default_options = default_options
|
5821
6036
|
|
6037
|
+
self._obj_marshalers = dict(default_obj_marshalers)
|
6038
|
+
self._generic_mapping_types = generic_mapping_types
|
6039
|
+
self._generic_iterable_types = generic_iterable_types
|
5822
6040
|
|
5823
|
-
|
6041
|
+
self._lock = threading.RLock()
|
6042
|
+
self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
|
6043
|
+
self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
|
5824
6044
|
|
6045
|
+
#
|
5825
6046
|
|
5826
|
-
|
5827
|
-
|
5828
|
-
|
5829
|
-
|
6047
|
+
def make_obj_marshaler(
|
6048
|
+
self,
|
6049
|
+
ty: ta.Any,
|
6050
|
+
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
6051
|
+
*,
|
6052
|
+
nonstrict_dataclasses: bool = False,
|
6053
|
+
) -> ObjMarshaler:
|
6054
|
+
if isinstance(ty, type):
|
6055
|
+
if abc.ABC in ty.__bases__:
|
6056
|
+
impls = [ity for ity in deep_subclasses(ty) if abc.ABC not in ity.__bases__] # type: ignore
|
6057
|
+
if all(ity.__qualname__.endswith(ty.__name__) for ity in impls):
|
6058
|
+
ins = {ity: snake_case(ity.__qualname__[:-len(ty.__name__)]) for ity in impls}
|
6059
|
+
else:
|
6060
|
+
ins = {ity: ity.__qualname__ for ity in impls}
|
6061
|
+
return PolymorphicObjMarshaler.of([
|
6062
|
+
PolymorphicObjMarshaler.Impl(
|
6063
|
+
ity,
|
6064
|
+
itn,
|
6065
|
+
rec(ity),
|
6066
|
+
)
|
6067
|
+
for ity, itn in ins.items()
|
6068
|
+
])
|
5830
6069
|
|
6070
|
+
if issubclass(ty, enum.Enum):
|
6071
|
+
return EnumObjMarshaler(ty)
|
5831
6072
|
|
5832
|
-
|
5833
|
-
|
5834
|
-
|
5835
|
-
|
6073
|
+
if dc.is_dataclass(ty):
|
6074
|
+
return DataclassObjMarshaler(
|
6075
|
+
ty,
|
6076
|
+
{f.name: rec(f.type) for f in dc.fields(ty)},
|
6077
|
+
nonstrict=nonstrict_dataclasses,
|
6078
|
+
)
|
5836
6079
|
|
5837
|
-
|
5838
|
-
|
5839
|
-
|
6080
|
+
if is_generic_alias(ty):
|
6081
|
+
try:
|
6082
|
+
mt = self._generic_mapping_types[ta.get_origin(ty)]
|
6083
|
+
except KeyError:
|
6084
|
+
pass
|
6085
|
+
else:
|
6086
|
+
k, v = ta.get_args(ty)
|
6087
|
+
return MappingObjMarshaler(mt, rec(k), rec(v))
|
5840
6088
|
|
6089
|
+
try:
|
6090
|
+
st = self._generic_iterable_types[ta.get_origin(ty)]
|
6091
|
+
except KeyError:
|
6092
|
+
pass
|
6093
|
+
else:
|
6094
|
+
[e] = ta.get_args(ty)
|
6095
|
+
return IterableObjMarshaler(st, rec(e))
|
5841
6096
|
|
5842
|
-
|
5843
|
-
|
5844
|
-
return o
|
6097
|
+
if is_union_alias(ty):
|
6098
|
+
return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
|
5845
6099
|
|
5846
|
-
|
5847
|
-
return o
|
6100
|
+
raise TypeError(ty)
|
5848
6101
|
|
6102
|
+
#
|
5849
6103
|
|
5850
|
-
|
5851
|
-
|
5852
|
-
|
6104
|
+
def register_opj_marshaler(self, ty: ta.Any, m: ObjMarshaler) -> None:
|
6105
|
+
with self._lock:
|
6106
|
+
if ty in self._obj_marshalers:
|
6107
|
+
raise KeyError(ty)
|
6108
|
+
self._obj_marshalers[ty] = m
|
5853
6109
|
|
5854
|
-
def
|
5855
|
-
|
6110
|
+
def get_obj_marshaler(
|
6111
|
+
self,
|
6112
|
+
ty: ta.Any,
|
6113
|
+
*,
|
6114
|
+
no_cache: bool = False,
|
6115
|
+
**kwargs: ta.Any,
|
6116
|
+
) -> ObjMarshaler:
|
6117
|
+
with self._lock:
|
6118
|
+
if not no_cache:
|
6119
|
+
try:
|
6120
|
+
return self._obj_marshalers[ty]
|
6121
|
+
except KeyError:
|
6122
|
+
pass
|
5856
6123
|
|
5857
|
-
|
5858
|
-
|
6124
|
+
try:
|
6125
|
+
return self._proxies[ty]
|
6126
|
+
except KeyError:
|
6127
|
+
pass
|
5859
6128
|
|
6129
|
+
rec = functools.partial(
|
6130
|
+
self.get_obj_marshaler,
|
6131
|
+
no_cache=no_cache,
|
6132
|
+
**kwargs,
|
6133
|
+
)
|
5860
6134
|
|
5861
|
-
|
5862
|
-
|
5863
|
-
|
6135
|
+
p = ProxyObjMarshaler()
|
6136
|
+
self._proxies[ty] = p
|
6137
|
+
try:
|
6138
|
+
m = self.make_obj_marshaler(ty, rec, **kwargs)
|
6139
|
+
finally:
|
6140
|
+
del self._proxies[ty]
|
6141
|
+
p.m = m
|
5864
6142
|
|
5865
|
-
|
5866
|
-
|
6143
|
+
if not no_cache:
|
6144
|
+
self._obj_marshalers[ty] = m
|
6145
|
+
return m
|
5867
6146
|
|
5868
|
-
|
5869
|
-
return self.ty(o)
|
6147
|
+
#
|
5870
6148
|
|
6149
|
+
def _make_context(self, opts: ta.Optional[ObjMarshalOptions]) -> 'ObjMarshalContext':
|
6150
|
+
return ObjMarshalContext(
|
6151
|
+
options=opts or self._default_options,
|
6152
|
+
manager=self,
|
6153
|
+
)
|
5871
6154
|
|
5872
|
-
|
5873
|
-
|
5874
|
-
|
6155
|
+
def marshal_obj(
|
6156
|
+
self,
|
6157
|
+
o: ta.Any,
|
6158
|
+
ty: ta.Any = None,
|
6159
|
+
opts: ta.Optional[ObjMarshalOptions] = None,
|
6160
|
+
) -> ta.Any:
|
6161
|
+
m = self.get_obj_marshaler(ty if ty is not None else type(o))
|
6162
|
+
return m.marshal(o, self._make_context(opts))
|
5875
6163
|
|
5876
|
-
def
|
5877
|
-
|
6164
|
+
def unmarshal_obj(
|
6165
|
+
self,
|
6166
|
+
o: ta.Any,
|
6167
|
+
ty: ta.Union[ta.Type[T], ta.Any],
|
6168
|
+
opts: ta.Optional[ObjMarshalOptions] = None,
|
6169
|
+
) -> T:
|
6170
|
+
m = self.get_obj_marshaler(ty)
|
6171
|
+
return m.unmarshal(o, self._make_context(opts))
|
6172
|
+
|
6173
|
+
def roundtrip_obj(
|
6174
|
+
self,
|
6175
|
+
o: ta.Any,
|
6176
|
+
ty: ta.Any = None,
|
6177
|
+
opts: ta.Optional[ObjMarshalOptions] = None,
|
6178
|
+
) -> ta.Any:
|
6179
|
+
if ty is None:
|
6180
|
+
ty = type(o)
|
6181
|
+
m: ta.Any = self.marshal_obj(o, ty, opts)
|
6182
|
+
u: ta.Any = self.unmarshal_obj(m, ty, opts)
|
6183
|
+
return u
|
5878
6184
|
|
5879
6185
|
|
5880
6186
|
@dc.dataclass(frozen=True)
|
5881
|
-
class
|
5882
|
-
|
6187
|
+
class ObjMarshalContext:
|
6188
|
+
options: ObjMarshalOptions
|
6189
|
+
manager: ObjMarshalerManager
|
5883
6190
|
|
5884
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5885
|
-
return base64.b64encode(o).decode('ascii')
|
5886
6191
|
|
5887
|
-
|
5888
|
-
return self.ty(base64.b64decode(o))
|
6192
|
+
##
|
5889
6193
|
|
5890
6194
|
|
5891
|
-
|
5892
|
-
class BytesSwitchedObjMarshaler(ObjMarshaler):
|
5893
|
-
m: ObjMarshaler
|
6195
|
+
OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
|
5894
6196
|
|
5895
|
-
|
5896
|
-
|
5897
|
-
return o
|
5898
|
-
return self.m.marshal(o, ctx)
|
6197
|
+
register_opj_marshaler = OBJ_MARSHALER_MANAGER.register_opj_marshaler
|
6198
|
+
get_obj_marshaler = OBJ_MARSHALER_MANAGER.get_obj_marshaler
|
5899
6199
|
|
5900
|
-
|
5901
|
-
|
5902
|
-
return o
|
5903
|
-
return self.m.unmarshal(o, ctx)
|
6200
|
+
marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
|
6201
|
+
unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj
|
5904
6202
|
|
5905
6203
|
|
5906
|
-
|
5907
|
-
|
5908
|
-
ty: type
|
6204
|
+
########################################
|
6205
|
+
# ../../../omlish/lite/runtime.py
|
5909
6206
|
|
5910
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5911
|
-
return o.name
|
5912
6207
|
|
5913
|
-
|
5914
|
-
|
6208
|
+
@cached_nullary
|
6209
|
+
def is_debugger_attached() -> bool:
|
6210
|
+
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
5915
6211
|
|
5916
6212
|
|
5917
|
-
|
5918
|
-
class OptionalObjMarshaler(ObjMarshaler):
|
5919
|
-
item: ObjMarshaler
|
6213
|
+
REQUIRED_PYTHON_VERSION = (3, 8)
|
5920
6214
|
|
5921
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5922
|
-
if o is None:
|
5923
|
-
return None
|
5924
|
-
return self.item.marshal(o, ctx)
|
5925
6215
|
|
5926
|
-
|
5927
|
-
|
5928
|
-
|
5929
|
-
return self.item.unmarshal(o, ctx)
|
6216
|
+
def check_runtime_version() -> None:
|
6217
|
+
if sys.version_info < REQUIRED_PYTHON_VERSION:
|
6218
|
+
raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
|
5930
6219
|
|
5931
6220
|
|
5932
|
-
|
5933
|
-
|
5934
|
-
|
5935
|
-
|
5936
|
-
|
6221
|
+
########################################
|
6222
|
+
# ../../../omlish/logs/json.py
|
6223
|
+
"""
|
6224
|
+
TODO:
|
6225
|
+
- translate json keys
|
6226
|
+
"""
|
5937
6227
|
|
5938
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5939
|
-
return {self.km.marshal(k, ctx): self.vm.marshal(v, ctx) for k, v in o.items()}
|
5940
6228
|
|
5941
|
-
|
5942
|
-
|
6229
|
+
class JsonLogFormatter(logging.Formatter):
|
6230
|
+
KEYS: ta.Mapping[str, bool] = {
|
6231
|
+
'name': False,
|
6232
|
+
'msg': False,
|
6233
|
+
'args': False,
|
6234
|
+
'levelname': False,
|
6235
|
+
'levelno': False,
|
6236
|
+
'pathname': False,
|
6237
|
+
'filename': False,
|
6238
|
+
'module': False,
|
6239
|
+
'exc_info': True,
|
6240
|
+
'exc_text': True,
|
6241
|
+
'stack_info': True,
|
6242
|
+
'lineno': False,
|
6243
|
+
'funcName': False,
|
6244
|
+
'created': False,
|
6245
|
+
'msecs': False,
|
6246
|
+
'relativeCreated': False,
|
6247
|
+
'thread': False,
|
6248
|
+
'threadName': False,
|
6249
|
+
'processName': False,
|
6250
|
+
'process': False,
|
6251
|
+
}
|
5943
6252
|
|
6253
|
+
def __init__(
|
6254
|
+
self,
|
6255
|
+
*args: ta.Any,
|
6256
|
+
json_dumps: ta.Optional[ta.Callable[[ta.Any], str]] = None,
|
6257
|
+
**kwargs: ta.Any,
|
6258
|
+
) -> None:
|
6259
|
+
super().__init__(*args, **kwargs)
|
5944
6260
|
|
5945
|
-
|
5946
|
-
|
5947
|
-
|
5948
|
-
item: ObjMarshaler
|
6261
|
+
if json_dumps is None:
|
6262
|
+
json_dumps = json_dumps_compact
|
6263
|
+
self._json_dumps = json_dumps
|
5949
6264
|
|
5950
|
-
def
|
5951
|
-
|
6265
|
+
def format(self, record: logging.LogRecord) -> str:
|
6266
|
+
dct = {
|
6267
|
+
k: v
|
6268
|
+
for k, o in self.KEYS.items()
|
6269
|
+
for v in [getattr(record, k)]
|
6270
|
+
if not (o and v is None)
|
6271
|
+
}
|
6272
|
+
return self._json_dumps(dct)
|
5952
6273
|
|
5953
|
-
|
5954
|
-
|
6274
|
+
|
6275
|
+
########################################
|
6276
|
+
# ../../../omdev/interp/types.py
|
6277
|
+
|
6278
|
+
|
6279
|
+
# See https://peps.python.org/pep-3149/
|
6280
|
+
INTERP_OPT_GLYPHS_BY_ATTR: ta.Mapping[str, str] = collections.OrderedDict([
|
6281
|
+
('debug', 'd'),
|
6282
|
+
('threaded', 't'),
|
6283
|
+
])
|
6284
|
+
|
6285
|
+
INTERP_OPT_ATTRS_BY_GLYPH: ta.Mapping[str, str] = collections.OrderedDict(
|
6286
|
+
(g, a) for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items()
|
6287
|
+
)
|
5955
6288
|
|
5956
6289
|
|
5957
6290
|
@dc.dataclass(frozen=True)
|
5958
|
-
class
|
5959
|
-
|
5960
|
-
|
5961
|
-
nonstrict: bool = False
|
6291
|
+
class InterpOpts:
|
6292
|
+
threaded: bool = False
|
6293
|
+
debug: bool = False
|
5962
6294
|
|
5963
|
-
def
|
5964
|
-
return
|
5965
|
-
k: m.marshal(getattr(o, k), ctx)
|
5966
|
-
for k, m in self.fs.items()
|
5967
|
-
}
|
6295
|
+
def __str__(self) -> str:
|
6296
|
+
return ''.join(g for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items() if getattr(self, a))
|
5968
6297
|
|
5969
|
-
|
5970
|
-
|
5971
|
-
|
5972
|
-
|
5973
|
-
|
5974
|
-
|
6298
|
+
@classmethod
|
6299
|
+
def parse(cls, s: str) -> 'InterpOpts':
|
6300
|
+
return cls(**{INTERP_OPT_ATTRS_BY_GLYPH[g]: True for g in s})
|
6301
|
+
|
6302
|
+
@classmethod
|
6303
|
+
def parse_suffix(cls, s: str) -> ta.Tuple[str, 'InterpOpts']:
|
6304
|
+
kw = {}
|
6305
|
+
while s and (a := INTERP_OPT_ATTRS_BY_GLYPH.get(s[-1])):
|
6306
|
+
s, kw[a] = s[:-1], True
|
6307
|
+
return s, cls(**kw)
|
5975
6308
|
|
5976
6309
|
|
5977
6310
|
@dc.dataclass(frozen=True)
|
5978
|
-
class
|
5979
|
-
|
5980
|
-
|
5981
|
-
tag: str
|
5982
|
-
m: ObjMarshaler
|
6311
|
+
class InterpVersion:
|
6312
|
+
version: Version
|
6313
|
+
opts: InterpOpts
|
5983
6314
|
|
5984
|
-
|
5985
|
-
|
6315
|
+
def __str__(self) -> str:
|
6316
|
+
return str(self.version) + str(self.opts)
|
5986
6317
|
|
5987
6318
|
@classmethod
|
5988
|
-
def
|
6319
|
+
def parse(cls, s: str) -> 'InterpVersion':
|
6320
|
+
s, o = InterpOpts.parse_suffix(s)
|
6321
|
+
v = Version(s)
|
5989
6322
|
return cls(
|
5990
|
-
|
5991
|
-
|
6323
|
+
version=v,
|
6324
|
+
opts=o,
|
5992
6325
|
)
|
5993
6326
|
|
5994
|
-
|
5995
|
-
|
5996
|
-
|
5997
|
-
|
5998
|
-
|
5999
|
-
|
6000
|
-
impl = self.impls_by_tag[t]
|
6001
|
-
return impl.m.unmarshal(v, ctx)
|
6327
|
+
@classmethod
|
6328
|
+
def try_parse(cls, s: str) -> ta.Optional['InterpVersion']:
|
6329
|
+
try:
|
6330
|
+
return cls.parse(s)
|
6331
|
+
except (KeyError, InvalidVersion):
|
6332
|
+
return None
|
6002
6333
|
|
6003
6334
|
|
6004
6335
|
@dc.dataclass(frozen=True)
|
6005
|
-
class
|
6006
|
-
|
6336
|
+
class InterpSpecifier:
|
6337
|
+
specifier: Specifier
|
6338
|
+
opts: InterpOpts
|
6007
6339
|
|
6008
|
-
def
|
6009
|
-
return
|
6340
|
+
def __str__(self) -> str:
|
6341
|
+
return str(self.specifier) + str(self.opts)
|
6010
6342
|
|
6011
|
-
|
6012
|
-
|
6343
|
+
@classmethod
|
6344
|
+
def parse(cls, s: str) -> 'InterpSpecifier':
|
6345
|
+
s, o = InterpOpts.parse_suffix(s)
|
6346
|
+
if not any(s.startswith(o) for o in Specifier.OPERATORS):
|
6347
|
+
s = '~=' + s
|
6348
|
+
if s.count('.') < 2:
|
6349
|
+
s += '.0'
|
6350
|
+
return cls(
|
6351
|
+
specifier=Specifier(s),
|
6352
|
+
opts=o,
|
6353
|
+
)
|
6013
6354
|
|
6355
|
+
def contains(self, iv: InterpVersion) -> bool:
|
6356
|
+
return self.specifier.contains(iv.version) and self.opts == iv.opts
|
6014
6357
|
|
6015
|
-
|
6016
|
-
|
6017
|
-
return str(check.isinstance(o, decimal.Decimal))
|
6358
|
+
def __contains__(self, iv: InterpVersion) -> bool:
|
6359
|
+
return self.contains(iv)
|
6018
6360
|
|
6019
|
-
def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
6020
|
-
return decimal.Decimal(check.isinstance(v, str))
|
6021
6361
|
|
6362
|
+
@dc.dataclass(frozen=True)
|
6363
|
+
class Interp:
|
6364
|
+
exe: str
|
6365
|
+
version: InterpVersion
|
6022
6366
|
|
6023
|
-
class FractionObjMarshaler(ObjMarshaler):
|
6024
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
6025
|
-
fr = check.isinstance(o, fractions.Fraction)
|
6026
|
-
return [fr.numerator, fr.denominator]
|
6027
6367
|
|
6028
|
-
|
6029
|
-
|
6030
|
-
return fractions.Fraction(num, denom)
|
6368
|
+
########################################
|
6369
|
+
# ../../configs.py
|
6031
6370
|
|
6032
6371
|
|
6033
|
-
|
6034
|
-
|
6035
|
-
|
6372
|
+
def parse_config_file(
|
6373
|
+
name: str,
|
6374
|
+
f: ta.TextIO,
|
6375
|
+
) -> ConfigMapping:
|
6376
|
+
if name.endswith('.toml'):
|
6377
|
+
return toml_loads(f.read())
|
6036
6378
|
|
6037
|
-
|
6038
|
-
|
6379
|
+
elif any(name.endswith(e) for e in ('.yml', '.yaml')):
|
6380
|
+
yaml = __import__('yaml')
|
6381
|
+
return yaml.safe_load(f)
|
6039
6382
|
|
6383
|
+
elif name.endswith('.ini'):
|
6384
|
+
import configparser
|
6385
|
+
cp = configparser.ConfigParser()
|
6386
|
+
cp.read_file(f)
|
6387
|
+
config_dct: ta.Dict[str, ta.Any] = {}
|
6388
|
+
for sec in cp.sections():
|
6389
|
+
cd = config_dct
|
6390
|
+
for k in sec.split('.'):
|
6391
|
+
cd = cd.setdefault(k, {})
|
6392
|
+
cd.update(cp.items(sec))
|
6393
|
+
return config_dct
|
6040
6394
|
|
6041
|
-
|
6395
|
+
else:
|
6396
|
+
return json.loads(f.read())
|
6042
6397
|
|
6043
6398
|
|
6044
|
-
|
6045
|
-
|
6046
|
-
|
6047
|
-
|
6048
|
-
|
6049
|
-
|
6050
|
-
|
6051
|
-
|
6399
|
+
def read_config_file(
|
6400
|
+
path: str,
|
6401
|
+
cls: ta.Type[T],
|
6402
|
+
*,
|
6403
|
+
prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
|
6404
|
+
) -> T:
|
6405
|
+
with open(path) as cf:
|
6406
|
+
config_dct = parse_config_file(os.path.basename(path), cf)
|
6052
6407
|
|
6053
|
-
|
6054
|
-
|
6055
|
-
fractions.Fraction: FractionObjMarshaler(),
|
6056
|
-
uuid.UUID: UuidObjMarshaler(),
|
6057
|
-
}
|
6408
|
+
if prepare is not None:
|
6409
|
+
config_dct = prepare(config_dct)
|
6058
6410
|
|
6059
|
-
|
6060
|
-
**{t: t for t in (dict,)},
|
6061
|
-
**{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
|
6062
|
-
}
|
6411
|
+
return unmarshal_obj(config_dct, cls)
|
6063
6412
|
|
6064
|
-
_OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
|
6065
|
-
**{t: t for t in (list, tuple, set, frozenset)},
|
6066
|
-
collections.abc.Set: frozenset,
|
6067
|
-
collections.abc.MutableSet: set,
|
6068
|
-
collections.abc.Sequence: tuple,
|
6069
|
-
collections.abc.MutableSequence: list,
|
6070
|
-
}
|
6071
6413
|
|
6414
|
+
def build_config_named_children(
|
6415
|
+
o: ta.Union[
|
6416
|
+
ta.Sequence[ConfigMapping],
|
6417
|
+
ta.Mapping[str, ConfigMapping],
|
6418
|
+
None,
|
6419
|
+
],
|
6420
|
+
*,
|
6421
|
+
name_key: str = 'name',
|
6422
|
+
) -> ta.Optional[ta.Sequence[ConfigMapping]]:
|
6423
|
+
if o is None:
|
6424
|
+
return None
|
6072
6425
|
|
6073
|
-
|
6426
|
+
lst: ta.List[ConfigMapping] = []
|
6427
|
+
if isinstance(o, ta.Mapping):
|
6428
|
+
for k, v in o.items():
|
6429
|
+
check.isinstance(v, ta.Mapping)
|
6430
|
+
if name_key in v:
|
6431
|
+
n = v[name_key]
|
6432
|
+
if k != n:
|
6433
|
+
raise KeyError(f'Given names do not match: {n} != {k}')
|
6434
|
+
lst.append(v)
|
6435
|
+
else:
|
6436
|
+
lst.append({name_key: k, **v})
|
6074
6437
|
|
6438
|
+
else:
|
6439
|
+
check.not_isinstance(o, str)
|
6440
|
+
lst.extend(o)
|
6075
6441
|
|
6076
|
-
|
6077
|
-
|
6078
|
-
|
6079
|
-
|
6080
|
-
|
6442
|
+
seen = set()
|
6443
|
+
for d in lst:
|
6444
|
+
n = d['name']
|
6445
|
+
if n in d:
|
6446
|
+
raise KeyError(f'Duplicate name: {n}')
|
6447
|
+
seen.add(n)
|
6081
6448
|
|
6082
|
-
|
6083
|
-
generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES, # noqa
|
6084
|
-
generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES, # noqa
|
6085
|
-
) -> None:
|
6086
|
-
super().__init__()
|
6449
|
+
return lst
|
6087
6450
|
|
6088
|
-
self._default_options = default_options
|
6089
6451
|
|
6090
|
-
|
6091
|
-
|
6092
|
-
self._generic_iterable_types = generic_iterable_types
|
6452
|
+
########################################
|
6453
|
+
# ../commands/marshal.py
|
6093
6454
|
|
6094
|
-
self._lock = threading.RLock()
|
6095
|
-
self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
|
6096
|
-
self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
|
6097
6455
|
|
6098
|
-
|
6456
|
+
def install_command_marshaling(
|
6457
|
+
cmds: CommandNameMap,
|
6458
|
+
msh: ObjMarshalerManager,
|
6459
|
+
) -> None:
|
6460
|
+
for fn in [
|
6461
|
+
lambda c: c,
|
6462
|
+
lambda c: c.Output,
|
6463
|
+
]:
|
6464
|
+
msh.register_opj_marshaler(
|
6465
|
+
fn(Command),
|
6466
|
+
PolymorphicObjMarshaler.of([
|
6467
|
+
PolymorphicObjMarshaler.Impl(
|
6468
|
+
fn(cmd),
|
6469
|
+
name,
|
6470
|
+
msh.get_obj_marshaler(fn(cmd)),
|
6471
|
+
)
|
6472
|
+
for name, cmd in cmds.items()
|
6473
|
+
]),
|
6474
|
+
)
|
6099
6475
|
|
6100
|
-
def make_obj_marshaler(
|
6101
|
-
self,
|
6102
|
-
ty: ta.Any,
|
6103
|
-
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
6104
|
-
*,
|
6105
|
-
nonstrict_dataclasses: bool = False,
|
6106
|
-
) -> ObjMarshaler:
|
6107
|
-
if isinstance(ty, type):
|
6108
|
-
if abc.ABC in ty.__bases__:
|
6109
|
-
impls = [ity for ity in deep_subclasses(ty) if abc.ABC not in ity.__bases__] # type: ignore
|
6110
|
-
if all(ity.__qualname__.endswith(ty.__name__) for ity in impls):
|
6111
|
-
ins = {ity: snake_case(ity.__qualname__[:-len(ty.__name__)]) for ity in impls}
|
6112
|
-
else:
|
6113
|
-
ins = {ity: ity.__qualname__ for ity in impls}
|
6114
|
-
return PolymorphicObjMarshaler.of([
|
6115
|
-
PolymorphicObjMarshaler.Impl(
|
6116
|
-
ity,
|
6117
|
-
itn,
|
6118
|
-
rec(ity),
|
6119
|
-
)
|
6120
|
-
for ity, itn in ins.items()
|
6121
|
-
])
|
6122
6476
|
|
6123
|
-
|
6124
|
-
|
6477
|
+
########################################
|
6478
|
+
# ../commands/ping.py
|
6125
6479
|
|
6126
|
-
if dc.is_dataclass(ty):
|
6127
|
-
return DataclassObjMarshaler(
|
6128
|
-
ty,
|
6129
|
-
{f.name: rec(f.type) for f in dc.fields(ty)},
|
6130
|
-
nonstrict=nonstrict_dataclasses,
|
6131
|
-
)
|
6132
6480
|
|
6133
|
-
|
6134
|
-
try:
|
6135
|
-
mt = self._generic_mapping_types[ta.get_origin(ty)]
|
6136
|
-
except KeyError:
|
6137
|
-
pass
|
6138
|
-
else:
|
6139
|
-
k, v = ta.get_args(ty)
|
6140
|
-
return MappingObjMarshaler(mt, rec(k), rec(v))
|
6481
|
+
##
|
6141
6482
|
|
6142
|
-
try:
|
6143
|
-
st = self._generic_iterable_types[ta.get_origin(ty)]
|
6144
|
-
except KeyError:
|
6145
|
-
pass
|
6146
|
-
else:
|
6147
|
-
[e] = ta.get_args(ty)
|
6148
|
-
return IterableObjMarshaler(st, rec(e))
|
6149
6483
|
|
6150
|
-
|
6151
|
-
|
6484
|
+
@dc.dataclass(frozen=True)
|
6485
|
+
class PingCommand(Command['PingCommand.Output']):
|
6486
|
+
time: float = dc.field(default_factory=time.time)
|
6152
6487
|
|
6153
|
-
|
6488
|
+
@dc.dataclass(frozen=True)
|
6489
|
+
class Output(Command.Output):
|
6490
|
+
time: float
|
6154
6491
|
|
6155
|
-
#
|
6156
6492
|
|
6157
|
-
|
6158
|
-
|
6159
|
-
|
6160
|
-
raise KeyError(ty)
|
6161
|
-
self._obj_marshalers[ty] = m
|
6493
|
+
class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
|
6494
|
+
async def execute(self, cmd: PingCommand) -> PingCommand.Output:
|
6495
|
+
return PingCommand.Output(cmd.time)
|
6162
6496
|
|
6163
|
-
def get_obj_marshaler(
|
6164
|
-
self,
|
6165
|
-
ty: ta.Any,
|
6166
|
-
*,
|
6167
|
-
no_cache: bool = False,
|
6168
|
-
**kwargs: ta.Any,
|
6169
|
-
) -> ObjMarshaler:
|
6170
|
-
with self._lock:
|
6171
|
-
if not no_cache:
|
6172
|
-
try:
|
6173
|
-
return self._obj_marshalers[ty]
|
6174
|
-
except KeyError:
|
6175
|
-
pass
|
6176
6497
|
|
6177
|
-
|
6178
|
-
|
6179
|
-
except KeyError:
|
6180
|
-
pass
|
6498
|
+
########################################
|
6499
|
+
# ../commands/types.py
|
6181
6500
|
|
6182
|
-
rec = functools.partial(
|
6183
|
-
self.get_obj_marshaler,
|
6184
|
-
no_cache=no_cache,
|
6185
|
-
**kwargs,
|
6186
|
-
)
|
6187
6501
|
|
6188
|
-
|
6189
|
-
self._proxies[ty] = p
|
6190
|
-
try:
|
6191
|
-
m = self.make_obj_marshaler(ty, rec, **kwargs)
|
6192
|
-
finally:
|
6193
|
-
del self._proxies[ty]
|
6194
|
-
p.m = m
|
6502
|
+
CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
|
6195
6503
|
|
6196
|
-
if not no_cache:
|
6197
|
-
self._obj_marshalers[ty] = m
|
6198
|
-
return m
|
6199
6504
|
|
6200
|
-
|
6505
|
+
########################################
|
6506
|
+
# ../deploy/commands.py
|
6201
6507
|
|
6202
|
-
def _make_context(self, opts: ta.Optional[ObjMarshalOptions]) -> 'ObjMarshalContext':
|
6203
|
-
return ObjMarshalContext(
|
6204
|
-
options=opts or self._default_options,
|
6205
|
-
manager=self,
|
6206
|
-
)
|
6207
6508
|
|
6208
|
-
|
6209
|
-
self,
|
6210
|
-
o: ta.Any,
|
6211
|
-
ty: ta.Any = None,
|
6212
|
-
opts: ta.Optional[ObjMarshalOptions] = None,
|
6213
|
-
) -> ta.Any:
|
6214
|
-
m = self.get_obj_marshaler(ty if ty is not None else type(o))
|
6215
|
-
return m.marshal(o, self._make_context(opts))
|
6509
|
+
##
|
6216
6510
|
|
6217
|
-
def unmarshal_obj(
|
6218
|
-
self,
|
6219
|
-
o: ta.Any,
|
6220
|
-
ty: ta.Union[ta.Type[T], ta.Any],
|
6221
|
-
opts: ta.Optional[ObjMarshalOptions] = None,
|
6222
|
-
) -> T:
|
6223
|
-
m = self.get_obj_marshaler(ty)
|
6224
|
-
return m.unmarshal(o, self._make_context(opts))
|
6225
6511
|
|
6226
|
-
|
6227
|
-
|
6228
|
-
|
6229
|
-
|
6230
|
-
|
6231
|
-
) -> ta.Any:
|
6232
|
-
if ty is None:
|
6233
|
-
ty = type(o)
|
6234
|
-
m: ta.Any = self.marshal_obj(o, ty, opts)
|
6235
|
-
u: ta.Any = self.unmarshal_obj(m, ty, opts)
|
6236
|
-
return u
|
6512
|
+
@dc.dataclass(frozen=True)
|
6513
|
+
class DeployCommand(Command['DeployCommand.Output']):
|
6514
|
+
@dc.dataclass(frozen=True)
|
6515
|
+
class Output(Command.Output):
|
6516
|
+
pass
|
6237
6517
|
|
6238
6518
|
|
6239
|
-
|
6240
|
-
|
6241
|
-
|
6242
|
-
manager: ObjMarshalerManager
|
6519
|
+
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
6520
|
+
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
6521
|
+
log.info('Deploying!')
|
6243
6522
|
|
6523
|
+
return DeployCommand.Output()
|
6244
6524
|
|
6245
|
-
##
|
6246
6525
|
|
6526
|
+
########################################
|
6527
|
+
# ../marshal.py
|
6247
6528
|
|
6248
|
-
OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
|
6249
6529
|
|
6250
|
-
|
6251
|
-
|
6252
|
-
|
6253
|
-
marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
|
6254
|
-
unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj
|
6530
|
+
@dc.dataclass(frozen=True)
|
6531
|
+
class ObjMarshalerInstaller:
|
6532
|
+
fn: ta.Callable[[ObjMarshalerManager], None]
|
6255
6533
|
|
6256
6534
|
|
6257
|
-
|
6258
|
-
# ../../../omlish/lite/runtime.py
|
6535
|
+
ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMarshalerInstaller])
|
6259
6536
|
|
6260
6537
|
|
6261
|
-
|
6262
|
-
|
6263
|
-
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
6538
|
+
########################################
|
6539
|
+
# ../remote/channel.py
|
6264
6540
|
|
6265
6541
|
|
6266
|
-
|
6542
|
+
##
|
6267
6543
|
|
6268
6544
|
|
6269
|
-
|
6270
|
-
|
6271
|
-
|
6545
|
+
class RemoteChannel(abc.ABC):
|
6546
|
+
@abc.abstractmethod
|
6547
|
+
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
|
6548
|
+
raise NotImplementedError
|
6272
6549
|
|
6550
|
+
@abc.abstractmethod
|
6551
|
+
def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
|
6552
|
+
raise NotImplementedError
|
6273
6553
|
|
6274
|
-
|
6275
|
-
|
6554
|
+
def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
|
6555
|
+
pass
|
6276
6556
|
|
6277
6557
|
|
6278
|
-
|
6279
|
-
INTERP_OPT_GLYPHS_BY_ATTR: ta.Mapping[str, str] = collections.OrderedDict([
|
6280
|
-
('debug', 'd'),
|
6281
|
-
('threaded', 't'),
|
6282
|
-
])
|
6558
|
+
##
|
6283
6559
|
|
6284
|
-
INTERP_OPT_ATTRS_BY_GLYPH: ta.Mapping[str, str] = collections.OrderedDict(
|
6285
|
-
(g, a) for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items()
|
6286
|
-
)
|
6287
6560
|
|
6561
|
+
class RemoteChannelImpl(RemoteChannel):
|
6562
|
+
def __init__(
|
6563
|
+
self,
|
6564
|
+
input: asyncio.StreamReader, # noqa
|
6565
|
+
output: asyncio.StreamWriter,
|
6566
|
+
*,
|
6567
|
+
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
6568
|
+
) -> None:
|
6569
|
+
super().__init__()
|
6288
6570
|
|
6289
|
-
|
6290
|
-
|
6291
|
-
|
6292
|
-
debug: bool = False
|
6571
|
+
self._input = input
|
6572
|
+
self._output = output
|
6573
|
+
self._msh = msh
|
6293
6574
|
|
6294
|
-
|
6295
|
-
|
6575
|
+
self._input_lock = asyncio.Lock()
|
6576
|
+
self._output_lock = asyncio.Lock()
|
6296
6577
|
|
6297
|
-
|
6298
|
-
|
6299
|
-
return cls(**{INTERP_OPT_ATTRS_BY_GLYPH[g]: True for g in s})
|
6578
|
+
def set_marshaler(self, msh: ObjMarshalerManager) -> None:
|
6579
|
+
self._msh = msh
|
6300
6580
|
|
6301
|
-
|
6302
|
-
def parse_suffix(cls, s: str) -> ta.Tuple[str, 'InterpOpts']:
|
6303
|
-
kw = {}
|
6304
|
-
while s and (a := INTERP_OPT_ATTRS_BY_GLYPH.get(s[-1])):
|
6305
|
-
s, kw[a] = s[:-1], True
|
6306
|
-
return s, cls(**kw)
|
6581
|
+
#
|
6307
6582
|
|
6583
|
+
async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
6584
|
+
j = json_dumps_compact(self._msh.marshal_obj(o, ty))
|
6585
|
+
d = j.encode('utf-8')
|
6308
6586
|
|
6309
|
-
|
6310
|
-
|
6311
|
-
|
6312
|
-
opts: InterpOpts
|
6587
|
+
self._output.write(struct.pack('<I', len(d)))
|
6588
|
+
self._output.write(d)
|
6589
|
+
await self._output.drain()
|
6313
6590
|
|
6314
|
-
def
|
6315
|
-
|
6591
|
+
async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
6592
|
+
async with self._output_lock:
|
6593
|
+
return await self._send_obj(o, ty)
|
6316
6594
|
|
6317
|
-
|
6318
|
-
def parse(cls, s: str) -> 'InterpVersion':
|
6319
|
-
s, o = InterpOpts.parse_suffix(s)
|
6320
|
-
v = Version(s)
|
6321
|
-
return cls(
|
6322
|
-
version=v,
|
6323
|
-
opts=o,
|
6324
|
-
)
|
6595
|
+
#
|
6325
6596
|
|
6326
|
-
|
6327
|
-
|
6328
|
-
|
6329
|
-
return cls.parse(s)
|
6330
|
-
except (KeyError, InvalidVersion):
|
6597
|
+
async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
6598
|
+
d = await self._input.read(4)
|
6599
|
+
if not d:
|
6331
6600
|
return None
|
6601
|
+
if len(d) != 4:
|
6602
|
+
raise EOFError
|
6332
6603
|
|
6604
|
+
sz = struct.unpack('<I', d)[0]
|
6605
|
+
d = await self._input.read(sz)
|
6606
|
+
if len(d) != sz:
|
6607
|
+
raise EOFError
|
6333
6608
|
|
6334
|
-
|
6335
|
-
|
6336
|
-
specifier: Specifier
|
6337
|
-
opts: InterpOpts
|
6338
|
-
|
6339
|
-
def __str__(self) -> str:
|
6340
|
-
return str(self.specifier) + str(self.opts)
|
6609
|
+
j = json.loads(d.decode('utf-8'))
|
6610
|
+
return self._msh.unmarshal_obj(j, ty)
|
6341
6611
|
|
6342
|
-
|
6343
|
-
|
6344
|
-
|
6345
|
-
if not any(s.startswith(o) for o in Specifier.OPERATORS):
|
6346
|
-
s = '~=' + s
|
6347
|
-
if s.count('.') < 2:
|
6348
|
-
s += '.0'
|
6349
|
-
return cls(
|
6350
|
-
specifier=Specifier(s),
|
6351
|
-
opts=o,
|
6352
|
-
)
|
6612
|
+
async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
6613
|
+
async with self._input_lock:
|
6614
|
+
return await self._recv_obj(ty)
|
6353
6615
|
|
6354
|
-
def contains(self, iv: InterpVersion) -> bool:
|
6355
|
-
return self.specifier.contains(iv.version) and self.opts == iv.opts
|
6356
6616
|
|
6357
|
-
|
6358
|
-
|
6617
|
+
########################################
|
6618
|
+
# ../system/config.py
|
6359
6619
|
|
6360
6620
|
|
6361
6621
|
@dc.dataclass(frozen=True)
|
6362
|
-
class
|
6363
|
-
|
6364
|
-
version: InterpVersion
|
6622
|
+
class SystemConfig:
|
6623
|
+
platform: ta.Optional[Platform] = None
|
6365
6624
|
|
6366
6625
|
|
6367
6626
|
########################################
|
6368
|
-
#
|
6627
|
+
# ../../../omlish/logs/standard.py
|
6628
|
+
"""
|
6629
|
+
TODO:
|
6630
|
+
- structured
|
6631
|
+
- prefixed
|
6632
|
+
- debug
|
6633
|
+
"""
|
6369
6634
|
|
6370
6635
|
|
6371
|
-
|
6372
|
-
name: str,
|
6373
|
-
f: ta.TextIO,
|
6374
|
-
) -> ConfigMapping:
|
6375
|
-
if name.endswith('.toml'):
|
6376
|
-
return toml_loads(f.read())
|
6636
|
+
##
|
6377
6637
|
|
6378
|
-
elif any(name.endswith(e) for e in ('.yml', '.yaml')):
|
6379
|
-
yaml = __import__('yaml')
|
6380
|
-
return yaml.safe_load(f)
|
6381
6638
|
|
6382
|
-
|
6383
|
-
|
6384
|
-
|
6385
|
-
|
6386
|
-
|
6387
|
-
|
6388
|
-
|
6389
|
-
|
6390
|
-
|
6391
|
-
cd.update(cp.items(sec))
|
6392
|
-
return config_dct
|
6639
|
+
STANDARD_LOG_FORMAT_PARTS = [
|
6640
|
+
('asctime', '%(asctime)-15s'),
|
6641
|
+
('process', 'pid=%(process)-6s'),
|
6642
|
+
('thread', 'tid=%(thread)x'),
|
6643
|
+
('levelname', '%(levelname)s'),
|
6644
|
+
('name', '%(name)s'),
|
6645
|
+
('separator', '::'),
|
6646
|
+
('message', '%(message)s'),
|
6647
|
+
]
|
6393
6648
|
|
6394
|
-
else:
|
6395
|
-
return json.loads(f.read())
|
6396
6649
|
|
6650
|
+
class StandardLogFormatter(logging.Formatter):
|
6651
|
+
@staticmethod
|
6652
|
+
def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
|
6653
|
+
return ' '.join(v for k, v in parts)
|
6397
6654
|
|
6398
|
-
|
6399
|
-
path: str,
|
6400
|
-
cls: ta.Type[T],
|
6401
|
-
*,
|
6402
|
-
prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
|
6403
|
-
) -> T:
|
6404
|
-
with open(path) as cf:
|
6405
|
-
config_dct = parse_config_file(os.path.basename(path), cf)
|
6655
|
+
converter = datetime.datetime.fromtimestamp # type: ignore
|
6406
6656
|
|
6407
|
-
|
6408
|
-
|
6657
|
+
def formatTime(self, record, datefmt=None):
|
6658
|
+
ct = self.converter(record.created) # type: ignore
|
6659
|
+
if datefmt:
|
6660
|
+
return ct.strftime(datefmt) # noqa
|
6661
|
+
else:
|
6662
|
+
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
6663
|
+
return '%s.%03d' % (t, record.msecs) # noqa
|
6409
6664
|
|
6410
|
-
return unmarshal_obj(config_dct, cls)
|
6411
6665
|
|
6666
|
+
##
|
6412
6667
|
|
6413
|
-
def build_config_named_children(
|
6414
|
-
o: ta.Union[
|
6415
|
-
ta.Sequence[ConfigMapping],
|
6416
|
-
ta.Mapping[str, ConfigMapping],
|
6417
|
-
None,
|
6418
|
-
],
|
6419
|
-
*,
|
6420
|
-
name_key: str = 'name',
|
6421
|
-
) -> ta.Optional[ta.Sequence[ConfigMapping]]:
|
6422
|
-
if o is None:
|
6423
|
-
return None
|
6424
6668
|
|
6425
|
-
|
6426
|
-
|
6427
|
-
for k, v in o.items():
|
6428
|
-
check.isinstance(v, ta.Mapping)
|
6429
|
-
if name_key in v:
|
6430
|
-
n = v[name_key]
|
6431
|
-
if k != n:
|
6432
|
-
raise KeyError(f'Given names do not match: {n} != {k}')
|
6433
|
-
lst.append(v)
|
6434
|
-
else:
|
6435
|
-
lst.append({name_key: k, **v})
|
6669
|
+
class StandardLogHandler(ProxyLogHandler):
|
6670
|
+
pass
|
6436
6671
|
|
6437
|
-
else:
|
6438
|
-
check.not_isinstance(o, str)
|
6439
|
-
lst.extend(o)
|
6440
6672
|
|
6441
|
-
|
6442
|
-
for d in lst:
|
6443
|
-
n = d['name']
|
6444
|
-
if n in d:
|
6445
|
-
raise KeyError(f'Duplicate name: {n}')
|
6446
|
-
seen.add(n)
|
6673
|
+
##
|
6447
6674
|
|
6448
|
-
return lst
|
6449
6675
|
|
6676
|
+
@contextlib.contextmanager
|
6677
|
+
def _locking_logging_module_lock() -> ta.Iterator[None]:
|
6678
|
+
if hasattr(logging, '_acquireLock'):
|
6679
|
+
logging._acquireLock() # noqa
|
6680
|
+
try:
|
6681
|
+
yield
|
6682
|
+
finally:
|
6683
|
+
logging._releaseLock() # type: ignore # noqa
|
6450
6684
|
|
6451
|
-
|
6452
|
-
#
|
6685
|
+
elif hasattr(logging, '_lock'):
|
6686
|
+
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
6687
|
+
with logging._lock: # noqa
|
6688
|
+
yield
|
6453
6689
|
|
6454
|
-
|
6455
|
-
|
6456
|
-
cmds: CommandNameMap,
|
6457
|
-
msh: ObjMarshalerManager,
|
6458
|
-
) -> None:
|
6459
|
-
for fn in [
|
6460
|
-
lambda c: c,
|
6461
|
-
lambda c: c.Output,
|
6462
|
-
]:
|
6463
|
-
msh.register_opj_marshaler(
|
6464
|
-
fn(Command),
|
6465
|
-
PolymorphicObjMarshaler.of([
|
6466
|
-
PolymorphicObjMarshaler.Impl(
|
6467
|
-
fn(cmd),
|
6468
|
-
name,
|
6469
|
-
msh.get_obj_marshaler(fn(cmd)),
|
6470
|
-
)
|
6471
|
-
for name, cmd in cmds.items()
|
6472
|
-
]),
|
6473
|
-
)
|
6474
|
-
|
6475
|
-
|
6476
|
-
########################################
|
6477
|
-
# ../commands/ping.py
|
6478
|
-
|
6479
|
-
|
6480
|
-
##
|
6481
|
-
|
6482
|
-
|
6483
|
-
@dc.dataclass(frozen=True)
|
6484
|
-
class PingCommand(Command['PingCommand.Output']):
|
6485
|
-
time: float = dc.field(default_factory=time.time)
|
6486
|
-
|
6487
|
-
@dc.dataclass(frozen=True)
|
6488
|
-
class Output(Command.Output):
|
6489
|
-
time: float
|
6490
|
-
|
6491
|
-
|
6492
|
-
class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
|
6493
|
-
async def execute(self, cmd: PingCommand) -> PingCommand.Output:
|
6494
|
-
return PingCommand.Output(cmd.time)
|
6495
|
-
|
6496
|
-
|
6497
|
-
########################################
|
6498
|
-
# ../commands/types.py
|
6499
|
-
|
6500
|
-
|
6501
|
-
CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
|
6502
|
-
|
6503
|
-
|
6504
|
-
########################################
|
6505
|
-
# ../deploy/commands.py
|
6506
|
-
|
6507
|
-
|
6508
|
-
##
|
6509
|
-
|
6510
|
-
|
6511
|
-
@dc.dataclass(frozen=True)
|
6512
|
-
class DeployCommand(Command['DeployCommand.Output']):
|
6513
|
-
@dc.dataclass(frozen=True)
|
6514
|
-
class Output(Command.Output):
|
6515
|
-
pass
|
6516
|
-
|
6517
|
-
|
6518
|
-
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
6519
|
-
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
6520
|
-
log.info('Deploying!')
|
6521
|
-
|
6522
|
-
return DeployCommand.Output()
|
6523
|
-
|
6524
|
-
|
6525
|
-
########################################
|
6526
|
-
# ../marshal.py
|
6527
|
-
|
6528
|
-
|
6529
|
-
@dc.dataclass(frozen=True)
|
6530
|
-
class ObjMarshalerInstaller:
|
6531
|
-
fn: ta.Callable[[ObjMarshalerManager], None]
|
6532
|
-
|
6533
|
-
|
6534
|
-
ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMarshalerInstaller])
|
6535
|
-
|
6536
|
-
|
6537
|
-
########################################
|
6538
|
-
# ../remote/channel.py
|
6539
|
-
|
6540
|
-
|
6541
|
-
##
|
6542
|
-
|
6543
|
-
|
6544
|
-
class RemoteChannel(abc.ABC):
|
6545
|
-
@abc.abstractmethod
|
6546
|
-
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
|
6547
|
-
raise NotImplementedError
|
6548
|
-
|
6549
|
-
@abc.abstractmethod
|
6550
|
-
def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
|
6551
|
-
raise NotImplementedError
|
6552
|
-
|
6553
|
-
def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
|
6554
|
-
pass
|
6555
|
-
|
6556
|
-
|
6557
|
-
##
|
6558
|
-
|
6559
|
-
|
6560
|
-
class RemoteChannelImpl(RemoteChannel):
|
6561
|
-
def __init__(
|
6562
|
-
self,
|
6563
|
-
input: asyncio.StreamReader, # noqa
|
6564
|
-
output: asyncio.StreamWriter,
|
6565
|
-
*,
|
6566
|
-
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
6567
|
-
) -> None:
|
6568
|
-
super().__init__()
|
6569
|
-
|
6570
|
-
self._input = input
|
6571
|
-
self._output = output
|
6572
|
-
self._msh = msh
|
6573
|
-
|
6574
|
-
self._input_lock = asyncio.Lock()
|
6575
|
-
self._output_lock = asyncio.Lock()
|
6576
|
-
|
6577
|
-
def set_marshaler(self, msh: ObjMarshalerManager) -> None:
|
6578
|
-
self._msh = msh
|
6579
|
-
|
6580
|
-
#
|
6581
|
-
|
6582
|
-
async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
6583
|
-
j = json_dumps_compact(self._msh.marshal_obj(o, ty))
|
6584
|
-
d = j.encode('utf-8')
|
6585
|
-
|
6586
|
-
self._output.write(struct.pack('<I', len(d)))
|
6587
|
-
self._output.write(d)
|
6588
|
-
await self._output.drain()
|
6589
|
-
|
6590
|
-
async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
6591
|
-
async with self._output_lock:
|
6592
|
-
return await self._send_obj(o, ty)
|
6593
|
-
|
6594
|
-
#
|
6595
|
-
|
6596
|
-
async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
6597
|
-
d = await self._input.read(4)
|
6598
|
-
if not d:
|
6599
|
-
return None
|
6600
|
-
if len(d) != 4:
|
6601
|
-
raise EOFError
|
6602
|
-
|
6603
|
-
sz = struct.unpack('<I', d)[0]
|
6604
|
-
d = await self._input.read(sz)
|
6605
|
-
if len(d) != sz:
|
6606
|
-
raise EOFError
|
6607
|
-
|
6608
|
-
j = json.loads(d.decode('utf-8'))
|
6609
|
-
return self._msh.unmarshal_obj(j, ty)
|
6610
|
-
|
6611
|
-
async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
6612
|
-
async with self._input_lock:
|
6613
|
-
return await self._recv_obj(ty)
|
6690
|
+
else:
|
6691
|
+
raise Exception("Can't find lock in logging module")
|
6614
6692
|
|
6615
6693
|
|
6616
|
-
|
6617
|
-
|
6618
|
-
|
6619
|
-
|
6620
|
-
|
6621
|
-
|
6622
|
-
|
6623
|
-
|
6624
|
-
|
6625
|
-
|
6626
|
-
|
6627
|
-
|
6628
|
-
class LinuxPlatform(Platform, abc.ABC):
|
6629
|
-
pass
|
6630
|
-
|
6631
|
-
|
6632
|
-
class UbuntuPlatform(LinuxPlatform):
|
6633
|
-
pass
|
6634
|
-
|
6635
|
-
|
6636
|
-
class AmazonLinuxPlatform(LinuxPlatform):
|
6637
|
-
pass
|
6638
|
-
|
6639
|
-
|
6640
|
-
class GenericLinuxPlatform(LinuxPlatform):
|
6641
|
-
pass
|
6642
|
-
|
6643
|
-
|
6644
|
-
class DarwinPlatform(Platform):
|
6645
|
-
pass
|
6694
|
+
def configure_standard_logging(
|
6695
|
+
level: ta.Union[int, str] = logging.INFO,
|
6696
|
+
*,
|
6697
|
+
json: bool = False,
|
6698
|
+
target: ta.Optional[logging.Logger] = None,
|
6699
|
+
force: bool = False,
|
6700
|
+
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
6701
|
+
) -> ta.Optional[StandardLogHandler]:
|
6702
|
+
with _locking_logging_module_lock():
|
6703
|
+
if target is None:
|
6704
|
+
target = logging.root
|
6646
6705
|
|
6706
|
+
#
|
6647
6707
|
|
6648
|
-
|
6649
|
-
|
6708
|
+
if not force:
|
6709
|
+
if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
|
6710
|
+
return None
|
6650
6711
|
|
6712
|
+
#
|
6651
6713
|
|
6652
|
-
|
6714
|
+
if handler_factory is not None:
|
6715
|
+
handler = handler_factory()
|
6716
|
+
else:
|
6717
|
+
handler = logging.StreamHandler()
|
6653
6718
|
|
6719
|
+
#
|
6654
6720
|
|
6655
|
-
|
6656
|
-
|
6721
|
+
formatter: logging.Formatter
|
6722
|
+
if json:
|
6723
|
+
formatter = JsonLogFormatter()
|
6724
|
+
else:
|
6725
|
+
formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
|
6726
|
+
handler.setFormatter(formatter)
|
6657
6727
|
|
6658
|
-
|
6659
|
-
if (osr := LinuxOsRelease.read()) is None:
|
6660
|
-
return GenericLinuxPlatform()
|
6728
|
+
#
|
6661
6729
|
|
6662
|
-
|
6663
|
-
return AmazonLinuxPlatform()
|
6730
|
+
handler.addFilter(TidLogFilter())
|
6664
6731
|
|
6665
|
-
|
6666
|
-
return UbuntuPlatform()
|
6732
|
+
#
|
6667
6733
|
|
6668
|
-
|
6669
|
-
return GenericLinuxPlatform()
|
6734
|
+
target.addHandler(handler)
|
6670
6735
|
|
6671
|
-
|
6672
|
-
return DarwinPlatform()
|
6736
|
+
#
|
6673
6737
|
|
6674
|
-
|
6675
|
-
|
6738
|
+
if level is not None:
|
6739
|
+
target.setLevel(level)
|
6676
6740
|
|
6741
|
+
#
|
6677
6742
|
|
6678
|
-
|
6679
|
-
def detect_system_platform() -> Platform:
|
6680
|
-
platform = _detect_system_platform()
|
6681
|
-
log.info('Detected platform: %r', platform)
|
6682
|
-
return platform
|
6743
|
+
return StandardLogHandler(handler)
|
6683
6744
|
|
6684
6745
|
|
6685
6746
|
########################################
|
6686
|
-
# ../../../omlish/
|
6747
|
+
# ../../../omlish/subprocesses.py
|
6687
6748
|
|
6688
6749
|
|
6689
6750
|
##
|
@@ -6734,8 +6795,8 @@ def subprocess_close(
|
|
6734
6795
|
##
|
6735
6796
|
|
6736
6797
|
|
6737
|
-
class
|
6738
|
-
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] =
|
6798
|
+
class BaseSubprocesses(abc.ABC): # noqa
|
6799
|
+
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
|
6739
6800
|
|
6740
6801
|
def __init__(
|
6741
6802
|
self,
|
@@ -6748,6 +6809,9 @@ class AbstractSubprocesses(abc.ABC): # noqa
|
|
6748
6809
|
self._log = log if log is not None else self.DEFAULT_LOGGER
|
6749
6810
|
self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
|
6750
6811
|
|
6812
|
+
def set_logger(self, log: ta.Optional[logging.Logger]) -> None:
|
6813
|
+
self._log = log
|
6814
|
+
|
6751
6815
|
#
|
6752
6816
|
|
6753
6817
|
def prepare_args(
|
@@ -6859,23 +6923,25 @@ class AbstractSubprocesses(abc.ABC): # noqa
|
|
6859
6923
|
##
|
6860
6924
|
|
6861
6925
|
|
6862
|
-
class
|
6926
|
+
class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
|
6927
|
+
@abc.abstractmethod
|
6863
6928
|
def check_call(
|
6864
6929
|
self,
|
6865
6930
|
*cmd: str,
|
6866
6931
|
stdout: ta.Any = sys.stderr,
|
6867
6932
|
**kwargs: ta.Any,
|
6868
6933
|
) -> None:
|
6869
|
-
|
6870
|
-
subprocess.check_call(cmd, **kwargs)
|
6934
|
+
raise NotImplementedError
|
6871
6935
|
|
6936
|
+
@abc.abstractmethod
|
6872
6937
|
def check_output(
|
6873
6938
|
self,
|
6874
6939
|
*cmd: str,
|
6875
6940
|
**kwargs: ta.Any,
|
6876
6941
|
) -> bytes:
|
6877
|
-
|
6878
|
-
|
6942
|
+
raise NotImplementedError
|
6943
|
+
|
6944
|
+
#
|
6879
6945
|
|
6880
6946
|
def check_output_str(
|
6881
6947
|
self,
|
@@ -6917,9 +6983,109 @@ class Subprocesses(AbstractSubprocesses):
|
|
6917
6983
|
return ret.decode().strip()
|
6918
6984
|
|
6919
6985
|
|
6986
|
+
##
|
6987
|
+
|
6988
|
+
|
6989
|
+
class Subprocesses(AbstractSubprocesses):
|
6990
|
+
def check_call(
|
6991
|
+
self,
|
6992
|
+
*cmd: str,
|
6993
|
+
stdout: ta.Any = sys.stderr,
|
6994
|
+
**kwargs: ta.Any,
|
6995
|
+
) -> None:
|
6996
|
+
with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
|
6997
|
+
subprocess.check_call(cmd, **kwargs)
|
6998
|
+
|
6999
|
+
def check_output(
|
7000
|
+
self,
|
7001
|
+
*cmd: str,
|
7002
|
+
**kwargs: ta.Any,
|
7003
|
+
) -> bytes:
|
7004
|
+
with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
|
7005
|
+
return subprocess.check_output(cmd, **kwargs)
|
7006
|
+
|
7007
|
+
|
6920
7008
|
subprocesses = Subprocesses()
|
6921
7009
|
|
6922
7010
|
|
7011
|
+
##
|
7012
|
+
|
7013
|
+
|
7014
|
+
class AbstractAsyncSubprocesses(BaseSubprocesses):
|
7015
|
+
@abc.abstractmethod
|
7016
|
+
async def check_call(
|
7017
|
+
self,
|
7018
|
+
*cmd: str,
|
7019
|
+
stdout: ta.Any = sys.stderr,
|
7020
|
+
**kwargs: ta.Any,
|
7021
|
+
) -> None:
|
7022
|
+
raise NotImplementedError
|
7023
|
+
|
7024
|
+
@abc.abstractmethod
|
7025
|
+
async def check_output(
|
7026
|
+
self,
|
7027
|
+
*cmd: str,
|
7028
|
+
**kwargs: ta.Any,
|
7029
|
+
) -> bytes:
|
7030
|
+
raise NotImplementedError
|
7031
|
+
|
7032
|
+
#
|
7033
|
+
|
7034
|
+
async def check_output_str(
|
7035
|
+
self,
|
7036
|
+
*cmd: str,
|
7037
|
+
**kwargs: ta.Any,
|
7038
|
+
) -> str:
|
7039
|
+
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
7040
|
+
|
7041
|
+
#
|
7042
|
+
|
7043
|
+
async def try_call(
|
7044
|
+
self,
|
7045
|
+
*cmd: str,
|
7046
|
+
**kwargs: ta.Any,
|
7047
|
+
) -> bool:
|
7048
|
+
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
7049
|
+
return False
|
7050
|
+
else:
|
7051
|
+
return True
|
7052
|
+
|
7053
|
+
async def try_output(
|
7054
|
+
self,
|
7055
|
+
*cmd: str,
|
7056
|
+
**kwargs: ta.Any,
|
7057
|
+
) -> ta.Optional[bytes]:
|
7058
|
+
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
7059
|
+
return None
|
7060
|
+
else:
|
7061
|
+
return ret
|
7062
|
+
|
7063
|
+
async def try_output_str(
|
7064
|
+
self,
|
7065
|
+
*cmd: str,
|
7066
|
+
**kwargs: ta.Any,
|
7067
|
+
) -> ta.Optional[str]:
|
7068
|
+
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
7069
|
+
return None
|
7070
|
+
else:
|
7071
|
+
return ret.decode().strip()
|
7072
|
+
|
7073
|
+
|
7074
|
+
########################################
|
7075
|
+
# ../bootstrap.py
|
7076
|
+
|
7077
|
+
|
7078
|
+
@dc.dataclass(frozen=True)
|
7079
|
+
class MainBootstrap:
|
7080
|
+
main_config: MainConfig = MainConfig()
|
7081
|
+
|
7082
|
+
deploy_config: DeployConfig = DeployConfig()
|
7083
|
+
|
7084
|
+
remote_config: RemoteConfig = RemoteConfig()
|
7085
|
+
|
7086
|
+
system_config: SystemConfig = SystemConfig()
|
7087
|
+
|
7088
|
+
|
6923
7089
|
########################################
|
6924
7090
|
# ../commands/local.py
|
6925
7091
|
|
@@ -7337,16 +7503,7 @@ class RemoteCommandExecutor(CommandExecutor):
|
|
7337
7503
|
|
7338
7504
|
|
7339
7505
|
########################################
|
7340
|
-
#
|
7341
|
-
|
7342
|
-
|
7343
|
-
@dc.dataclass(frozen=True)
|
7344
|
-
class SystemConfig:
|
7345
|
-
platform: ta.Optional[Platform] = None
|
7346
|
-
|
7347
|
-
|
7348
|
-
########################################
|
7349
|
-
# ../../../omlish/lite/asyncio/subprocesses.py
|
7506
|
+
# ../../../omlish/asyncs/asyncio/subprocesses.py
|
7350
7507
|
|
7351
7508
|
|
7352
7509
|
##
|
@@ -7357,6 +7514,8 @@ class AsyncioProcessCommunicator:
|
|
7357
7514
|
self,
|
7358
7515
|
proc: asyncio.subprocess.Process,
|
7359
7516
|
loop: ta.Optional[ta.Any] = None,
|
7517
|
+
*,
|
7518
|
+
log: ta.Optional[logging.Logger] = None,
|
7360
7519
|
) -> None:
|
7361
7520
|
super().__init__()
|
7362
7521
|
|
@@ -7365,6 +7524,7 @@ class AsyncioProcessCommunicator:
|
|
7365
7524
|
|
7366
7525
|
self._proc = proc
|
7367
7526
|
self._loop = loop
|
7527
|
+
self._log = log
|
7368
7528
|
|
7369
7529
|
self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check.isinstance(
|
7370
7530
|
proc._transport, # type: ignore # noqa
|
@@ -7380,19 +7540,19 @@ class AsyncioProcessCommunicator:
|
|
7380
7540
|
try:
|
7381
7541
|
if input is not None:
|
7382
7542
|
stdin.write(input)
|
7383
|
-
if self._debug:
|
7384
|
-
|
7543
|
+
if self._debug and self._log is not None:
|
7544
|
+
self._log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
|
7385
7545
|
|
7386
7546
|
await stdin.drain()
|
7387
7547
|
|
7388
7548
|
except (BrokenPipeError, ConnectionResetError) as exc:
|
7389
7549
|
# communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
|
7390
7550
|
# exceptions.
|
7391
|
-
if self._debug:
|
7392
|
-
|
7551
|
+
if self._debug and self._log is not None:
|
7552
|
+
self._log.debug('%r communicate: stdin got %r', self, exc)
|
7393
7553
|
|
7394
|
-
if self._debug:
|
7395
|
-
|
7554
|
+
if self._debug and self._log is not None:
|
7555
|
+
self._log.debug('%r communicate: close stdin', self)
|
7396
7556
|
|
7397
7557
|
stdin.close()
|
7398
7558
|
|
@@ -7408,15 +7568,15 @@ class AsyncioProcessCommunicator:
|
|
7408
7568
|
check.equal(fd, 1)
|
7409
7569
|
stream = check.not_none(self._proc.stdout)
|
7410
7570
|
|
7411
|
-
if self._debug:
|
7571
|
+
if self._debug and self._log is not None:
|
7412
7572
|
name = 'stdout' if fd == 1 else 'stderr'
|
7413
|
-
|
7573
|
+
self._log.debug('%r communicate: read %s', self, name)
|
7414
7574
|
|
7415
7575
|
output = await stream.read()
|
7416
7576
|
|
7417
|
-
if self._debug:
|
7577
|
+
if self._debug and self._log is not None:
|
7418
7578
|
name = 'stdout' if fd == 1 else 'stderr'
|
7419
|
-
|
7579
|
+
self._log.debug('%r communicate: close %s', self, name)
|
7420
7580
|
|
7421
7581
|
transport.close()
|
7422
7582
|
|
@@ -7465,7 +7625,7 @@ class AsyncioProcessCommunicator:
|
|
7465
7625
|
##
|
7466
7626
|
|
7467
7627
|
|
7468
|
-
class AsyncioSubprocesses(
|
7628
|
+
class AsyncioSubprocesses(AbstractAsyncSubprocesses):
|
7469
7629
|
async def communicate(
|
7470
7630
|
self,
|
7471
7631
|
proc: asyncio.subprocess.Process,
|
@@ -7562,45 +7722,6 @@ class AsyncioSubprocesses(AbstractSubprocesses):
|
|
7562
7722
|
with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs): # noqa
|
7563
7723
|
return check.not_none((await self.run(*cmd, **kwargs)).stdout)
|
7564
7724
|
|
7565
|
-
async def check_output_str(
|
7566
|
-
self,
|
7567
|
-
*cmd: str,
|
7568
|
-
**kwargs: ta.Any,
|
7569
|
-
) -> str:
|
7570
|
-
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
7571
|
-
|
7572
|
-
#
|
7573
|
-
|
7574
|
-
async def try_call(
|
7575
|
-
self,
|
7576
|
-
*cmd: str,
|
7577
|
-
**kwargs: ta.Any,
|
7578
|
-
) -> bool:
|
7579
|
-
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
7580
|
-
return False
|
7581
|
-
else:
|
7582
|
-
return True
|
7583
|
-
|
7584
|
-
async def try_output(
|
7585
|
-
self,
|
7586
|
-
*cmd: str,
|
7587
|
-
**kwargs: ta.Any,
|
7588
|
-
) -> ta.Optional[bytes]:
|
7589
|
-
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
7590
|
-
return None
|
7591
|
-
else:
|
7592
|
-
return ret
|
7593
|
-
|
7594
|
-
async def try_output_str(
|
7595
|
-
self,
|
7596
|
-
*cmd: str,
|
7597
|
-
**kwargs: ta.Any,
|
7598
|
-
) -> ta.Optional[str]:
|
7599
|
-
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
7600
|
-
return None
|
7601
|
-
else:
|
7602
|
-
return ret.decode().strip()
|
7603
|
-
|
7604
7725
|
|
7605
7726
|
asyncio_subprocesses = AsyncioSubprocesses()
|
7606
7727
|
|
@@ -7700,21 +7821,6 @@ class InterpInspector:
|
|
7700
7821
|
INTERP_INSPECTOR = InterpInspector()
|
7701
7822
|
|
7702
7823
|
|
7703
|
-
########################################
|
7704
|
-
# ../bootstrap.py
|
7705
|
-
|
7706
|
-
|
7707
|
-
@dc.dataclass(frozen=True)
|
7708
|
-
class MainBootstrap:
|
7709
|
-
main_config: MainConfig = MainConfig()
|
7710
|
-
|
7711
|
-
deploy_config: DeployConfig = DeployConfig()
|
7712
|
-
|
7713
|
-
remote_config: RemoteConfig = RemoteConfig()
|
7714
|
-
|
7715
|
-
system_config: SystemConfig = SystemConfig()
|
7716
|
-
|
7717
|
-
|
7718
7824
|
########################################
|
7719
7825
|
# ../commands/subprocess.py
|
7720
7826
|
|
@@ -7801,39 +7907,22 @@ github.com/wrmsr/omlish@rev
|
|
7801
7907
|
##
|
7802
7908
|
|
7803
7909
|
|
7804
|
-
@dc.dataclass(frozen=True)
|
7805
|
-
class DeployGitRepo:
|
7806
|
-
host: ta.Optional[str] = None
|
7807
|
-
username: ta.Optional[str] = None
|
7808
|
-
path: ta.Optional[str] = None
|
7809
|
-
|
7810
|
-
def __post_init__(self) -> None:
|
7811
|
-
check.not_in('..', check.non_empty_str(self.host))
|
7812
|
-
check.not_in('.', check.non_empty_str(self.path))
|
7813
|
-
|
7814
|
-
|
7815
|
-
@dc.dataclass(frozen=True)
|
7816
|
-
class DeployGitSpec:
|
7817
|
-
repo: DeployGitRepo
|
7818
|
-
rev: DeployRev
|
7819
|
-
|
7820
|
-
|
7821
|
-
##
|
7822
|
-
|
7823
|
-
|
7824
7910
|
class DeployGitManager(DeployPathOwner):
|
7825
7911
|
def __init__(
|
7826
7912
|
self,
|
7827
7913
|
*,
|
7828
|
-
deploy_home: DeployHome,
|
7914
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
7829
7915
|
) -> None:
|
7830
7916
|
super().__init__()
|
7831
7917
|
|
7832
7918
|
self._deploy_home = deploy_home
|
7833
|
-
self._dir = os.path.join(deploy_home, 'git')
|
7834
7919
|
|
7835
7920
|
self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
|
7836
7921
|
|
7922
|
+
@cached_nullary
|
7923
|
+
def _dir(self) -> str:
|
7924
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'git')
|
7925
|
+
|
7837
7926
|
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
7838
7927
|
return {
|
7839
7928
|
DeployPath.parse('git'),
|
@@ -7850,7 +7939,7 @@ class DeployGitManager(DeployPathOwner):
|
|
7850
7939
|
self._git = git
|
7851
7940
|
self._repo = repo
|
7852
7941
|
self._dir = os.path.join(
|
7853
|
-
self._git._dir, # noqa
|
7942
|
+
self._git._dir(), # noqa
|
7854
7943
|
check.non_empty_str(repo.host),
|
7855
7944
|
check.non_empty_str(repo.path),
|
7856
7945
|
)
|
@@ -7907,8 +7996,8 @@ class DeployGitManager(DeployPathOwner):
|
|
7907
7996
|
repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
|
7908
7997
|
return repo_dir
|
7909
7998
|
|
7910
|
-
async def checkout(self,
|
7911
|
-
await self.get_repo_dir(
|
7999
|
+
async def checkout(self, repo: DeployGitRepo, rev: DeployRev, dst_dir: str) -> None:
|
8000
|
+
await self.get_repo_dir(repo).checkout(rev, dst_dir)
|
7912
8001
|
|
7913
8002
|
|
7914
8003
|
########################################
|
@@ -7924,12 +8013,15 @@ class DeployVenvManager(DeployPathOwner):
|
|
7924
8013
|
def __init__(
|
7925
8014
|
self,
|
7926
8015
|
*,
|
7927
|
-
deploy_home: DeployHome,
|
8016
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
7928
8017
|
) -> None:
|
7929
8018
|
super().__init__()
|
7930
8019
|
|
7931
8020
|
self._deploy_home = deploy_home
|
7932
|
-
|
8021
|
+
|
8022
|
+
@cached_nullary
|
8023
|
+
def _dir(self) -> str:
|
8024
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
|
7933
8025
|
|
7934
8026
|
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
7935
8027
|
return {
|
@@ -7966,66 +8058,208 @@ class DeployVenvManager(DeployPathOwner):
|
|
7966
8058
|
|
7967
8059
|
async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
|
7968
8060
|
await self.setup_venv(
|
7969
|
-
os.path.join(self._deploy_home, 'apps', app_tag.app, app_tag.tag),
|
7970
|
-
os.path.join(self.
|
8061
|
+
os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
|
8062
|
+
os.path.join(self._dir(), app_tag.app, app_tag.tag),
|
7971
8063
|
)
|
7972
8064
|
|
7973
8065
|
|
7974
8066
|
########################################
|
7975
|
-
# ../remote/
|
8067
|
+
# ../remote/_main.py
|
7976
8068
|
|
7977
8069
|
|
7978
8070
|
##
|
7979
8071
|
|
7980
8072
|
|
7981
|
-
class
|
7982
|
-
|
7983
|
-
|
7984
|
-
|
7985
|
-
shell_quote: bool = False
|
8073
|
+
class _RemoteExecutionLogHandler(logging.Handler):
|
8074
|
+
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
8075
|
+
super().__init__()
|
8076
|
+
self._fn = fn
|
7986
8077
|
|
7987
|
-
|
7988
|
-
|
8078
|
+
def emit(self, record):
|
8079
|
+
msg = self.format(record)
|
8080
|
+
self._fn(msg)
|
7989
8081
|
|
7990
|
-
stderr: ta.Optional[str] = None # SubprocessChannelOption
|
7991
8082
|
|
7992
|
-
|
7993
|
-
class Spawned:
|
7994
|
-
stdin: asyncio.StreamWriter
|
7995
|
-
stdout: asyncio.StreamReader
|
7996
|
-
stderr: ta.Optional[asyncio.StreamReader]
|
8083
|
+
##
|
7997
8084
|
|
7998
|
-
|
7999
|
-
|
8085
|
+
|
8086
|
+
class _RemoteExecutionMain:
|
8087
|
+
def __init__(
|
8000
8088
|
self,
|
8001
|
-
|
8002
|
-
|
8003
|
-
|
8004
|
-
timeout: ta.Optional[float] = None,
|
8005
|
-
debug: bool = False,
|
8006
|
-
) -> ta.AsyncContextManager[Spawned]:
|
8007
|
-
raise NotImplementedError
|
8089
|
+
chan: RemoteChannel,
|
8090
|
+
) -> None:
|
8091
|
+
super().__init__()
|
8008
8092
|
|
8093
|
+
self._chan = chan
|
8009
8094
|
|
8010
|
-
|
8095
|
+
self.__bootstrap: ta.Optional[MainBootstrap] = None
|
8096
|
+
self.__injector: ta.Optional[Injector] = None
|
8011
8097
|
|
8098
|
+
@property
|
8099
|
+
def _bootstrap(self) -> MainBootstrap:
|
8100
|
+
return check.not_none(self.__bootstrap)
|
8012
8101
|
|
8013
|
-
|
8014
|
-
|
8015
|
-
|
8016
|
-
shell: bool
|
8102
|
+
@property
|
8103
|
+
def _injector(self) -> Injector:
|
8104
|
+
return check.not_none(self.__injector)
|
8017
8105
|
|
8018
|
-
|
8019
|
-
|
8020
|
-
|
8021
|
-
|
8022
|
-
|
8023
|
-
|
8024
|
-
|
8025
|
-
|
8026
|
-
|
8027
|
-
|
8028
|
-
|
8106
|
+
#
|
8107
|
+
|
8108
|
+
def _timebomb_main(
|
8109
|
+
self,
|
8110
|
+
delay_s: float,
|
8111
|
+
*,
|
8112
|
+
sig: int = signal.SIGINT,
|
8113
|
+
code: int = 1,
|
8114
|
+
) -> None:
|
8115
|
+
time.sleep(delay_s)
|
8116
|
+
|
8117
|
+
if (pgid := os.getpgid(0)) == os.getpid():
|
8118
|
+
os.killpg(pgid, sig)
|
8119
|
+
|
8120
|
+
os._exit(code) # noqa
|
8121
|
+
|
8122
|
+
@cached_nullary
|
8123
|
+
def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
|
8124
|
+
if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
|
8125
|
+
return None
|
8126
|
+
|
8127
|
+
thr = threading.Thread(
|
8128
|
+
target=functools.partial(self._timebomb_main, tbd),
|
8129
|
+
name=f'{self.__class__.__name__}.timebomb',
|
8130
|
+
daemon=True,
|
8131
|
+
)
|
8132
|
+
|
8133
|
+
thr.start()
|
8134
|
+
|
8135
|
+
log.debug('Started timebomb thread: %r', thr)
|
8136
|
+
|
8137
|
+
return thr
|
8138
|
+
|
8139
|
+
#
|
8140
|
+
|
8141
|
+
@cached_nullary
|
8142
|
+
def _log_handler(self) -> _RemoteLogHandler:
|
8143
|
+
return _RemoteLogHandler(self._chan)
|
8144
|
+
|
8145
|
+
#
|
8146
|
+
|
8147
|
+
async def _setup(self) -> None:
|
8148
|
+
check.none(self.__bootstrap)
|
8149
|
+
check.none(self.__injector)
|
8150
|
+
|
8151
|
+
# Bootstrap
|
8152
|
+
|
8153
|
+
self.__bootstrap = check.not_none(await self._chan.recv_obj(MainBootstrap))
|
8154
|
+
|
8155
|
+
if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
|
8156
|
+
pycharm_debug_connect(prd)
|
8157
|
+
|
8158
|
+
self.__injector = main_bootstrap(self._bootstrap)
|
8159
|
+
|
8160
|
+
self._chan.set_marshaler(self._injector[ObjMarshalerManager])
|
8161
|
+
|
8162
|
+
# Post-bootstrap
|
8163
|
+
|
8164
|
+
if self._bootstrap.remote_config.set_pgid:
|
8165
|
+
if os.getpgid(0) != os.getpid():
|
8166
|
+
log.debug('Setting pgid')
|
8167
|
+
os.setpgid(0, 0)
|
8168
|
+
|
8169
|
+
if (ds := self._bootstrap.remote_config.deathsig) is not None:
|
8170
|
+
log.debug('Setting deathsig: %s', ds)
|
8171
|
+
set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
|
8172
|
+
|
8173
|
+
self._timebomb_thread()
|
8174
|
+
|
8175
|
+
if self._bootstrap.remote_config.forward_logging:
|
8176
|
+
log.debug('Installing log forwarder')
|
8177
|
+
logging.root.addHandler(self._log_handler())
|
8178
|
+
|
8179
|
+
#
|
8180
|
+
|
8181
|
+
async def run(self) -> None:
|
8182
|
+
await self._setup()
|
8183
|
+
|
8184
|
+
executor = self._injector[LocalCommandExecutor]
|
8185
|
+
|
8186
|
+
handler = _RemoteCommandHandler(self._chan, executor)
|
8187
|
+
|
8188
|
+
await handler.run()
|
8189
|
+
|
8190
|
+
|
8191
|
+
def _remote_execution_main() -> None:
|
8192
|
+
rt = pyremote_bootstrap_finalize() # noqa
|
8193
|
+
|
8194
|
+
async def inner() -> None:
|
8195
|
+
input = await asyncio_open_stream_reader(rt.input) # noqa
|
8196
|
+
output = await asyncio_open_stream_writer(rt.output)
|
8197
|
+
|
8198
|
+
chan = RemoteChannelImpl(
|
8199
|
+
input,
|
8200
|
+
output,
|
8201
|
+
)
|
8202
|
+
|
8203
|
+
await _RemoteExecutionMain(chan).run()
|
8204
|
+
|
8205
|
+
asyncio.run(inner())
|
8206
|
+
|
8207
|
+
|
8208
|
+
########################################
|
8209
|
+
# ../remote/spawning.py
|
8210
|
+
|
8211
|
+
|
8212
|
+
##
|
8213
|
+
|
8214
|
+
|
8215
|
+
class RemoteSpawning(abc.ABC):
|
8216
|
+
@dc.dataclass(frozen=True)
|
8217
|
+
class Target:
|
8218
|
+
shell: ta.Optional[str] = None
|
8219
|
+
shell_quote: bool = False
|
8220
|
+
|
8221
|
+
DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
|
8222
|
+
python: str = DEFAULT_PYTHON
|
8223
|
+
|
8224
|
+
stderr: ta.Optional[str] = None # SubprocessChannelOption
|
8225
|
+
|
8226
|
+
@dc.dataclass(frozen=True)
|
8227
|
+
class Spawned:
|
8228
|
+
stdin: asyncio.StreamWriter
|
8229
|
+
stdout: asyncio.StreamReader
|
8230
|
+
stderr: ta.Optional[asyncio.StreamReader]
|
8231
|
+
|
8232
|
+
@abc.abstractmethod
|
8233
|
+
def spawn(
|
8234
|
+
self,
|
8235
|
+
tgt: Target,
|
8236
|
+
src: str,
|
8237
|
+
*,
|
8238
|
+
timeout: ta.Optional[float] = None,
|
8239
|
+
debug: bool = False,
|
8240
|
+
) -> ta.AsyncContextManager[Spawned]:
|
8241
|
+
raise NotImplementedError
|
8242
|
+
|
8243
|
+
|
8244
|
+
##
|
8245
|
+
|
8246
|
+
|
8247
|
+
class SubprocessRemoteSpawning(RemoteSpawning):
|
8248
|
+
class _PreparedCmd(ta.NamedTuple): # noqa
|
8249
|
+
cmd: ta.Sequence[str]
|
8250
|
+
shell: bool
|
8251
|
+
|
8252
|
+
def _prepare_cmd(
|
8253
|
+
self,
|
8254
|
+
tgt: RemoteSpawning.Target,
|
8255
|
+
src: str,
|
8256
|
+
) -> _PreparedCmd:
|
8257
|
+
if tgt.shell is not None:
|
8258
|
+
sh_src = f'{tgt.python} -c {shlex.quote(src)}'
|
8259
|
+
if tgt.shell_quote:
|
8260
|
+
sh_src = shlex.quote(sh_src)
|
8261
|
+
sh_cmd = f'{tgt.shell} {sh_src}'
|
8262
|
+
return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
|
8029
8263
|
|
8030
8264
|
else:
|
8031
8265
|
return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
|
@@ -8366,14 +8600,14 @@ def make_deploy_tag(
|
|
8366
8600
|
now = datetime.datetime.utcnow() # noqa
|
8367
8601
|
now_fmt = '%Y%m%dT%H%M%S'
|
8368
8602
|
now_str = now.strftime(now_fmt)
|
8369
|
-
return DeployTag('-'.join([
|
8603
|
+
return DeployTag('-'.join([now_str, rev]))
|
8370
8604
|
|
8371
8605
|
|
8372
8606
|
class DeployAppManager(DeployPathOwner):
|
8373
8607
|
def __init__(
|
8374
8608
|
self,
|
8375
8609
|
*,
|
8376
|
-
deploy_home: DeployHome,
|
8610
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
8377
8611
|
git: DeployGitManager,
|
8378
8612
|
venvs: DeployVenvManager,
|
8379
8613
|
) -> None:
|
@@ -8383,7 +8617,9 @@ class DeployAppManager(DeployPathOwner):
|
|
8383
8617
|
self._git = git
|
8384
8618
|
self._venvs = venvs
|
8385
8619
|
|
8386
|
-
|
8620
|
+
@cached_nullary
|
8621
|
+
def _dir(self) -> str:
|
8622
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
|
8387
8623
|
|
8388
8624
|
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
8389
8625
|
return {
|
@@ -8392,20 +8628,16 @@ class DeployAppManager(DeployPathOwner):
|
|
8392
8628
|
|
8393
8629
|
async def prepare_app(
|
8394
8630
|
self,
|
8395
|
-
|
8396
|
-
rev: DeployRev,
|
8397
|
-
repo: DeployGitRepo,
|
8631
|
+
spec: DeploySpec,
|
8398
8632
|
):
|
8399
|
-
app_tag = DeployAppTag(app, make_deploy_tag(rev))
|
8400
|
-
app_dir = os.path.join(self._dir, app, app_tag.tag)
|
8633
|
+
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev))
|
8634
|
+
app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
|
8401
8635
|
|
8402
8636
|
#
|
8403
8637
|
|
8404
8638
|
await self._git.checkout(
|
8405
|
-
|
8406
|
-
|
8407
|
-
rev=rev,
|
8408
|
-
),
|
8639
|
+
spec.repo,
|
8640
|
+
spec.rev,
|
8409
8641
|
app_dir,
|
8410
8642
|
)
|
8411
8643
|
|
@@ -8415,145 +8647,122 @@ class DeployAppManager(DeployPathOwner):
|
|
8415
8647
|
|
8416
8648
|
|
8417
8649
|
########################################
|
8418
|
-
# ../remote/
|
8419
|
-
|
8420
|
-
|
8421
|
-
##
|
8422
|
-
|
8423
|
-
|
8424
|
-
class _RemoteExecutionLogHandler(logging.Handler):
|
8425
|
-
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
8426
|
-
super().__init__()
|
8427
|
-
self._fn = fn
|
8428
|
-
|
8429
|
-
def emit(self, record):
|
8430
|
-
msg = self.format(record)
|
8431
|
-
self._fn(msg)
|
8650
|
+
# ../remote/connection.py
|
8432
8651
|
|
8433
8652
|
|
8434
8653
|
##
|
8435
8654
|
|
8436
8655
|
|
8437
|
-
class
|
8656
|
+
class PyremoteRemoteExecutionConnector:
|
8438
8657
|
def __init__(
|
8439
8658
|
self,
|
8440
|
-
|
8659
|
+
*,
|
8660
|
+
spawning: RemoteSpawning,
|
8661
|
+
msh: ObjMarshalerManager,
|
8662
|
+
payload_file: ta.Optional[RemoteExecutionPayloadFile] = None,
|
8441
8663
|
) -> None:
|
8442
8664
|
super().__init__()
|
8443
8665
|
|
8444
|
-
self.
|
8666
|
+
self._spawning = spawning
|
8667
|
+
self._msh = msh
|
8668
|
+
self._payload_file = payload_file
|
8445
8669
|
|
8446
|
-
|
8447
|
-
self.__injector: ta.Optional[Injector] = None
|
8670
|
+
#
|
8448
8671
|
|
8449
|
-
@
|
8450
|
-
def
|
8451
|
-
return
|
8672
|
+
@cached_nullary
|
8673
|
+
def _payload_src(self) -> str:
|
8674
|
+
return get_remote_payload_src(file=self._payload_file)
|
8452
8675
|
|
8453
|
-
@
|
8454
|
-
def
|
8455
|
-
return
|
8676
|
+
@cached_nullary
|
8677
|
+
def _remote_src(self) -> ta.Sequence[str]:
|
8678
|
+
return [
|
8679
|
+
self._payload_src(),
|
8680
|
+
'_remote_execution_main()',
|
8681
|
+
]
|
8682
|
+
|
8683
|
+
@cached_nullary
|
8684
|
+
def _spawn_src(self) -> str:
|
8685
|
+
return pyremote_build_bootstrap_cmd(__package__ or 'manage')
|
8456
8686
|
|
8457
8687
|
#
|
8458
8688
|
|
8459
|
-
|
8689
|
+
@contextlib.asynccontextmanager
|
8690
|
+
async def connect(
|
8460
8691
|
self,
|
8461
|
-
|
8462
|
-
|
8463
|
-
|
8464
|
-
|
8465
|
-
|
8466
|
-
time.sleep(delay_s)
|
8692
|
+
tgt: RemoteSpawning.Target,
|
8693
|
+
bs: MainBootstrap,
|
8694
|
+
) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
|
8695
|
+
spawn_src = self._spawn_src()
|
8696
|
+
remote_src = self._remote_src()
|
8467
8697
|
|
8468
|
-
|
8469
|
-
|
8698
|
+
async with self._spawning.spawn(
|
8699
|
+
tgt,
|
8700
|
+
spawn_src,
|
8701
|
+
debug=bs.main_config.debug,
|
8702
|
+
) as proc:
|
8703
|
+
res = await PyremoteBootstrapDriver( # noqa
|
8704
|
+
remote_src,
|
8705
|
+
PyremoteBootstrapOptions(
|
8706
|
+
debug=bs.main_config.debug,
|
8707
|
+
),
|
8708
|
+
).async_run(
|
8709
|
+
proc.stdout,
|
8710
|
+
proc.stdin,
|
8711
|
+
)
|
8470
8712
|
|
8471
|
-
|
8713
|
+
chan = RemoteChannelImpl(
|
8714
|
+
proc.stdout,
|
8715
|
+
proc.stdin,
|
8716
|
+
msh=self._msh,
|
8717
|
+
)
|
8472
8718
|
|
8473
|
-
|
8474
|
-
def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
|
8475
|
-
if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
|
8476
|
-
return None
|
8719
|
+
await chan.send_obj(bs)
|
8477
8720
|
|
8478
|
-
|
8479
|
-
|
8480
|
-
|
8481
|
-
daemon=True,
|
8482
|
-
)
|
8721
|
+
rce: RemoteCommandExecutor
|
8722
|
+
async with aclosing(RemoteCommandExecutor(chan)) as rce:
|
8723
|
+
await rce.start()
|
8483
8724
|
|
8484
|
-
|
8485
|
-
|
8486
|
-
log.debug('Started timebomb thread: %r', thr)
|
8487
|
-
|
8488
|
-
return thr
|
8489
|
-
|
8490
|
-
#
|
8491
|
-
|
8492
|
-
@cached_nullary
|
8493
|
-
def _log_handler(self) -> _RemoteLogHandler:
|
8494
|
-
return _RemoteLogHandler(self._chan)
|
8495
|
-
|
8496
|
-
#
|
8497
|
-
|
8498
|
-
async def _setup(self) -> None:
|
8499
|
-
check.none(self.__bootstrap)
|
8500
|
-
check.none(self.__injector)
|
8501
|
-
|
8502
|
-
# Bootstrap
|
8503
|
-
|
8504
|
-
self.__bootstrap = check.not_none(await self._chan.recv_obj(MainBootstrap))
|
8505
|
-
|
8506
|
-
if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
|
8507
|
-
pycharm_debug_connect(prd)
|
8508
|
-
|
8509
|
-
self.__injector = main_bootstrap(self._bootstrap)
|
8510
|
-
|
8511
|
-
self._chan.set_marshaler(self._injector[ObjMarshalerManager])
|
8512
|
-
|
8513
|
-
# Post-bootstrap
|
8514
|
-
|
8515
|
-
if self._bootstrap.remote_config.set_pgid:
|
8516
|
-
if os.getpgid(0) != os.getpid():
|
8517
|
-
log.debug('Setting pgid')
|
8518
|
-
os.setpgid(0, 0)
|
8519
|
-
|
8520
|
-
if (ds := self._bootstrap.remote_config.deathsig) is not None:
|
8521
|
-
log.debug('Setting deathsig: %s', ds)
|
8522
|
-
set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
|
8523
|
-
|
8524
|
-
self._timebomb_thread()
|
8525
|
-
|
8526
|
-
if self._bootstrap.remote_config.forward_logging:
|
8527
|
-
log.debug('Installing log forwarder')
|
8528
|
-
logging.root.addHandler(self._log_handler())
|
8725
|
+
yield rce
|
8529
8726
|
|
8530
|
-
#
|
8531
8727
|
|
8532
|
-
|
8533
|
-
await self._setup()
|
8534
|
-
|
8535
|
-
executor = self._injector[LocalCommandExecutor]
|
8728
|
+
##
|
8536
8729
|
|
8537
|
-
handler = _RemoteCommandHandler(self._chan, executor)
|
8538
8730
|
|
8539
|
-
|
8731
|
+
class InProcessRemoteExecutionConnector:
|
8732
|
+
def __init__(
|
8733
|
+
self,
|
8734
|
+
*,
|
8735
|
+
msh: ObjMarshalerManager,
|
8736
|
+
local_executor: LocalCommandExecutor,
|
8737
|
+
) -> None:
|
8738
|
+
super().__init__()
|
8540
8739
|
|
8740
|
+
self._msh = msh
|
8741
|
+
self._local_executor = local_executor
|
8541
8742
|
|
8542
|
-
|
8543
|
-
|
8743
|
+
@contextlib.asynccontextmanager
|
8744
|
+
async def connect(self) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
|
8745
|
+
r0, w0 = asyncio_create_bytes_channel()
|
8746
|
+
r1, w1 = asyncio_create_bytes_channel()
|
8544
8747
|
|
8545
|
-
|
8546
|
-
|
8547
|
-
output = await asyncio_open_stream_writer(rt.output)
|
8748
|
+
remote_chan = RemoteChannelImpl(r0, w1, msh=self._msh)
|
8749
|
+
local_chan = RemoteChannelImpl(r1, w0, msh=self._msh)
|
8548
8750
|
|
8549
|
-
|
8550
|
-
|
8551
|
-
|
8751
|
+
rch = _RemoteCommandHandler(
|
8752
|
+
remote_chan,
|
8753
|
+
self._local_executor,
|
8552
8754
|
)
|
8755
|
+
rch_task = asyncio.create_task(rch.run()) # noqa
|
8756
|
+
try:
|
8757
|
+
rce: RemoteCommandExecutor
|
8758
|
+
async with aclosing(RemoteCommandExecutor(local_chan)) as rce:
|
8759
|
+
await rce.start()
|
8553
8760
|
|
8554
|
-
|
8761
|
+
yield rce
|
8555
8762
|
|
8556
|
-
|
8763
|
+
finally:
|
8764
|
+
rch.stop()
|
8765
|
+
await rch_task
|
8557
8766
|
|
8558
8767
|
|
8559
8768
|
########################################
|
@@ -8899,521 +9108,256 @@ class PyenvVersionInstaller:
|
|
8899
9108
|
self._pyenv.exe(),
|
8900
9109
|
'install',
|
8901
9110
|
*conf_args,
|
8902
|
-
]
|
8903
|
-
|
8904
|
-
await asyncio_subprocesses.check_call(
|
8905
|
-
*full_args,
|
8906
|
-
env=env,
|
8907
|
-
)
|
8908
|
-
|
8909
|
-
exe = os.path.join(await self.install_dir(), 'bin', 'python')
|
8910
|
-
if not os.path.isfile(exe):
|
8911
|
-
raise RuntimeError(f'Interpreter not found: {exe}')
|
8912
|
-
return exe
|
8913
|
-
|
8914
|
-
|
8915
|
-
##
|
8916
|
-
|
8917
|
-
|
8918
|
-
class PyenvInterpProvider(InterpProvider):
|
8919
|
-
def __init__(
|
8920
|
-
self,
|
8921
|
-
pyenv: Pyenv = Pyenv(),
|
8922
|
-
|
8923
|
-
inspect: bool = False,
|
8924
|
-
inspector: InterpInspector = INTERP_INSPECTOR,
|
8925
|
-
|
8926
|
-
*,
|
8927
|
-
|
8928
|
-
try_update: bool = False,
|
8929
|
-
) -> None:
|
8930
|
-
super().__init__()
|
8931
|
-
|
8932
|
-
self._pyenv = pyenv
|
8933
|
-
|
8934
|
-
self._inspect = inspect
|
8935
|
-
self._inspector = inspector
|
8936
|
-
|
8937
|
-
self._try_update = try_update
|
8938
|
-
|
8939
|
-
#
|
8940
|
-
|
8941
|
-
@staticmethod
|
8942
|
-
def guess_version(s: str) -> ta.Optional[InterpVersion]:
|
8943
|
-
def strip_sfx(s: str, sfx: str) -> ta.Tuple[str, bool]:
|
8944
|
-
if s.endswith(sfx):
|
8945
|
-
return s[:-len(sfx)], True
|
8946
|
-
return s, False
|
8947
|
-
ok = {}
|
8948
|
-
s, ok['debug'] = strip_sfx(s, '-debug')
|
8949
|
-
s, ok['threaded'] = strip_sfx(s, 't')
|
8950
|
-
try:
|
8951
|
-
v = Version(s)
|
8952
|
-
except InvalidVersion:
|
8953
|
-
return None
|
8954
|
-
return InterpVersion(v, InterpOpts(**ok))
|
8955
|
-
|
8956
|
-
class Installed(ta.NamedTuple):
|
8957
|
-
name: str
|
8958
|
-
exe: str
|
8959
|
-
version: InterpVersion
|
8960
|
-
|
8961
|
-
async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
8962
|
-
iv: ta.Optional[InterpVersion]
|
8963
|
-
if self._inspect:
|
8964
|
-
try:
|
8965
|
-
iv = check.not_none(await self._inspector.inspect(ep)).iv
|
8966
|
-
except Exception as e: # noqa
|
8967
|
-
return None
|
8968
|
-
else:
|
8969
|
-
iv = self.guess_version(vn)
|
8970
|
-
if iv is None:
|
8971
|
-
return None
|
8972
|
-
return PyenvInterpProvider.Installed(
|
8973
|
-
name=vn,
|
8974
|
-
exe=ep,
|
8975
|
-
version=iv,
|
8976
|
-
)
|
8977
|
-
|
8978
|
-
async def installed(self) -> ta.Sequence[Installed]:
|
8979
|
-
ret: ta.List[PyenvInterpProvider.Installed] = []
|
8980
|
-
for vn, ep in await self._pyenv.version_exes():
|
8981
|
-
if (i := await self._make_installed(vn, ep)) is None:
|
8982
|
-
log.debug('Invalid pyenv version: %s', vn)
|
8983
|
-
continue
|
8984
|
-
ret.append(i)
|
8985
|
-
return ret
|
8986
|
-
|
8987
|
-
#
|
8988
|
-
|
8989
|
-
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
8990
|
-
return [i.version for i in await self.installed()]
|
8991
|
-
|
8992
|
-
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
8993
|
-
for i in await self.installed():
|
8994
|
-
if i.version == version:
|
8995
|
-
return Interp(
|
8996
|
-
exe=i.exe,
|
8997
|
-
version=i.version,
|
8998
|
-
)
|
8999
|
-
raise KeyError(version)
|
9000
|
-
|
9001
|
-
#
|
9002
|
-
|
9003
|
-
async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9004
|
-
lst = []
|
9005
|
-
|
9006
|
-
for vs in await self._pyenv.installable_versions():
|
9007
|
-
if (iv := self.guess_version(vs)) is None:
|
9008
|
-
continue
|
9009
|
-
if iv.opts.debug:
|
9010
|
-
raise Exception('Pyenv installable versions not expected to have debug suffix')
|
9011
|
-
for d in [False, True]:
|
9012
|
-
lst.append(dc.replace(iv, opts=dc.replace(iv.opts, debug=d)))
|
9013
|
-
|
9014
|
-
return lst
|
9015
|
-
|
9016
|
-
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9017
|
-
lst = await self._get_installable_versions(spec)
|
9018
|
-
|
9019
|
-
if self._try_update and not any(v in spec for v in lst):
|
9020
|
-
if self._pyenv.update():
|
9021
|
-
lst = await self._get_installable_versions(spec)
|
9022
|
-
|
9023
|
-
return lst
|
9024
|
-
|
9025
|
-
async def install_version(self, version: InterpVersion) -> Interp:
|
9026
|
-
inst_version = str(version.version)
|
9027
|
-
inst_opts = version.opts
|
9028
|
-
if inst_opts.threaded:
|
9029
|
-
inst_version += 't'
|
9030
|
-
inst_opts = dc.replace(inst_opts, threaded=False)
|
9031
|
-
|
9032
|
-
installer = PyenvVersionInstaller(
|
9033
|
-
inst_version,
|
9034
|
-
interp_opts=inst_opts,
|
9035
|
-
)
|
9036
|
-
|
9037
|
-
exe = await installer.install()
|
9038
|
-
return Interp(exe, version)
|
9039
|
-
|
9040
|
-
|
9041
|
-
########################################
|
9042
|
-
# ../../../omdev/interp/system.py
|
9043
|
-
"""
|
9044
|
-
TODO:
|
9045
|
-
- python, python3, python3.12, ...
|
9046
|
-
- check if path py's are venvs: sys.prefix != sys.base_prefix
|
9047
|
-
"""
|
9048
|
-
|
9049
|
-
|
9050
|
-
##
|
9051
|
-
|
9052
|
-
|
9053
|
-
@dc.dataclass(frozen=True)
|
9054
|
-
class SystemInterpProvider(InterpProvider):
|
9055
|
-
cmd: str = 'python3'
|
9056
|
-
path: ta.Optional[str] = None
|
9057
|
-
|
9058
|
-
inspect: bool = False
|
9059
|
-
inspector: InterpInspector = INTERP_INSPECTOR
|
9060
|
-
|
9061
|
-
#
|
9062
|
-
|
9063
|
-
@staticmethod
|
9064
|
-
def _re_which(
|
9065
|
-
pat: re.Pattern,
|
9066
|
-
*,
|
9067
|
-
mode: int = os.F_OK | os.X_OK,
|
9068
|
-
path: ta.Optional[str] = None,
|
9069
|
-
) -> ta.List[str]:
|
9070
|
-
if path is None:
|
9071
|
-
path = os.environ.get('PATH', None)
|
9072
|
-
if path is None:
|
9073
|
-
try:
|
9074
|
-
path = os.confstr('CS_PATH')
|
9075
|
-
except (AttributeError, ValueError):
|
9076
|
-
path = os.defpath
|
9077
|
-
|
9078
|
-
if not path:
|
9079
|
-
return []
|
9080
|
-
|
9081
|
-
path = os.fsdecode(path)
|
9082
|
-
pathlst = path.split(os.pathsep)
|
9083
|
-
|
9084
|
-
def _access_check(fn: str, mode: int) -> bool:
|
9085
|
-
return os.path.exists(fn) and os.access(fn, mode)
|
9086
|
-
|
9087
|
-
out = []
|
9088
|
-
seen = set()
|
9089
|
-
for d in pathlst:
|
9090
|
-
normdir = os.path.normcase(d)
|
9091
|
-
if normdir not in seen:
|
9092
|
-
seen.add(normdir)
|
9093
|
-
if not _access_check(normdir, mode):
|
9094
|
-
continue
|
9095
|
-
for thefile in os.listdir(d):
|
9096
|
-
name = os.path.join(d, thefile)
|
9097
|
-
if not (
|
9098
|
-
os.path.isfile(name) and
|
9099
|
-
pat.fullmatch(thefile) and
|
9100
|
-
_access_check(name, mode)
|
9101
|
-
):
|
9102
|
-
continue
|
9103
|
-
out.append(name)
|
9104
|
-
|
9105
|
-
return out
|
9106
|
-
|
9107
|
-
@cached_nullary
|
9108
|
-
def exes(self) -> ta.List[str]:
|
9109
|
-
return self._re_which(
|
9110
|
-
re.compile(r'python3(\.\d+)?'),
|
9111
|
-
path=self.path,
|
9112
|
-
)
|
9113
|
-
|
9114
|
-
#
|
9115
|
-
|
9116
|
-
async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
9117
|
-
if not self.inspect:
|
9118
|
-
s = os.path.basename(exe)
|
9119
|
-
if s.startswith('python'):
|
9120
|
-
s = s[len('python'):]
|
9121
|
-
if '.' in s:
|
9122
|
-
try:
|
9123
|
-
return InterpVersion.parse(s)
|
9124
|
-
except InvalidVersion:
|
9125
|
-
pass
|
9126
|
-
ii = await self.inspector.inspect(exe)
|
9127
|
-
return ii.iv if ii is not None else None
|
9128
|
-
|
9129
|
-
async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
9130
|
-
lst = []
|
9131
|
-
for e in self.exes():
|
9132
|
-
if (ev := await self.get_exe_version(e)) is None:
|
9133
|
-
log.debug('Invalid system version: %s', e)
|
9134
|
-
continue
|
9135
|
-
lst.append((e, ev))
|
9136
|
-
return lst
|
9137
|
-
|
9138
|
-
#
|
9139
|
-
|
9140
|
-
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9141
|
-
return [ev for e, ev in await self.exe_versions()]
|
9142
|
-
|
9143
|
-
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
9144
|
-
for e, ev in await self.exe_versions():
|
9145
|
-
if ev != version:
|
9146
|
-
continue
|
9147
|
-
return Interp(
|
9148
|
-
exe=e,
|
9149
|
-
version=ev,
|
9150
|
-
)
|
9151
|
-
raise KeyError(version)
|
9152
|
-
|
9153
|
-
|
9154
|
-
########################################
|
9155
|
-
# ../remote/connection.py
|
9156
|
-
|
9157
|
-
|
9158
|
-
##
|
9159
|
-
|
9160
|
-
|
9161
|
-
class PyremoteRemoteExecutionConnector:
|
9162
|
-
def __init__(
|
9163
|
-
self,
|
9164
|
-
*,
|
9165
|
-
spawning: RemoteSpawning,
|
9166
|
-
msh: ObjMarshalerManager,
|
9167
|
-
payload_file: ta.Optional[RemoteExecutionPayloadFile] = None,
|
9168
|
-
) -> None:
|
9169
|
-
super().__init__()
|
9170
|
-
|
9171
|
-
self._spawning = spawning
|
9172
|
-
self._msh = msh
|
9173
|
-
self._payload_file = payload_file
|
9174
|
-
|
9175
|
-
#
|
9176
|
-
|
9177
|
-
@cached_nullary
|
9178
|
-
def _payload_src(self) -> str:
|
9179
|
-
return get_remote_payload_src(file=self._payload_file)
|
9180
|
-
|
9181
|
-
@cached_nullary
|
9182
|
-
def _remote_src(self) -> ta.Sequence[str]:
|
9183
|
-
return [
|
9184
|
-
self._payload_src(),
|
9185
|
-
'_remote_execution_main()',
|
9186
|
-
]
|
9187
|
-
|
9188
|
-
@cached_nullary
|
9189
|
-
def _spawn_src(self) -> str:
|
9190
|
-
return pyremote_build_bootstrap_cmd(__package__ or 'manage')
|
9191
|
-
|
9192
|
-
#
|
9193
|
-
|
9194
|
-
@contextlib.asynccontextmanager
|
9195
|
-
async def connect(
|
9196
|
-
self,
|
9197
|
-
tgt: RemoteSpawning.Target,
|
9198
|
-
bs: MainBootstrap,
|
9199
|
-
) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
|
9200
|
-
spawn_src = self._spawn_src()
|
9201
|
-
remote_src = self._remote_src()
|
9202
|
-
|
9203
|
-
async with self._spawning.spawn(
|
9204
|
-
tgt,
|
9205
|
-
spawn_src,
|
9206
|
-
debug=bs.main_config.debug,
|
9207
|
-
) as proc:
|
9208
|
-
res = await PyremoteBootstrapDriver( # noqa
|
9209
|
-
remote_src,
|
9210
|
-
PyremoteBootstrapOptions(
|
9211
|
-
debug=bs.main_config.debug,
|
9212
|
-
),
|
9213
|
-
).async_run(
|
9214
|
-
proc.stdout,
|
9215
|
-
proc.stdin,
|
9216
|
-
)
|
9217
|
-
|
9218
|
-
chan = RemoteChannelImpl(
|
9219
|
-
proc.stdout,
|
9220
|
-
proc.stdin,
|
9221
|
-
msh=self._msh,
|
9222
|
-
)
|
9223
|
-
|
9224
|
-
await chan.send_obj(bs)
|
9225
|
-
|
9226
|
-
rce: RemoteCommandExecutor
|
9227
|
-
async with aclosing(RemoteCommandExecutor(chan)) as rce:
|
9228
|
-
await rce.start()
|
9111
|
+
]
|
9229
9112
|
|
9230
|
-
|
9113
|
+
await asyncio_subprocesses.check_call(
|
9114
|
+
*full_args,
|
9115
|
+
env=env,
|
9116
|
+
)
|
9117
|
+
|
9118
|
+
exe = os.path.join(await self.install_dir(), 'bin', 'python')
|
9119
|
+
if not os.path.isfile(exe):
|
9120
|
+
raise RuntimeError(f'Interpreter not found: {exe}')
|
9121
|
+
return exe
|
9231
9122
|
|
9232
9123
|
|
9233
9124
|
##
|
9234
9125
|
|
9235
9126
|
|
9236
|
-
class
|
9127
|
+
class PyenvInterpProvider(InterpProvider):
|
9237
9128
|
def __init__(
|
9238
9129
|
self,
|
9130
|
+
pyenv: Pyenv = Pyenv(),
|
9131
|
+
|
9132
|
+
inspect: bool = False,
|
9133
|
+
inspector: InterpInspector = INTERP_INSPECTOR,
|
9134
|
+
|
9239
9135
|
*,
|
9240
|
-
|
9241
|
-
|
9136
|
+
|
9137
|
+
try_update: bool = False,
|
9242
9138
|
) -> None:
|
9243
9139
|
super().__init__()
|
9244
9140
|
|
9245
|
-
self.
|
9246
|
-
self._local_executor = local_executor
|
9141
|
+
self._pyenv = pyenv
|
9247
9142
|
|
9248
|
-
|
9249
|
-
|
9250
|
-
r0, w0 = asyncio_create_bytes_channel()
|
9251
|
-
r1, w1 = asyncio_create_bytes_channel()
|
9143
|
+
self._inspect = inspect
|
9144
|
+
self._inspector = inspector
|
9252
9145
|
|
9253
|
-
|
9254
|
-
local_chan = RemoteChannelImpl(r1, w0, msh=self._msh)
|
9146
|
+
self._try_update = try_update
|
9255
9147
|
|
9256
|
-
|
9257
|
-
|
9258
|
-
|
9259
|
-
|
9260
|
-
|
9148
|
+
#
|
9149
|
+
|
9150
|
+
@staticmethod
|
9151
|
+
def guess_version(s: str) -> ta.Optional[InterpVersion]:
|
9152
|
+
def strip_sfx(s: str, sfx: str) -> ta.Tuple[str, bool]:
|
9153
|
+
if s.endswith(sfx):
|
9154
|
+
return s[:-len(sfx)], True
|
9155
|
+
return s, False
|
9156
|
+
ok = {}
|
9157
|
+
s, ok['debug'] = strip_sfx(s, '-debug')
|
9158
|
+
s, ok['threaded'] = strip_sfx(s, 't')
|
9261
9159
|
try:
|
9262
|
-
|
9263
|
-
|
9264
|
-
|
9160
|
+
v = Version(s)
|
9161
|
+
except InvalidVersion:
|
9162
|
+
return None
|
9163
|
+
return InterpVersion(v, InterpOpts(**ok))
|
9265
9164
|
|
9266
|
-
|
9165
|
+
class Installed(ta.NamedTuple):
|
9166
|
+
name: str
|
9167
|
+
exe: str
|
9168
|
+
version: InterpVersion
|
9267
9169
|
|
9268
|
-
|
9269
|
-
|
9270
|
-
|
9170
|
+
async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
9171
|
+
iv: ta.Optional[InterpVersion]
|
9172
|
+
if self._inspect:
|
9173
|
+
try:
|
9174
|
+
iv = check.not_none(await self._inspector.inspect(ep)).iv
|
9175
|
+
except Exception as e: # noqa
|
9176
|
+
return None
|
9177
|
+
else:
|
9178
|
+
iv = self.guess_version(vn)
|
9179
|
+
if iv is None:
|
9180
|
+
return None
|
9181
|
+
return PyenvInterpProvider.Installed(
|
9182
|
+
name=vn,
|
9183
|
+
exe=ep,
|
9184
|
+
version=iv,
|
9185
|
+
)
|
9271
9186
|
|
9187
|
+
async def installed(self) -> ta.Sequence[Installed]:
|
9188
|
+
ret: ta.List[PyenvInterpProvider.Installed] = []
|
9189
|
+
for vn, ep in await self._pyenv.version_exes():
|
9190
|
+
if (i := await self._make_installed(vn, ep)) is None:
|
9191
|
+
log.debug('Invalid pyenv version: %s', vn)
|
9192
|
+
continue
|
9193
|
+
ret.append(i)
|
9194
|
+
return ret
|
9272
9195
|
|
9273
|
-
|
9274
|
-
# ../system/inject.py
|
9196
|
+
#
|
9275
9197
|
|
9198
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9199
|
+
return [i.version for i in await self.installed()]
|
9276
9200
|
|
9277
|
-
def
|
9278
|
-
|
9279
|
-
|
9280
|
-
|
9281
|
-
|
9282
|
-
|
9283
|
-
|
9201
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
9202
|
+
for i in await self.installed():
|
9203
|
+
if i.version == version:
|
9204
|
+
return Interp(
|
9205
|
+
exe=i.exe,
|
9206
|
+
version=i.version,
|
9207
|
+
)
|
9208
|
+
raise KeyError(version)
|
9284
9209
|
|
9285
9210
|
#
|
9286
9211
|
|
9287
|
-
|
9288
|
-
|
9212
|
+
async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9213
|
+
lst = []
|
9289
9214
|
|
9290
|
-
|
9215
|
+
for vs in await self._pyenv.installable_versions():
|
9216
|
+
if (iv := self.guess_version(vs)) is None:
|
9217
|
+
continue
|
9218
|
+
if iv.opts.debug:
|
9219
|
+
raise Exception('Pyenv installable versions not expected to have debug suffix')
|
9220
|
+
for d in [False, True]:
|
9221
|
+
lst.append(dc.replace(iv, opts=dc.replace(iv.opts, debug=d)))
|
9291
9222
|
|
9292
|
-
|
9293
|
-
lst.extend([
|
9294
|
-
inj.bind(YumSystemPackageManager, singleton=True),
|
9295
|
-
inj.bind(SystemPackageManager, to_key=YumSystemPackageManager),
|
9296
|
-
])
|
9223
|
+
return lst
|
9297
9224
|
|
9298
|
-
|
9299
|
-
lst.
|
9300
|
-
inj.bind(AptSystemPackageManager, singleton=True),
|
9301
|
-
inj.bind(SystemPackageManager, to_key=AptSystemPackageManager),
|
9302
|
-
])
|
9225
|
+
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9226
|
+
lst = await self._get_installable_versions(spec)
|
9303
9227
|
|
9304
|
-
|
9305
|
-
|
9306
|
-
|
9307
|
-
inj.bind(SystemPackageManager, to_key=BrewSystemPackageManager),
|
9308
|
-
])
|
9228
|
+
if self._try_update and not any(v in spec for v in lst):
|
9229
|
+
if self._pyenv.update():
|
9230
|
+
lst = await self._get_installable_versions(spec)
|
9309
9231
|
|
9310
|
-
|
9232
|
+
return lst
|
9311
9233
|
|
9312
|
-
|
9313
|
-
|
9314
|
-
|
9234
|
+
async def install_version(self, version: InterpVersion) -> Interp:
|
9235
|
+
inst_version = str(version.version)
|
9236
|
+
inst_opts = version.opts
|
9237
|
+
if inst_opts.threaded:
|
9238
|
+
inst_version += 't'
|
9239
|
+
inst_opts = dc.replace(inst_opts, threaded=False)
|
9315
9240
|
|
9316
|
-
|
9241
|
+
installer = PyenvVersionInstaller(
|
9242
|
+
inst_version,
|
9243
|
+
interp_opts=inst_opts,
|
9244
|
+
)
|
9317
9245
|
|
9318
|
-
|
9246
|
+
exe = await installer.install()
|
9247
|
+
return Interp(exe, version)
|
9319
9248
|
|
9320
9249
|
|
9321
9250
|
########################################
|
9322
|
-
# ../../../omdev/interp/
|
9323
|
-
|
9324
|
-
|
9325
|
-
|
9326
|
-
|
9327
|
-
|
9251
|
+
# ../../../omdev/interp/system.py
|
9252
|
+
"""
|
9253
|
+
TODO:
|
9254
|
+
- python, python3, python3.12, ...
|
9255
|
+
- check if path py's are venvs: sys.prefix != sys.base_prefix
|
9256
|
+
"""
|
9328
9257
|
|
9329
9258
|
|
9330
|
-
|
9331
|
-
def __init__(
|
9332
|
-
self,
|
9333
|
-
providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
|
9334
|
-
) -> None:
|
9335
|
-
super().__init__()
|
9259
|
+
##
|
9336
9260
|
|
9337
|
-
self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
|
9338
9261
|
|
9339
|
-
|
9340
|
-
|
9341
|
-
|
9342
|
-
|
9343
|
-
for si in await p.get_installed_versions(spec)
|
9344
|
-
if spec.contains(si)
|
9345
|
-
]
|
9262
|
+
@dc.dataclass(frozen=True)
|
9263
|
+
class SystemInterpProvider(InterpProvider):
|
9264
|
+
cmd: str = 'python3'
|
9265
|
+
path: ta.Optional[str] = None
|
9346
9266
|
|
9347
|
-
|
9348
|
-
|
9349
|
-
return None
|
9267
|
+
inspect: bool = False
|
9268
|
+
inspector: InterpInspector = INTERP_INSPECTOR
|
9350
9269
|
|
9351
|
-
|
9352
|
-
bp = list(self._providers.values())[bi]
|
9353
|
-
return (bp, bv)
|
9270
|
+
#
|
9354
9271
|
|
9355
|
-
|
9356
|
-
|
9357
|
-
|
9272
|
+
@staticmethod
|
9273
|
+
def _re_which(
|
9274
|
+
pat: re.Pattern,
|
9358
9275
|
*,
|
9359
|
-
|
9360
|
-
|
9361
|
-
|
9362
|
-
if
|
9363
|
-
|
9364
|
-
|
9276
|
+
mode: int = os.F_OK | os.X_OK,
|
9277
|
+
path: ta.Optional[str] = None,
|
9278
|
+
) -> ta.List[str]:
|
9279
|
+
if path is None:
|
9280
|
+
path = os.environ.get('PATH', None)
|
9281
|
+
if path is None:
|
9282
|
+
try:
|
9283
|
+
path = os.confstr('CS_PATH')
|
9284
|
+
except (AttributeError, ValueError):
|
9285
|
+
path = os.defpath
|
9365
9286
|
|
9366
|
-
if not
|
9367
|
-
return
|
9287
|
+
if not path:
|
9288
|
+
return []
|
9368
9289
|
|
9369
|
-
|
9290
|
+
path = os.fsdecode(path)
|
9291
|
+
pathlst = path.split(os.pathsep)
|
9370
9292
|
|
9371
|
-
|
9372
|
-
|
9373
|
-
|
9374
|
-
|
9375
|
-
|
9376
|
-
|
9293
|
+
def _access_check(fn: str, mode: int) -> bool:
|
9294
|
+
return os.path.exists(fn) and os.access(fn, mode)
|
9295
|
+
|
9296
|
+
out = []
|
9297
|
+
seen = set()
|
9298
|
+
for d in pathlst:
|
9299
|
+
normdir = os.path.normcase(d)
|
9300
|
+
if normdir not in seen:
|
9301
|
+
seen.add(normdir)
|
9302
|
+
if not _access_check(normdir, mode):
|
9303
|
+
continue
|
9304
|
+
for thefile in os.listdir(d):
|
9305
|
+
name = os.path.join(d, thefile)
|
9306
|
+
if not (
|
9307
|
+
os.path.isfile(name) and
|
9308
|
+
pat.fullmatch(thefile) and
|
9309
|
+
_access_check(name, mode)
|
9310
|
+
):
|
9311
|
+
continue
|
9312
|
+
out.append(name)
|
9377
9313
|
|
9378
|
-
|
9379
|
-
return await tp.install_version(bv)
|
9314
|
+
return out
|
9380
9315
|
|
9381
|
-
|
9382
|
-
|
9383
|
-
|
9384
|
-
|
9385
|
-
|
9386
|
-
|
9387
|
-
if spec.contains(si)
|
9388
|
-
]
|
9389
|
-
if lst:
|
9390
|
-
print(f' {n}')
|
9391
|
-
for si in lst:
|
9392
|
-
print(f' {si}')
|
9316
|
+
@cached_nullary
|
9317
|
+
def exes(self) -> ta.List[str]:
|
9318
|
+
return self._re_which(
|
9319
|
+
re.compile(r'python3(\.\d+)?'),
|
9320
|
+
path=self.path,
|
9321
|
+
)
|
9393
9322
|
|
9394
|
-
|
9323
|
+
#
|
9395
9324
|
|
9396
|
-
|
9397
|
-
|
9398
|
-
|
9399
|
-
|
9400
|
-
|
9401
|
-
|
9402
|
-
|
9403
|
-
|
9404
|
-
|
9405
|
-
|
9406
|
-
|
9325
|
+
async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
9326
|
+
if not self.inspect:
|
9327
|
+
s = os.path.basename(exe)
|
9328
|
+
if s.startswith('python'):
|
9329
|
+
s = s[len('python'):]
|
9330
|
+
if '.' in s:
|
9331
|
+
try:
|
9332
|
+
return InterpVersion.parse(s)
|
9333
|
+
except InvalidVersion:
|
9334
|
+
pass
|
9335
|
+
ii = await self.inspector.inspect(exe)
|
9336
|
+
return ii.iv if ii is not None else None
|
9407
9337
|
|
9338
|
+
async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
9339
|
+
lst = []
|
9340
|
+
for e in self.exes():
|
9341
|
+
if (ev := await self.get_exe_version(e)) is None:
|
9342
|
+
log.debug('Invalid system version: %s', e)
|
9343
|
+
continue
|
9344
|
+
lst.append((e, ev))
|
9345
|
+
return lst
|
9408
9346
|
|
9409
|
-
|
9410
|
-
# pyenv is preferred to system interpreters as it tends to have more support for things like tkinter
|
9411
|
-
PyenvInterpProvider(try_update=True),
|
9347
|
+
#
|
9412
9348
|
|
9413
|
-
|
9349
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9350
|
+
return [ev for e, ev in await self.exe_versions()]
|
9414
9351
|
|
9415
|
-
|
9416
|
-
|
9352
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
9353
|
+
for e, ev in await self.exe_versions():
|
9354
|
+
if ev != version:
|
9355
|
+
continue
|
9356
|
+
return Interp(
|
9357
|
+
exe=e,
|
9358
|
+
version=ev,
|
9359
|
+
)
|
9360
|
+
raise KeyError(version)
|
9417
9361
|
|
9418
9362
|
|
9419
9363
|
########################################
|
@@ -9444,6 +9388,54 @@ def bind_remote(
|
|
9444
9388
|
return inj.as_bindings(*lst)
|
9445
9389
|
|
9446
9390
|
|
9391
|
+
########################################
|
9392
|
+
# ../system/inject.py
|
9393
|
+
|
9394
|
+
|
9395
|
+
def bind_system(
|
9396
|
+
*,
|
9397
|
+
system_config: SystemConfig,
|
9398
|
+
) -> InjectorBindings:
|
9399
|
+
lst: ta.List[InjectorBindingOrBindings] = [
|
9400
|
+
inj.bind(system_config),
|
9401
|
+
]
|
9402
|
+
|
9403
|
+
#
|
9404
|
+
|
9405
|
+
platform = system_config.platform or detect_system_platform()
|
9406
|
+
lst.append(inj.bind(platform, key=Platform))
|
9407
|
+
|
9408
|
+
#
|
9409
|
+
|
9410
|
+
if isinstance(platform, AmazonLinuxPlatform):
|
9411
|
+
lst.extend([
|
9412
|
+
inj.bind(YumSystemPackageManager, singleton=True),
|
9413
|
+
inj.bind(SystemPackageManager, to_key=YumSystemPackageManager),
|
9414
|
+
])
|
9415
|
+
|
9416
|
+
elif isinstance(platform, LinuxPlatform):
|
9417
|
+
lst.extend([
|
9418
|
+
inj.bind(AptSystemPackageManager, singleton=True),
|
9419
|
+
inj.bind(SystemPackageManager, to_key=AptSystemPackageManager),
|
9420
|
+
])
|
9421
|
+
|
9422
|
+
elif isinstance(platform, DarwinPlatform):
|
9423
|
+
lst.extend([
|
9424
|
+
inj.bind(BrewSystemPackageManager, singleton=True),
|
9425
|
+
inj.bind(SystemPackageManager, to_key=BrewSystemPackageManager),
|
9426
|
+
])
|
9427
|
+
|
9428
|
+
#
|
9429
|
+
|
9430
|
+
lst.extend([
|
9431
|
+
bind_command(CheckSystemPackageCommand, CheckSystemPackageCommandExecutor),
|
9432
|
+
])
|
9433
|
+
|
9434
|
+
#
|
9435
|
+
|
9436
|
+
return inj.as_bindings(*lst)
|
9437
|
+
|
9438
|
+
|
9447
9439
|
########################################
|
9448
9440
|
# ../targets/connection.py
|
9449
9441
|
|
@@ -9579,33 +9571,101 @@ class SshManageTargetConnector(ManageTargetConnector):
|
|
9579
9571
|
|
9580
9572
|
|
9581
9573
|
########################################
|
9582
|
-
#
|
9574
|
+
# ../../../omdev/interp/resolvers.py
|
9583
9575
|
|
9584
9576
|
|
9585
|
-
|
9577
|
+
INTERP_PROVIDER_TYPES_BY_NAME: ta.Mapping[str, ta.Type[InterpProvider]] = {
|
9578
|
+
cls.name: cls for cls in deep_subclasses(InterpProvider) if abc.ABC not in cls.__bases__ # type: ignore
|
9579
|
+
}
|
9586
9580
|
|
9587
9581
|
|
9588
|
-
|
9589
|
-
|
9590
|
-
|
9591
|
-
|
9582
|
+
class InterpResolver:
|
9583
|
+
def __init__(
|
9584
|
+
self,
|
9585
|
+
providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
|
9586
|
+
) -> None:
|
9587
|
+
super().__init__()
|
9592
9588
|
|
9593
|
-
|
9594
|
-
class Output(Command.Output):
|
9595
|
-
exe: str
|
9596
|
-
version: str
|
9597
|
-
opts: InterpOpts
|
9589
|
+
self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
|
9598
9590
|
|
9591
|
+
async def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
|
9592
|
+
lst = [
|
9593
|
+
(i, si)
|
9594
|
+
for i, p in enumerate(self._providers.values())
|
9595
|
+
for si in await p.get_installed_versions(spec)
|
9596
|
+
if spec.contains(si)
|
9597
|
+
]
|
9599
9598
|
|
9600
|
-
|
9601
|
-
|
9602
|
-
|
9603
|
-
|
9604
|
-
|
9605
|
-
|
9606
|
-
|
9607
|
-
|
9599
|
+
slst = sorted(lst, key=lambda t: (-t[0], t[1].version))
|
9600
|
+
if not slst:
|
9601
|
+
return None
|
9602
|
+
|
9603
|
+
bi, bv = slst[-1]
|
9604
|
+
bp = list(self._providers.values())[bi]
|
9605
|
+
return (bp, bv)
|
9606
|
+
|
9607
|
+
async def resolve(
|
9608
|
+
self,
|
9609
|
+
spec: InterpSpecifier,
|
9610
|
+
*,
|
9611
|
+
install: bool = False,
|
9612
|
+
) -> ta.Optional[Interp]:
|
9613
|
+
tup = await self._resolve_installed(spec)
|
9614
|
+
if tup is not None:
|
9615
|
+
bp, bv = tup
|
9616
|
+
return await bp.get_installed_version(bv)
|
9617
|
+
|
9618
|
+
if not install:
|
9619
|
+
return None
|
9620
|
+
|
9621
|
+
tp = list(self._providers.values())[0] # noqa
|
9622
|
+
|
9623
|
+
sv = sorted(
|
9624
|
+
[s for s in await tp.get_installable_versions(spec) if s in spec],
|
9625
|
+
key=lambda s: s.version,
|
9608
9626
|
)
|
9627
|
+
if not sv:
|
9628
|
+
return None
|
9629
|
+
|
9630
|
+
bv = sv[-1]
|
9631
|
+
return await tp.install_version(bv)
|
9632
|
+
|
9633
|
+
async def list(self, spec: InterpSpecifier) -> None:
|
9634
|
+
print('installed:')
|
9635
|
+
for n, p in self._providers.items():
|
9636
|
+
lst = [
|
9637
|
+
si
|
9638
|
+
for si in await p.get_installed_versions(spec)
|
9639
|
+
if spec.contains(si)
|
9640
|
+
]
|
9641
|
+
if lst:
|
9642
|
+
print(f' {n}')
|
9643
|
+
for si in lst:
|
9644
|
+
print(f' {si}')
|
9645
|
+
|
9646
|
+
print()
|
9647
|
+
|
9648
|
+
print('installable:')
|
9649
|
+
for n, p in self._providers.items():
|
9650
|
+
lst = [
|
9651
|
+
si
|
9652
|
+
for si in await p.get_installable_versions(spec)
|
9653
|
+
if spec.contains(si)
|
9654
|
+
]
|
9655
|
+
if lst:
|
9656
|
+
print(f' {n}')
|
9657
|
+
for si in lst:
|
9658
|
+
print(f' {si}')
|
9659
|
+
|
9660
|
+
|
9661
|
+
DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
|
9662
|
+
# pyenv is preferred to system interpreters as it tends to have more support for things like tkinter
|
9663
|
+
PyenvInterpProvider(try_update=True),
|
9664
|
+
|
9665
|
+
RunningInterpProvider(),
|
9666
|
+
|
9667
|
+
SystemInterpProvider(),
|
9668
|
+
]])
|
9609
9669
|
|
9610
9670
|
|
9611
9671
|
########################################
|
@@ -9637,6 +9697,36 @@ def bind_targets() -> InjectorBindings:
|
|
9637
9697
|
return inj.as_bindings(*lst)
|
9638
9698
|
|
9639
9699
|
|
9700
|
+
########################################
|
9701
|
+
# ../deploy/interp.py
|
9702
|
+
|
9703
|
+
|
9704
|
+
##
|
9705
|
+
|
9706
|
+
|
9707
|
+
@dc.dataclass(frozen=True)
|
9708
|
+
class InterpCommand(Command['InterpCommand.Output']):
|
9709
|
+
spec: str
|
9710
|
+
install: bool = False
|
9711
|
+
|
9712
|
+
@dc.dataclass(frozen=True)
|
9713
|
+
class Output(Command.Output):
|
9714
|
+
exe: str
|
9715
|
+
version: str
|
9716
|
+
opts: InterpOpts
|
9717
|
+
|
9718
|
+
|
9719
|
+
class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
|
9720
|
+
async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
|
9721
|
+
i = InterpSpecifier.parse(check.not_none(cmd.spec))
|
9722
|
+
o = check.not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
|
9723
|
+
return InterpCommand.Output(
|
9724
|
+
exe=o.exe,
|
9725
|
+
version=str(o.version.version),
|
9726
|
+
opts=o.version.opts,
|
9727
|
+
)
|
9728
|
+
|
9729
|
+
|
9640
9730
|
########################################
|
9641
9731
|
# ../deploy/inject.py
|
9642
9732
|
|