ominfra 0.0.0.dev157__py3-none-any.whl → 0.0.0.dev158__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/clouds/aws/journald2aws/main.py +1 -1
- ominfra/journald/tailer.py +2 -2
- ominfra/manage/bootstrap_.py +1 -1
- ominfra/manage/commands/subprocess.py +4 -4
- ominfra/manage/deploy/apps.py +14 -15
- ominfra/manage/deploy/config.py +3 -0
- ominfra/manage/deploy/git.py +11 -27
- ominfra/manage/deploy/paths.py +48 -48
- ominfra/manage/deploy/specs.py +32 -0
- ominfra/manage/deploy/venvs.py +10 -5
- ominfra/manage/remote/spawning.py +3 -3
- ominfra/manage/system/packages.py +1 -1
- ominfra/pyremote.py +26 -26
- ominfra/scripts/journald2aws.py +461 -350
- ominfra/scripts/manage.py +1783 -1693
- ominfra/scripts/supervisor.py +353 -332
- ominfra/supervisor/http.py +1 -1
- ominfra/supervisor/main.py +2 -2
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/RECORD +24 -23
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -100,7 +100,7 @@ CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
|
|
100
100
|
|
101
101
|
# deploy/paths.py
|
102
102
|
DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
|
103
|
-
|
103
|
+
DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
|
104
104
|
|
105
105
|
# ../../omlish/argparse/cli.py
|
106
106
|
ArgparseCommandFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
|
@@ -118,7 +118,7 @@ InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
|
|
118
118
|
# ../configs.py
|
119
119
|
ConfigMapping = ta.Mapping[str, ta.Any]
|
120
120
|
|
121
|
-
# ../../omlish/
|
121
|
+
# ../../omlish/subprocesses.py
|
122
122
|
SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
|
123
123
|
|
124
124
|
# system/packages.py
|
@@ -1365,6 +1365,9 @@ class MainConfig:
|
|
1365
1365
|
# ../deploy/config.py
|
1366
1366
|
|
1367
1367
|
|
1368
|
+
##
|
1369
|
+
|
1370
|
+
|
1368
1371
|
@dc.dataclass(frozen=True)
|
1369
1372
|
class DeployConfig:
|
1370
1373
|
deploy_home: ta.Optional[str] = None
|
@@ -1539,7 +1542,7 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
1539
1542
|
# Get pid
|
1540
1543
|
pid = os.getpid()
|
1541
1544
|
|
1542
|
-
# Two copies of
|
1545
|
+
# Two copies of payload src to be sent to parent
|
1543
1546
|
r0, w0 = os.pipe()
|
1544
1547
|
r1, w1 = os.pipe()
|
1545
1548
|
|
@@ -1578,17 +1581,17 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
1578
1581
|
# Write pid
|
1579
1582
|
os.write(1, struct.pack('<Q', pid))
|
1580
1583
|
|
1581
|
-
# Read
|
1582
|
-
|
1583
|
-
if len(
|
1584
|
+
# Read payload src from stdin
|
1585
|
+
payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
|
1586
|
+
if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
|
1584
1587
|
raise EOFError
|
1585
|
-
|
1588
|
+
payload_src = zlib.decompress(payload_z)
|
1586
1589
|
|
1587
|
-
# Write both copies of
|
1588
|
-
# and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
1590
|
+
# Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
|
1591
|
+
# fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
1589
1592
|
for w in [w0, w1]:
|
1590
1593
|
fp = os.fdopen(w, 'wb', 0)
|
1591
|
-
fp.write(
|
1594
|
+
fp.write(payload_src)
|
1592
1595
|
fp.close()
|
1593
1596
|
|
1594
1597
|
# Write second ack
|
@@ -1652,7 +1655,7 @@ class PyremotePayloadRuntime:
|
|
1652
1655
|
input: ta.BinaryIO
|
1653
1656
|
output: ta.BinaryIO
|
1654
1657
|
context_name: str
|
1655
|
-
|
1658
|
+
payload_src: str
|
1656
1659
|
options: PyremoteBootstrapOptions
|
1657
1660
|
env_info: PyremoteEnvInfo
|
1658
1661
|
|
@@ -1660,9 +1663,9 @@ class PyremotePayloadRuntime:
|
|
1660
1663
|
def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
1661
1664
|
# If src file var is not present we need to do initial finalization
|
1662
1665
|
if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
|
1663
|
-
# Read second copy of
|
1666
|
+
# Read second copy of payload src
|
1664
1667
|
r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
|
1665
|
-
|
1668
|
+
payload_src = r1.read().decode('utf-8')
|
1666
1669
|
r1.close()
|
1667
1670
|
|
1668
1671
|
# Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
|
@@ -1680,7 +1683,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
1680
1683
|
# Write temp source file
|
1681
1684
|
import tempfile
|
1682
1685
|
tfd, tfn = tempfile.mkstemp('-pyremote.py')
|
1683
|
-
os.write(tfd,
|
1686
|
+
os.write(tfd, payload_src.encode('utf-8'))
|
1684
1687
|
os.close(tfd)
|
1685
1688
|
|
1686
1689
|
# Set vars
|
@@ -1699,7 +1702,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
1699
1702
|
|
1700
1703
|
# Read temp source file
|
1701
1704
|
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
1702
|
-
|
1705
|
+
payload_src = sf.read()
|
1703
1706
|
|
1704
1707
|
# Restore vars
|
1705
1708
|
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
@@ -1732,7 +1735,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
1732
1735
|
input=input,
|
1733
1736
|
output=output,
|
1734
1737
|
context_name=context_name,
|
1735
|
-
|
1738
|
+
payload_src=payload_src,
|
1736
1739
|
options=options,
|
1737
1740
|
env_info=env_info,
|
1738
1741
|
)
|
@@ -1744,31 +1747,31 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
1744
1747
|
class PyremoteBootstrapDriver:
|
1745
1748
|
def __init__(
|
1746
1749
|
self,
|
1747
|
-
|
1750
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
1748
1751
|
options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
|
1749
1752
|
) -> None:
|
1750
1753
|
super().__init__()
|
1751
1754
|
|
1752
|
-
self.
|
1755
|
+
self._payload_src = payload_src
|
1753
1756
|
self._options = options
|
1754
1757
|
|
1755
|
-
self.
|
1756
|
-
self.
|
1758
|
+
self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
|
1759
|
+
self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
|
1757
1760
|
|
1758
1761
|
self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
|
1759
1762
|
#
|
1760
1763
|
|
1761
1764
|
@classmethod
|
1762
|
-
def
|
1765
|
+
def _prepare_payload_src(
|
1763
1766
|
cls,
|
1764
|
-
|
1767
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
1765
1768
|
options: PyremoteBootstrapOptions,
|
1766
1769
|
) -> str:
|
1767
1770
|
parts: ta.List[str]
|
1768
|
-
if isinstance(
|
1769
|
-
parts = [
|
1771
|
+
if isinstance(payload_src, str):
|
1772
|
+
parts = [payload_src]
|
1770
1773
|
else:
|
1771
|
-
parts = list(
|
1774
|
+
parts = list(payload_src)
|
1772
1775
|
|
1773
1776
|
if (mn := options.main_name_override) is not None:
|
1774
1777
|
parts.insert(0, f'__name__ = {mn!r}')
|
@@ -1804,9 +1807,9 @@ class PyremoteBootstrapDriver:
|
|
1804
1807
|
d = yield from self._read(8)
|
1805
1808
|
pid = struct.unpack('<Q', d)[0]
|
1806
1809
|
|
1807
|
-
# Write
|
1808
|
-
yield from self._write(struct.pack('<I', len(self.
|
1809
|
-
yield from self._write(self.
|
1810
|
+
# Write payload src
|
1811
|
+
yield from self._write(struct.pack('<I', len(self._payload_z)))
|
1812
|
+
yield from self._write(self._payload_z)
|
1810
1813
|
|
1811
1814
|
# Read second ack (after writing src copies)
|
1812
1815
|
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
|
@@ -2540,6 +2543,13 @@ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON
|
|
2540
2543
|
json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
|
2541
2544
|
|
2542
2545
|
|
2546
|
+
########################################
|
2547
|
+
# ../../../omlish/lite/logs.py
|
2548
|
+
|
2549
|
+
|
2550
|
+
log = logging.getLogger(__name__)
|
2551
|
+
|
2552
|
+
|
2543
2553
|
########################################
|
2544
2554
|
# ../../../omlish/lite/maybes.py
|
2545
2555
|
|
@@ -2754,6 +2764,116 @@ def format_num_bytes(num_bytes: int) -> str:
|
|
2754
2764
|
return f'{num_bytes / 1024 ** (len(FORMAT_NUM_BYTES_SUFFIXES) - 1):.2f}{FORMAT_NUM_BYTES_SUFFIXES[-1]}'
|
2755
2765
|
|
2756
2766
|
|
2767
|
+
########################################
|
2768
|
+
# ../../../omlish/logs/filters.py
|
2769
|
+
|
2770
|
+
|
2771
|
+
class TidLogFilter(logging.Filter):
|
2772
|
+
def filter(self, record):
|
2773
|
+
record.tid = threading.get_native_id()
|
2774
|
+
return True
|
2775
|
+
|
2776
|
+
|
2777
|
+
########################################
|
2778
|
+
# ../../../omlish/logs/proxy.py
|
2779
|
+
|
2780
|
+
|
2781
|
+
class ProxyLogFilterer(logging.Filterer):
|
2782
|
+
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
2783
|
+
self._underlying = underlying
|
2784
|
+
|
2785
|
+
@property
|
2786
|
+
def underlying(self) -> logging.Filterer:
|
2787
|
+
return self._underlying
|
2788
|
+
|
2789
|
+
@property
|
2790
|
+
def filters(self):
|
2791
|
+
return self._underlying.filters
|
2792
|
+
|
2793
|
+
@filters.setter
|
2794
|
+
def filters(self, filters):
|
2795
|
+
self._underlying.filters = filters
|
2796
|
+
|
2797
|
+
def addFilter(self, filter): # noqa
|
2798
|
+
self._underlying.addFilter(filter)
|
2799
|
+
|
2800
|
+
def removeFilter(self, filter): # noqa
|
2801
|
+
self._underlying.removeFilter(filter)
|
2802
|
+
|
2803
|
+
def filter(self, record):
|
2804
|
+
return self._underlying.filter(record)
|
2805
|
+
|
2806
|
+
|
2807
|
+
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
2808
|
+
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
2809
|
+
ProxyLogFilterer.__init__(self, underlying)
|
2810
|
+
|
2811
|
+
_underlying: logging.Handler
|
2812
|
+
|
2813
|
+
@property
|
2814
|
+
def underlying(self) -> logging.Handler:
|
2815
|
+
return self._underlying
|
2816
|
+
|
2817
|
+
def get_name(self):
|
2818
|
+
return self._underlying.get_name()
|
2819
|
+
|
2820
|
+
def set_name(self, name):
|
2821
|
+
self._underlying.set_name(name)
|
2822
|
+
|
2823
|
+
@property
|
2824
|
+
def name(self):
|
2825
|
+
return self._underlying.name
|
2826
|
+
|
2827
|
+
@property
|
2828
|
+
def level(self):
|
2829
|
+
return self._underlying.level
|
2830
|
+
|
2831
|
+
@level.setter
|
2832
|
+
def level(self, level):
|
2833
|
+
self._underlying.level = level
|
2834
|
+
|
2835
|
+
@property
|
2836
|
+
def formatter(self):
|
2837
|
+
return self._underlying.formatter
|
2838
|
+
|
2839
|
+
@formatter.setter
|
2840
|
+
def formatter(self, formatter):
|
2841
|
+
self._underlying.formatter = formatter
|
2842
|
+
|
2843
|
+
def createLock(self):
|
2844
|
+
self._underlying.createLock()
|
2845
|
+
|
2846
|
+
def acquire(self):
|
2847
|
+
self._underlying.acquire()
|
2848
|
+
|
2849
|
+
def release(self):
|
2850
|
+
self._underlying.release()
|
2851
|
+
|
2852
|
+
def setLevel(self, level):
|
2853
|
+
self._underlying.setLevel(level)
|
2854
|
+
|
2855
|
+
def format(self, record):
|
2856
|
+
return self._underlying.format(record)
|
2857
|
+
|
2858
|
+
def emit(self, record):
|
2859
|
+
self._underlying.emit(record)
|
2860
|
+
|
2861
|
+
def handle(self, record):
|
2862
|
+
return self._underlying.handle(record)
|
2863
|
+
|
2864
|
+
def setFormatter(self, fmt):
|
2865
|
+
self._underlying.setFormatter(fmt)
|
2866
|
+
|
2867
|
+
def flush(self):
|
2868
|
+
self._underlying.flush()
|
2869
|
+
|
2870
|
+
def close(self):
|
2871
|
+
self._underlying.close()
|
2872
|
+
|
2873
|
+
def handleError(self, record):
|
2874
|
+
self._underlying.handleError(record)
|
2875
|
+
|
2876
|
+
|
2757
2877
|
########################################
|
2758
2878
|
# ../../../omlish/os/deathsig.py
|
2759
2879
|
|
@@ -3939,22 +4059,22 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
|
3939
4059
|
~deploy
|
3940
4060
|
deploy.pid (flock)
|
3941
4061
|
/app
|
3942
|
-
/<
|
4062
|
+
/<appplaceholder> - shallow clone
|
3943
4063
|
/conf
|
3944
4064
|
/env
|
3945
|
-
<
|
4065
|
+
<appplaceholder>.env
|
3946
4066
|
/nginx
|
3947
|
-
<
|
4067
|
+
<appplaceholder>.conf
|
3948
4068
|
/supervisor
|
3949
|
-
<
|
4069
|
+
<appplaceholder>.conf
|
3950
4070
|
/venv
|
3951
|
-
/<
|
4071
|
+
/<appplaceholder>
|
3952
4072
|
|
3953
4073
|
?
|
3954
4074
|
/logs
|
3955
|
-
/wrmsr--omlish--<
|
4075
|
+
/wrmsr--omlish--<placeholder>
|
3956
4076
|
|
3957
|
-
|
4077
|
+
placeholder = <name>--<rev>--<when>
|
3958
4078
|
|
3959
4079
|
==
|
3960
4080
|
|
@@ -3975,10 +4095,10 @@ for dn in [
|
|
3975
4095
|
##
|
3976
4096
|
|
3977
4097
|
|
3978
|
-
|
3979
|
-
|
4098
|
+
DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER = '@'
|
4099
|
+
DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
|
3980
4100
|
|
3981
|
-
|
4101
|
+
DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
|
3982
4102
|
'app',
|
3983
4103
|
'tag', # <rev>-<dt>
|
3984
4104
|
])
|
@@ -3996,7 +4116,7 @@ class DeployPathPart(abc.ABC): # noqa
|
|
3996
4116
|
raise NotImplementedError
|
3997
4117
|
|
3998
4118
|
@abc.abstractmethod
|
3999
|
-
def render(self,
|
4119
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4000
4120
|
raise NotImplementedError
|
4001
4121
|
|
4002
4122
|
|
@@ -4010,9 +4130,9 @@ class DirDeployPathPart(DeployPathPart, abc.ABC):
|
|
4010
4130
|
|
4011
4131
|
@classmethod
|
4012
4132
|
def parse(cls, s: str) -> 'DirDeployPathPart':
|
4013
|
-
if
|
4014
|
-
check.equal(s[0],
|
4015
|
-
return
|
4133
|
+
if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
|
4134
|
+
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
4135
|
+
return PlaceholderDirDeployPathPart(s[1:])
|
4016
4136
|
else:
|
4017
4137
|
return ConstDirDeployPathPart(s)
|
4018
4138
|
|
@@ -4024,13 +4144,13 @@ class FileDeployPathPart(DeployPathPart, abc.ABC):
|
|
4024
4144
|
|
4025
4145
|
@classmethod
|
4026
4146
|
def parse(cls, s: str) -> 'FileDeployPathPart':
|
4027
|
-
if
|
4028
|
-
check.equal(s[0],
|
4029
|
-
if not any(c in s for c in
|
4030
|
-
return
|
4147
|
+
if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
|
4148
|
+
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
4149
|
+
if not any(c in s for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS):
|
4150
|
+
return PlaceholderFileDeployPathPart(s[1:], '')
|
4031
4151
|
else:
|
4032
|
-
p = min(f for c in
|
4033
|
-
return
|
4152
|
+
p = min(f for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS if (f := s.find(c)) > 0)
|
4153
|
+
return PlaceholderFileDeployPathPart(s[1:p], s[p:])
|
4034
4154
|
else:
|
4035
4155
|
return ConstFileDeployPathPart(s)
|
4036
4156
|
|
@@ -4045,9 +4165,9 @@ class ConstDeployPathPart(DeployPathPart, abc.ABC):
|
|
4045
4165
|
def __post_init__(self) -> None:
|
4046
4166
|
check.non_empty_str(self.name)
|
4047
4167
|
check.not_in('/', self.name)
|
4048
|
-
check.not_in(
|
4168
|
+
check.not_in(DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, self.name)
|
4049
4169
|
|
4050
|
-
def render(self,
|
4170
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4051
4171
|
return self.name
|
4052
4172
|
|
4053
4173
|
|
@@ -4063,40 +4183,40 @@ class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
|
|
4063
4183
|
|
4064
4184
|
|
4065
4185
|
@dc.dataclass(frozen=True)
|
4066
|
-
class
|
4067
|
-
|
4186
|
+
class PlaceholderDeployPathPart(DeployPathPart, abc.ABC):
|
4187
|
+
placeholder: str # DeployPathPlaceholder
|
4068
4188
|
|
4069
4189
|
def __post_init__(self) -> None:
|
4070
|
-
check.non_empty_str(self.
|
4071
|
-
for c in [*
|
4072
|
-
check.not_in(c, self.
|
4073
|
-
check.in_(self.
|
4074
|
-
|
4075
|
-
def
|
4076
|
-
if
|
4077
|
-
return
|
4190
|
+
check.non_empty_str(self.placeholder)
|
4191
|
+
for c in [*DEPLOY_PATH_PLACEHOLDER_SEPARATORS, DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
|
4192
|
+
check.not_in(c, self.placeholder)
|
4193
|
+
check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
|
4194
|
+
|
4195
|
+
def _render_placeholder(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4196
|
+
if placeholders is not None:
|
4197
|
+
return placeholders[self.placeholder] # type: ignore
|
4078
4198
|
else:
|
4079
|
-
return
|
4199
|
+
return DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER + self.placeholder
|
4080
4200
|
|
4081
4201
|
|
4082
4202
|
@dc.dataclass(frozen=True)
|
4083
|
-
class
|
4084
|
-
def render(self,
|
4085
|
-
return self.
|
4203
|
+
class PlaceholderDirDeployPathPart(PlaceholderDeployPathPart, DirDeployPathPart):
|
4204
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4205
|
+
return self._render_placeholder(placeholders)
|
4086
4206
|
|
4087
4207
|
|
4088
4208
|
@dc.dataclass(frozen=True)
|
4089
|
-
class
|
4209
|
+
class PlaceholderFileDeployPathPart(PlaceholderDeployPathPart, FileDeployPathPart):
|
4090
4210
|
suffix: str
|
4091
4211
|
|
4092
4212
|
def __post_init__(self) -> None:
|
4093
4213
|
super().__post_init__()
|
4094
4214
|
if self.suffix:
|
4095
|
-
for c in [
|
4215
|
+
for c in [DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
|
4096
4216
|
check.not_in(c, self.suffix)
|
4097
4217
|
|
4098
|
-
def render(self,
|
4099
|
-
return self.
|
4218
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4219
|
+
return self._render_placeholder(placeholders) + self.suffix
|
4100
4220
|
|
4101
4221
|
|
4102
4222
|
##
|
@@ -4113,22 +4233,22 @@ class DeployPath:
|
|
4113
4233
|
|
4114
4234
|
pd = {}
|
4115
4235
|
for i, p in enumerate(self.parts):
|
4116
|
-
if isinstance(p,
|
4117
|
-
if p.
|
4118
|
-
raise DeployPathError('Duplicate
|
4119
|
-
pd[p.
|
4236
|
+
if isinstance(p, PlaceholderDeployPathPart):
|
4237
|
+
if p.placeholder in pd:
|
4238
|
+
raise DeployPathError('Duplicate placeholders in path', self)
|
4239
|
+
pd[p.placeholder] = i
|
4120
4240
|
|
4121
4241
|
if 'tag' in pd:
|
4122
4242
|
if 'app' not in pd or pd['app'] >= pd['tag']:
|
4123
|
-
raise DeployPathError('Tag
|
4243
|
+
raise DeployPathError('Tag placeholder in path without preceding app', self)
|
4124
4244
|
|
4125
4245
|
@property
|
4126
4246
|
def kind(self) -> ta.Literal['file', 'dir']:
|
4127
4247
|
return self.parts[-1].kind
|
4128
4248
|
|
4129
|
-
def render(self,
|
4249
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4130
4250
|
return os.path.join( # noqa
|
4131
|
-
*[p.render(
|
4251
|
+
*[p.render(placeholders) for p in self.parts],
|
4132
4252
|
*([''] if self.kind == 'dir' else []),
|
4133
4253
|
)
|
4134
4254
|
|
@@ -4156,6 +4276,34 @@ class DeployPathOwner(abc.ABC):
|
|
4156
4276
|
raise NotImplementedError
|
4157
4277
|
|
4158
4278
|
|
4279
|
+
########################################
|
4280
|
+
# ../deploy/specs.py
|
4281
|
+
|
4282
|
+
|
4283
|
+
##
|
4284
|
+
|
4285
|
+
|
4286
|
+
@dc.dataclass(frozen=True)
|
4287
|
+
class DeployGitRepo:
|
4288
|
+
host: ta.Optional[str] = None
|
4289
|
+
username: ta.Optional[str] = None
|
4290
|
+
path: ta.Optional[str] = None
|
4291
|
+
|
4292
|
+
def __post_init__(self) -> None:
|
4293
|
+
check.not_in('..', check.non_empty_str(self.host))
|
4294
|
+
check.not_in('.', check.non_empty_str(self.path))
|
4295
|
+
|
4296
|
+
|
4297
|
+
##
|
4298
|
+
|
4299
|
+
|
4300
|
+
@dc.dataclass(frozen=True)
|
4301
|
+
class DeploySpec:
|
4302
|
+
app: DeployApp
|
4303
|
+
repo: DeployGitRepo
|
4304
|
+
rev: DeployRev
|
4305
|
+
|
4306
|
+
|
4159
4307
|
########################################
|
4160
4308
|
# ../remote/config.py
|
4161
4309
|
|
@@ -4216,6 +4364,75 @@ def get_remote_payload_src(
|
|
4216
4364
|
return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
|
4217
4365
|
|
4218
4366
|
|
4367
|
+
########################################
|
4368
|
+
# ../system/platforms.py
|
4369
|
+
|
4370
|
+
|
4371
|
+
##
|
4372
|
+
|
4373
|
+
|
4374
|
+
@dc.dataclass(frozen=True)
|
4375
|
+
class Platform(abc.ABC): # noqa
|
4376
|
+
pass
|
4377
|
+
|
4378
|
+
|
4379
|
+
class LinuxPlatform(Platform, abc.ABC):
|
4380
|
+
pass
|
4381
|
+
|
4382
|
+
|
4383
|
+
class UbuntuPlatform(LinuxPlatform):
|
4384
|
+
pass
|
4385
|
+
|
4386
|
+
|
4387
|
+
class AmazonLinuxPlatform(LinuxPlatform):
|
4388
|
+
pass
|
4389
|
+
|
4390
|
+
|
4391
|
+
class GenericLinuxPlatform(LinuxPlatform):
|
4392
|
+
pass
|
4393
|
+
|
4394
|
+
|
4395
|
+
class DarwinPlatform(Platform):
|
4396
|
+
pass
|
4397
|
+
|
4398
|
+
|
4399
|
+
class UnknownPlatform(Platform):
|
4400
|
+
pass
|
4401
|
+
|
4402
|
+
|
4403
|
+
##
|
4404
|
+
|
4405
|
+
|
4406
|
+
def _detect_system_platform() -> Platform:
|
4407
|
+
plat = sys.platform
|
4408
|
+
|
4409
|
+
if plat == 'linux':
|
4410
|
+
if (osr := LinuxOsRelease.read()) is None:
|
4411
|
+
return GenericLinuxPlatform()
|
4412
|
+
|
4413
|
+
if osr.id == 'amzn':
|
4414
|
+
return AmazonLinuxPlatform()
|
4415
|
+
|
4416
|
+
elif osr.id == 'ubuntu':
|
4417
|
+
return UbuntuPlatform()
|
4418
|
+
|
4419
|
+
else:
|
4420
|
+
return GenericLinuxPlatform()
|
4421
|
+
|
4422
|
+
elif plat == 'darwin':
|
4423
|
+
return DarwinPlatform()
|
4424
|
+
|
4425
|
+
else:
|
4426
|
+
return UnknownPlatform()
|
4427
|
+
|
4428
|
+
|
4429
|
+
@cached_nullary
|
4430
|
+
def detect_system_platform() -> Platform:
|
4431
|
+
platform = _detect_system_platform()
|
4432
|
+
log.info('Detected platform: %r', platform)
|
4433
|
+
return platform
|
4434
|
+
|
4435
|
+
|
4219
4436
|
########################################
|
4220
4437
|
# ../targets/targets.py
|
4221
4438
|
"""
|
@@ -5540,1150 +5757,994 @@ inj = Injection
|
|
5540
5757
|
|
5541
5758
|
|
5542
5759
|
########################################
|
5543
|
-
# ../../../omlish/lite/
|
5760
|
+
# ../../../omlish/lite/marshal.py
|
5544
5761
|
"""
|
5545
5762
|
TODO:
|
5546
|
-
-
|
5547
|
-
-
|
5763
|
+
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
5764
|
+
- namedtuple
|
5765
|
+
- literals
|
5766
|
+
- newtypes?
|
5548
5767
|
"""
|
5549
5768
|
|
5550
5769
|
|
5551
|
-
|
5770
|
+
##
|
5552
5771
|
|
5553
5772
|
|
5554
|
-
|
5773
|
+
@dc.dataclass(frozen=True)
|
5774
|
+
class ObjMarshalOptions:
|
5775
|
+
raw_bytes: bool = False
|
5776
|
+
nonstrict_dataclasses: bool = False
|
5555
5777
|
|
5556
5778
|
|
5557
|
-
class
|
5779
|
+
class ObjMarshaler(abc.ABC):
|
5780
|
+
@abc.abstractmethod
|
5781
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5782
|
+
raise NotImplementedError
|
5558
5783
|
|
5559
|
-
|
5560
|
-
|
5561
|
-
|
5784
|
+
@abc.abstractmethod
|
5785
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5786
|
+
raise NotImplementedError
|
5562
5787
|
|
5563
5788
|
|
5564
|
-
|
5789
|
+
class NopObjMarshaler(ObjMarshaler):
|
5790
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5791
|
+
return o
|
5565
5792
|
|
5793
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5794
|
+
return o
|
5566
5795
|
|
5567
|
-
class JsonLogFormatter(logging.Formatter):
|
5568
5796
|
|
5569
|
-
|
5570
|
-
|
5571
|
-
|
5572
|
-
'args': False,
|
5573
|
-
'levelname': False,
|
5574
|
-
'levelno': False,
|
5575
|
-
'pathname': False,
|
5576
|
-
'filename': False,
|
5577
|
-
'module': False,
|
5578
|
-
'exc_info': True,
|
5579
|
-
'exc_text': True,
|
5580
|
-
'stack_info': True,
|
5581
|
-
'lineno': False,
|
5582
|
-
'funcName': False,
|
5583
|
-
'created': False,
|
5584
|
-
'msecs': False,
|
5585
|
-
'relativeCreated': False,
|
5586
|
-
'thread': False,
|
5587
|
-
'threadName': False,
|
5588
|
-
'processName': False,
|
5589
|
-
'process': False,
|
5590
|
-
}
|
5797
|
+
@dc.dataclass()
|
5798
|
+
class ProxyObjMarshaler(ObjMarshaler):
|
5799
|
+
m: ta.Optional[ObjMarshaler] = None
|
5591
5800
|
|
5592
|
-
def
|
5593
|
-
|
5594
|
-
k: v
|
5595
|
-
for k, o in self.KEYS.items()
|
5596
|
-
for v in [getattr(record, k)]
|
5597
|
-
if not (o and v is None)
|
5598
|
-
}
|
5599
|
-
return json_dumps_compact(dct)
|
5801
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5802
|
+
return check.not_none(self.m).marshal(o, ctx)
|
5600
5803
|
|
5804
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5805
|
+
return check.not_none(self.m).unmarshal(o, ctx)
|
5601
5806
|
|
5602
|
-
##
|
5603
5807
|
|
5808
|
+
@dc.dataclass(frozen=True)
|
5809
|
+
class CastObjMarshaler(ObjMarshaler):
|
5810
|
+
ty: type
|
5604
5811
|
|
5605
|
-
|
5606
|
-
|
5607
|
-
('process', 'pid=%(process)-6s'),
|
5608
|
-
('thread', 'tid=%(thread)x'),
|
5609
|
-
('levelname', '%(levelname)s'),
|
5610
|
-
('name', '%(name)s'),
|
5611
|
-
('separator', '::'),
|
5612
|
-
('message', '%(message)s'),
|
5613
|
-
]
|
5812
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5813
|
+
return o
|
5614
5814
|
|
5815
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5816
|
+
return self.ty(o)
|
5615
5817
|
|
5616
|
-
class StandardLogFormatter(logging.Formatter):
|
5617
5818
|
|
5618
|
-
|
5619
|
-
def
|
5620
|
-
return
|
5819
|
+
class DynamicObjMarshaler(ObjMarshaler):
|
5820
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5821
|
+
return ctx.manager.marshal_obj(o, opts=ctx.options)
|
5621
5822
|
|
5622
|
-
|
5823
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5824
|
+
return o
|
5623
5825
|
|
5624
|
-
def formatTime(self, record, datefmt=None):
|
5625
|
-
ct = self.converter(record.created) # type: ignore
|
5626
|
-
if datefmt:
|
5627
|
-
return ct.strftime(datefmt) # noqa
|
5628
|
-
else:
|
5629
|
-
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
5630
|
-
return '%s.%03d' % (t, record.msecs) # noqa
|
5631
5826
|
|
5827
|
+
@dc.dataclass(frozen=True)
|
5828
|
+
class Base64ObjMarshaler(ObjMarshaler):
|
5829
|
+
ty: type
|
5632
5830
|
|
5633
|
-
|
5831
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5832
|
+
return base64.b64encode(o).decode('ascii')
|
5634
5833
|
|
5834
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5835
|
+
return self.ty(base64.b64decode(o))
|
5635
5836
|
|
5636
|
-
class ProxyLogFilterer(logging.Filterer):
|
5637
|
-
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
5638
|
-
self._underlying = underlying
|
5639
5837
|
|
5640
|
-
|
5641
|
-
|
5642
|
-
|
5838
|
+
@dc.dataclass(frozen=True)
|
5839
|
+
class BytesSwitchedObjMarshaler(ObjMarshaler):
|
5840
|
+
m: ObjMarshaler
|
5643
5841
|
|
5644
|
-
|
5645
|
-
|
5646
|
-
|
5842
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5843
|
+
if ctx.options.raw_bytes:
|
5844
|
+
return o
|
5845
|
+
return self.m.marshal(o, ctx)
|
5647
5846
|
|
5648
|
-
|
5649
|
-
|
5650
|
-
|
5847
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5848
|
+
if ctx.options.raw_bytes:
|
5849
|
+
return o
|
5850
|
+
return self.m.unmarshal(o, ctx)
|
5651
5851
|
|
5652
|
-
def addFilter(self, filter): # noqa
|
5653
|
-
self._underlying.addFilter(filter)
|
5654
5852
|
|
5655
|
-
|
5656
|
-
|
5853
|
+
@dc.dataclass(frozen=True)
|
5854
|
+
class EnumObjMarshaler(ObjMarshaler):
|
5855
|
+
ty: type
|
5657
5856
|
|
5658
|
-
def
|
5659
|
-
return
|
5857
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5858
|
+
return o.name
|
5660
5859
|
|
5860
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5861
|
+
return self.ty.__members__[o] # type: ignore
|
5661
5862
|
|
5662
|
-
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
5663
|
-
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
5664
|
-
ProxyLogFilterer.__init__(self, underlying)
|
5665
5863
|
|
5666
|
-
|
5864
|
+
@dc.dataclass(frozen=True)
|
5865
|
+
class OptionalObjMarshaler(ObjMarshaler):
|
5866
|
+
item: ObjMarshaler
|
5667
5867
|
|
5668
|
-
|
5669
|
-
|
5670
|
-
|
5868
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5869
|
+
if o is None:
|
5870
|
+
return None
|
5871
|
+
return self.item.marshal(o, ctx)
|
5671
5872
|
|
5672
|
-
def
|
5673
|
-
|
5873
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5874
|
+
if o is None:
|
5875
|
+
return None
|
5876
|
+
return self.item.unmarshal(o, ctx)
|
5674
5877
|
|
5675
|
-
def set_name(self, name):
|
5676
|
-
self._underlying.set_name(name)
|
5677
5878
|
|
5678
|
-
|
5679
|
-
|
5680
|
-
|
5879
|
+
@dc.dataclass(frozen=True)
|
5880
|
+
class MappingObjMarshaler(ObjMarshaler):
|
5881
|
+
ty: type
|
5882
|
+
km: ObjMarshaler
|
5883
|
+
vm: ObjMarshaler
|
5681
5884
|
|
5682
|
-
|
5683
|
-
|
5684
|
-
return self._underlying.level
|
5885
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5886
|
+
return {self.km.marshal(k, ctx): self.vm.marshal(v, ctx) for k, v in o.items()}
|
5685
5887
|
|
5686
|
-
|
5687
|
-
|
5688
|
-
self._underlying.level = level
|
5888
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5889
|
+
return self.ty((self.km.unmarshal(k, ctx), self.vm.unmarshal(v, ctx)) for k, v in o.items())
|
5689
5890
|
|
5690
|
-
@property
|
5691
|
-
def formatter(self):
|
5692
|
-
return self._underlying.formatter
|
5693
5891
|
|
5694
|
-
|
5695
|
-
|
5696
|
-
|
5892
|
+
@dc.dataclass(frozen=True)
|
5893
|
+
class IterableObjMarshaler(ObjMarshaler):
|
5894
|
+
ty: type
|
5895
|
+
item: ObjMarshaler
|
5697
5896
|
|
5698
|
-
def
|
5699
|
-
self.
|
5897
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5898
|
+
return [self.item.marshal(e, ctx) for e in o]
|
5700
5899
|
|
5701
|
-
def
|
5702
|
-
self.
|
5900
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5901
|
+
return self.ty(self.item.unmarshal(e, ctx) for e in o)
|
5703
5902
|
|
5704
|
-
def release(self):
|
5705
|
-
self._underlying.release()
|
5706
5903
|
|
5707
|
-
|
5708
|
-
|
5904
|
+
@dc.dataclass(frozen=True)
|
5905
|
+
class DataclassObjMarshaler(ObjMarshaler):
|
5906
|
+
ty: type
|
5907
|
+
fs: ta.Mapping[str, ObjMarshaler]
|
5908
|
+
nonstrict: bool = False
|
5709
5909
|
|
5710
|
-
def
|
5711
|
-
return
|
5910
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5911
|
+
return {
|
5912
|
+
k: m.marshal(getattr(o, k), ctx)
|
5913
|
+
for k, m in self.fs.items()
|
5914
|
+
}
|
5712
5915
|
|
5713
|
-
def
|
5714
|
-
self.
|
5916
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5917
|
+
return self.ty(**{
|
5918
|
+
k: self.fs[k].unmarshal(v, ctx)
|
5919
|
+
for k, v in o.items()
|
5920
|
+
if not (self.nonstrict or ctx.options.nonstrict_dataclasses) or k in self.fs
|
5921
|
+
})
|
5715
5922
|
|
5716
|
-
def handle(self, record):
|
5717
|
-
return self._underlying.handle(record)
|
5718
5923
|
|
5719
|
-
|
5720
|
-
|
5924
|
+
@dc.dataclass(frozen=True)
|
5925
|
+
class PolymorphicObjMarshaler(ObjMarshaler):
|
5926
|
+
class Impl(ta.NamedTuple):
|
5927
|
+
ty: type
|
5928
|
+
tag: str
|
5929
|
+
m: ObjMarshaler
|
5721
5930
|
|
5722
|
-
|
5723
|
-
|
5931
|
+
impls_by_ty: ta.Mapping[type, Impl]
|
5932
|
+
impls_by_tag: ta.Mapping[str, Impl]
|
5724
5933
|
|
5725
|
-
|
5726
|
-
|
5934
|
+
@classmethod
|
5935
|
+
def of(cls, impls: ta.Iterable[Impl]) -> 'PolymorphicObjMarshaler':
|
5936
|
+
return cls(
|
5937
|
+
{i.ty: i for i in impls},
|
5938
|
+
{i.tag: i for i in impls},
|
5939
|
+
)
|
5727
5940
|
|
5728
|
-
def
|
5729
|
-
self.
|
5941
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5942
|
+
impl = self.impls_by_ty[type(o)]
|
5943
|
+
return {impl.tag: impl.m.marshal(o, ctx)}
|
5730
5944
|
|
5945
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5946
|
+
[(t, v)] = o.items()
|
5947
|
+
impl = self.impls_by_tag[t]
|
5948
|
+
return impl.m.unmarshal(v, ctx)
|
5731
5949
|
|
5732
|
-
##
|
5733
5950
|
|
5951
|
+
@dc.dataclass(frozen=True)
|
5952
|
+
class DatetimeObjMarshaler(ObjMarshaler):
|
5953
|
+
ty: type
|
5734
5954
|
|
5735
|
-
|
5736
|
-
|
5955
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5956
|
+
return o.isoformat()
|
5737
5957
|
|
5958
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5959
|
+
return self.ty.fromisoformat(o) # type: ignore
|
5738
5960
|
|
5739
|
-
##
|
5740
5961
|
|
5962
|
+
class DecimalObjMarshaler(ObjMarshaler):
|
5963
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5964
|
+
return str(check.isinstance(o, decimal.Decimal))
|
5741
5965
|
|
5742
|
-
|
5743
|
-
|
5744
|
-
if hasattr(logging, '_acquireLock'):
|
5745
|
-
logging._acquireLock() # noqa
|
5746
|
-
try:
|
5747
|
-
yield
|
5748
|
-
finally:
|
5749
|
-
logging._releaseLock() # type: ignore # noqa
|
5966
|
+
def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5967
|
+
return decimal.Decimal(check.isinstance(v, str))
|
5750
5968
|
|
5751
|
-
elif hasattr(logging, '_lock'):
|
5752
|
-
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
5753
|
-
with logging._lock: # noqa
|
5754
|
-
yield
|
5755
5969
|
|
5756
|
-
|
5757
|
-
|
5970
|
+
class FractionObjMarshaler(ObjMarshaler):
|
5971
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5972
|
+
fr = check.isinstance(o, fractions.Fraction)
|
5973
|
+
return [fr.numerator, fr.denominator]
|
5758
5974
|
|
5975
|
+
def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5976
|
+
num, denom = check.isinstance(v, list)
|
5977
|
+
return fractions.Fraction(num, denom)
|
5759
5978
|
|
5760
|
-
def configure_standard_logging(
|
5761
|
-
level: ta.Union[int, str] = logging.INFO,
|
5762
|
-
*,
|
5763
|
-
json: bool = False,
|
5764
|
-
target: ta.Optional[logging.Logger] = None,
|
5765
|
-
force: bool = False,
|
5766
|
-
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
5767
|
-
) -> ta.Optional[StandardLogHandler]:
|
5768
|
-
with _locking_logging_module_lock():
|
5769
|
-
if target is None:
|
5770
|
-
target = logging.root
|
5771
5979
|
|
5772
|
-
|
5980
|
+
class UuidObjMarshaler(ObjMarshaler):
|
5981
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5982
|
+
return str(o)
|
5773
5983
|
|
5774
|
-
|
5775
|
-
|
5776
|
-
return None
|
5984
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5985
|
+
return uuid.UUID(o)
|
5777
5986
|
|
5778
|
-
#
|
5779
5987
|
|
5780
|
-
|
5781
|
-
handler = handler_factory()
|
5782
|
-
else:
|
5783
|
-
handler = logging.StreamHandler()
|
5988
|
+
##
|
5784
5989
|
|
5785
|
-
#
|
5786
5990
|
|
5787
|
-
|
5788
|
-
|
5789
|
-
|
5790
|
-
|
5791
|
-
|
5792
|
-
|
5991
|
+
_DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
|
5992
|
+
**{t: NopObjMarshaler() for t in (type(None),)},
|
5993
|
+
**{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
|
5994
|
+
**{t: BytesSwitchedObjMarshaler(Base64ObjMarshaler(t)) for t in (bytes, bytearray)},
|
5995
|
+
**{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
|
5996
|
+
**{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
|
5793
5997
|
|
5794
|
-
|
5998
|
+
ta.Any: DynamicObjMarshaler(),
|
5795
5999
|
|
5796
|
-
|
6000
|
+
**{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
|
6001
|
+
decimal.Decimal: DecimalObjMarshaler(),
|
6002
|
+
fractions.Fraction: FractionObjMarshaler(),
|
6003
|
+
uuid.UUID: UuidObjMarshaler(),
|
6004
|
+
}
|
5797
6005
|
|
5798
|
-
|
6006
|
+
_OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
|
6007
|
+
**{t: t for t in (dict,)},
|
6008
|
+
**{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
|
6009
|
+
}
|
5799
6010
|
|
5800
|
-
|
6011
|
+
_OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
|
6012
|
+
**{t: t for t in (list, tuple, set, frozenset)},
|
6013
|
+
collections.abc.Set: frozenset,
|
6014
|
+
collections.abc.MutableSet: set,
|
6015
|
+
collections.abc.Sequence: tuple,
|
6016
|
+
collections.abc.MutableSequence: list,
|
6017
|
+
}
|
5801
6018
|
|
5802
|
-
#
|
5803
6019
|
|
5804
|
-
|
5805
|
-
target.setLevel(level)
|
6020
|
+
##
|
5806
6021
|
|
5807
|
-
#
|
5808
6022
|
|
5809
|
-
|
6023
|
+
class ObjMarshalerManager:
|
6024
|
+
def __init__(
|
6025
|
+
self,
|
6026
|
+
*,
|
6027
|
+
default_options: ObjMarshalOptions = ObjMarshalOptions(),
|
5810
6028
|
|
6029
|
+
default_obj_marshalers: ta.Dict[ta.Any, ObjMarshaler] = _DEFAULT_OBJ_MARSHALERS, # noqa
|
6030
|
+
generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES, # noqa
|
6031
|
+
generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES, # noqa
|
6032
|
+
) -> None:
|
6033
|
+
super().__init__()
|
5811
6034
|
|
5812
|
-
|
5813
|
-
# ../../../omlish/lite/marshal.py
|
5814
|
-
"""
|
5815
|
-
TODO:
|
5816
|
-
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
5817
|
-
- namedtuple
|
5818
|
-
- literals
|
5819
|
-
- newtypes?
|
5820
|
-
"""
|
6035
|
+
self._default_options = default_options
|
5821
6036
|
|
6037
|
+
self._obj_marshalers = dict(default_obj_marshalers)
|
6038
|
+
self._generic_mapping_types = generic_mapping_types
|
6039
|
+
self._generic_iterable_types = generic_iterable_types
|
5822
6040
|
|
5823
|
-
|
6041
|
+
self._lock = threading.RLock()
|
6042
|
+
self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
|
6043
|
+
self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
|
5824
6044
|
|
6045
|
+
#
|
5825
6046
|
|
5826
|
-
|
5827
|
-
|
5828
|
-
|
5829
|
-
|
6047
|
+
def make_obj_marshaler(
|
6048
|
+
self,
|
6049
|
+
ty: ta.Any,
|
6050
|
+
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
6051
|
+
*,
|
6052
|
+
nonstrict_dataclasses: bool = False,
|
6053
|
+
) -> ObjMarshaler:
|
6054
|
+
if isinstance(ty, type):
|
6055
|
+
if abc.ABC in ty.__bases__:
|
6056
|
+
impls = [ity for ity in deep_subclasses(ty) if abc.ABC not in ity.__bases__] # type: ignore
|
6057
|
+
if all(ity.__qualname__.endswith(ty.__name__) for ity in impls):
|
6058
|
+
ins = {ity: snake_case(ity.__qualname__[:-len(ty.__name__)]) for ity in impls}
|
6059
|
+
else:
|
6060
|
+
ins = {ity: ity.__qualname__ for ity in impls}
|
6061
|
+
return PolymorphicObjMarshaler.of([
|
6062
|
+
PolymorphicObjMarshaler.Impl(
|
6063
|
+
ity,
|
6064
|
+
itn,
|
6065
|
+
rec(ity),
|
6066
|
+
)
|
6067
|
+
for ity, itn in ins.items()
|
6068
|
+
])
|
5830
6069
|
|
6070
|
+
if issubclass(ty, enum.Enum):
|
6071
|
+
return EnumObjMarshaler(ty)
|
5831
6072
|
|
5832
|
-
|
5833
|
-
|
5834
|
-
|
5835
|
-
|
6073
|
+
if dc.is_dataclass(ty):
|
6074
|
+
return DataclassObjMarshaler(
|
6075
|
+
ty,
|
6076
|
+
{f.name: rec(f.type) for f in dc.fields(ty)},
|
6077
|
+
nonstrict=nonstrict_dataclasses,
|
6078
|
+
)
|
5836
6079
|
|
5837
|
-
|
5838
|
-
|
5839
|
-
|
6080
|
+
if is_generic_alias(ty):
|
6081
|
+
try:
|
6082
|
+
mt = self._generic_mapping_types[ta.get_origin(ty)]
|
6083
|
+
except KeyError:
|
6084
|
+
pass
|
6085
|
+
else:
|
6086
|
+
k, v = ta.get_args(ty)
|
6087
|
+
return MappingObjMarshaler(mt, rec(k), rec(v))
|
5840
6088
|
|
6089
|
+
try:
|
6090
|
+
st = self._generic_iterable_types[ta.get_origin(ty)]
|
6091
|
+
except KeyError:
|
6092
|
+
pass
|
6093
|
+
else:
|
6094
|
+
[e] = ta.get_args(ty)
|
6095
|
+
return IterableObjMarshaler(st, rec(e))
|
5841
6096
|
|
5842
|
-
|
5843
|
-
|
5844
|
-
return o
|
6097
|
+
if is_union_alias(ty):
|
6098
|
+
return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
|
5845
6099
|
|
5846
|
-
|
5847
|
-
return o
|
6100
|
+
raise TypeError(ty)
|
5848
6101
|
|
6102
|
+
#
|
5849
6103
|
|
5850
|
-
|
5851
|
-
|
5852
|
-
|
6104
|
+
def register_opj_marshaler(self, ty: ta.Any, m: ObjMarshaler) -> None:
|
6105
|
+
with self._lock:
|
6106
|
+
if ty in self._obj_marshalers:
|
6107
|
+
raise KeyError(ty)
|
6108
|
+
self._obj_marshalers[ty] = m
|
5853
6109
|
|
5854
|
-
def
|
5855
|
-
|
6110
|
+
def get_obj_marshaler(
|
6111
|
+
self,
|
6112
|
+
ty: ta.Any,
|
6113
|
+
*,
|
6114
|
+
no_cache: bool = False,
|
6115
|
+
**kwargs: ta.Any,
|
6116
|
+
) -> ObjMarshaler:
|
6117
|
+
with self._lock:
|
6118
|
+
if not no_cache:
|
6119
|
+
try:
|
6120
|
+
return self._obj_marshalers[ty]
|
6121
|
+
except KeyError:
|
6122
|
+
pass
|
5856
6123
|
|
5857
|
-
|
5858
|
-
|
6124
|
+
try:
|
6125
|
+
return self._proxies[ty]
|
6126
|
+
except KeyError:
|
6127
|
+
pass
|
5859
6128
|
|
6129
|
+
rec = functools.partial(
|
6130
|
+
self.get_obj_marshaler,
|
6131
|
+
no_cache=no_cache,
|
6132
|
+
**kwargs,
|
6133
|
+
)
|
5860
6134
|
|
5861
|
-
|
5862
|
-
|
5863
|
-
|
6135
|
+
p = ProxyObjMarshaler()
|
6136
|
+
self._proxies[ty] = p
|
6137
|
+
try:
|
6138
|
+
m = self.make_obj_marshaler(ty, rec, **kwargs)
|
6139
|
+
finally:
|
6140
|
+
del self._proxies[ty]
|
6141
|
+
p.m = m
|
5864
6142
|
|
5865
|
-
|
5866
|
-
|
6143
|
+
if not no_cache:
|
6144
|
+
self._obj_marshalers[ty] = m
|
6145
|
+
return m
|
5867
6146
|
|
5868
|
-
|
5869
|
-
return self.ty(o)
|
6147
|
+
#
|
5870
6148
|
|
6149
|
+
def _make_context(self, opts: ta.Optional[ObjMarshalOptions]) -> 'ObjMarshalContext':
|
6150
|
+
return ObjMarshalContext(
|
6151
|
+
options=opts or self._default_options,
|
6152
|
+
manager=self,
|
6153
|
+
)
|
5871
6154
|
|
5872
|
-
|
5873
|
-
|
5874
|
-
|
6155
|
+
def marshal_obj(
|
6156
|
+
self,
|
6157
|
+
o: ta.Any,
|
6158
|
+
ty: ta.Any = None,
|
6159
|
+
opts: ta.Optional[ObjMarshalOptions] = None,
|
6160
|
+
) -> ta.Any:
|
6161
|
+
m = self.get_obj_marshaler(ty if ty is not None else type(o))
|
6162
|
+
return m.marshal(o, self._make_context(opts))
|
5875
6163
|
|
5876
|
-
def
|
5877
|
-
|
6164
|
+
def unmarshal_obj(
|
6165
|
+
self,
|
6166
|
+
o: ta.Any,
|
6167
|
+
ty: ta.Union[ta.Type[T], ta.Any],
|
6168
|
+
opts: ta.Optional[ObjMarshalOptions] = None,
|
6169
|
+
) -> T:
|
6170
|
+
m = self.get_obj_marshaler(ty)
|
6171
|
+
return m.unmarshal(o, self._make_context(opts))
|
6172
|
+
|
6173
|
+
def roundtrip_obj(
|
6174
|
+
self,
|
6175
|
+
o: ta.Any,
|
6176
|
+
ty: ta.Any = None,
|
6177
|
+
opts: ta.Optional[ObjMarshalOptions] = None,
|
6178
|
+
) -> ta.Any:
|
6179
|
+
if ty is None:
|
6180
|
+
ty = type(o)
|
6181
|
+
m: ta.Any = self.marshal_obj(o, ty, opts)
|
6182
|
+
u: ta.Any = self.unmarshal_obj(m, ty, opts)
|
6183
|
+
return u
|
5878
6184
|
|
5879
6185
|
|
5880
6186
|
@dc.dataclass(frozen=True)
|
5881
|
-
class
|
5882
|
-
|
6187
|
+
class ObjMarshalContext:
|
6188
|
+
options: ObjMarshalOptions
|
6189
|
+
manager: ObjMarshalerManager
|
5883
6190
|
|
5884
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5885
|
-
return base64.b64encode(o).decode('ascii')
|
5886
6191
|
|
5887
|
-
|
5888
|
-
return self.ty(base64.b64decode(o))
|
6192
|
+
##
|
5889
6193
|
|
5890
6194
|
|
5891
|
-
|
5892
|
-
class BytesSwitchedObjMarshaler(ObjMarshaler):
|
5893
|
-
m: ObjMarshaler
|
6195
|
+
OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
|
5894
6196
|
|
5895
|
-
|
5896
|
-
|
5897
|
-
return o
|
5898
|
-
return self.m.marshal(o, ctx)
|
6197
|
+
register_opj_marshaler = OBJ_MARSHALER_MANAGER.register_opj_marshaler
|
6198
|
+
get_obj_marshaler = OBJ_MARSHALER_MANAGER.get_obj_marshaler
|
5899
6199
|
|
5900
|
-
|
5901
|
-
|
5902
|
-
return o
|
5903
|
-
return self.m.unmarshal(o, ctx)
|
6200
|
+
marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
|
6201
|
+
unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj
|
5904
6202
|
|
5905
6203
|
|
5906
|
-
|
5907
|
-
|
5908
|
-
ty: type
|
6204
|
+
########################################
|
6205
|
+
# ../../../omlish/lite/runtime.py
|
5909
6206
|
|
5910
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5911
|
-
return o.name
|
5912
6207
|
|
5913
|
-
|
5914
|
-
|
6208
|
+
@cached_nullary
|
6209
|
+
def is_debugger_attached() -> bool:
|
6210
|
+
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
5915
6211
|
|
5916
6212
|
|
5917
|
-
|
5918
|
-
class OptionalObjMarshaler(ObjMarshaler):
|
5919
|
-
item: ObjMarshaler
|
6213
|
+
REQUIRED_PYTHON_VERSION = (3, 8)
|
5920
6214
|
|
5921
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5922
|
-
if o is None:
|
5923
|
-
return None
|
5924
|
-
return self.item.marshal(o, ctx)
|
5925
6215
|
|
5926
|
-
|
5927
|
-
|
5928
|
-
|
5929
|
-
return self.item.unmarshal(o, ctx)
|
6216
|
+
def check_runtime_version() -> None:
|
6217
|
+
if sys.version_info < REQUIRED_PYTHON_VERSION:
|
6218
|
+
raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
|
5930
6219
|
|
5931
6220
|
|
5932
|
-
|
5933
|
-
|
5934
|
-
|
5935
|
-
|
5936
|
-
|
6221
|
+
########################################
|
6222
|
+
# ../../../omlish/logs/json.py
|
6223
|
+
"""
|
6224
|
+
TODO:
|
6225
|
+
- translate json keys
|
6226
|
+
"""
|
5937
6227
|
|
5938
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5939
|
-
return {self.km.marshal(k, ctx): self.vm.marshal(v, ctx) for k, v in o.items()}
|
5940
6228
|
|
5941
|
-
|
5942
|
-
|
6229
|
+
class JsonLogFormatter(logging.Formatter):
|
6230
|
+
KEYS: ta.Mapping[str, bool] = {
|
6231
|
+
'name': False,
|
6232
|
+
'msg': False,
|
6233
|
+
'args': False,
|
6234
|
+
'levelname': False,
|
6235
|
+
'levelno': False,
|
6236
|
+
'pathname': False,
|
6237
|
+
'filename': False,
|
6238
|
+
'module': False,
|
6239
|
+
'exc_info': True,
|
6240
|
+
'exc_text': True,
|
6241
|
+
'stack_info': True,
|
6242
|
+
'lineno': False,
|
6243
|
+
'funcName': False,
|
6244
|
+
'created': False,
|
6245
|
+
'msecs': False,
|
6246
|
+
'relativeCreated': False,
|
6247
|
+
'thread': False,
|
6248
|
+
'threadName': False,
|
6249
|
+
'processName': False,
|
6250
|
+
'process': False,
|
6251
|
+
}
|
5943
6252
|
|
6253
|
+
def __init__(
|
6254
|
+
self,
|
6255
|
+
*args: ta.Any,
|
6256
|
+
json_dumps: ta.Optional[ta.Callable[[ta.Any], str]] = None,
|
6257
|
+
**kwargs: ta.Any,
|
6258
|
+
) -> None:
|
6259
|
+
super().__init__(*args, **kwargs)
|
5944
6260
|
|
5945
|
-
|
5946
|
-
|
5947
|
-
|
5948
|
-
item: ObjMarshaler
|
6261
|
+
if json_dumps is None:
|
6262
|
+
json_dumps = json_dumps_compact
|
6263
|
+
self._json_dumps = json_dumps
|
5949
6264
|
|
5950
|
-
def
|
5951
|
-
|
6265
|
+
def format(self, record: logging.LogRecord) -> str:
|
6266
|
+
dct = {
|
6267
|
+
k: v
|
6268
|
+
for k, o in self.KEYS.items()
|
6269
|
+
for v in [getattr(record, k)]
|
6270
|
+
if not (o and v is None)
|
6271
|
+
}
|
6272
|
+
return self._json_dumps(dct)
|
5952
6273
|
|
5953
|
-
|
5954
|
-
|
6274
|
+
|
6275
|
+
########################################
|
6276
|
+
# ../../../omdev/interp/types.py
|
6277
|
+
|
6278
|
+
|
6279
|
+
# See https://peps.python.org/pep-3149/
|
6280
|
+
INTERP_OPT_GLYPHS_BY_ATTR: ta.Mapping[str, str] = collections.OrderedDict([
|
6281
|
+
('debug', 'd'),
|
6282
|
+
('threaded', 't'),
|
6283
|
+
])
|
6284
|
+
|
6285
|
+
INTERP_OPT_ATTRS_BY_GLYPH: ta.Mapping[str, str] = collections.OrderedDict(
|
6286
|
+
(g, a) for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items()
|
6287
|
+
)
|
5955
6288
|
|
5956
6289
|
|
5957
6290
|
@dc.dataclass(frozen=True)
|
5958
|
-
class
|
5959
|
-
|
5960
|
-
|
5961
|
-
nonstrict: bool = False
|
6291
|
+
class InterpOpts:
|
6292
|
+
threaded: bool = False
|
6293
|
+
debug: bool = False
|
5962
6294
|
|
5963
|
-
def
|
5964
|
-
return
|
5965
|
-
k: m.marshal(getattr(o, k), ctx)
|
5966
|
-
for k, m in self.fs.items()
|
5967
|
-
}
|
6295
|
+
def __str__(self) -> str:
|
6296
|
+
return ''.join(g for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items() if getattr(self, a))
|
5968
6297
|
|
5969
|
-
|
5970
|
-
|
5971
|
-
|
5972
|
-
|
5973
|
-
|
5974
|
-
|
6298
|
+
@classmethod
|
6299
|
+
def parse(cls, s: str) -> 'InterpOpts':
|
6300
|
+
return cls(**{INTERP_OPT_ATTRS_BY_GLYPH[g]: True for g in s})
|
6301
|
+
|
6302
|
+
@classmethod
|
6303
|
+
def parse_suffix(cls, s: str) -> ta.Tuple[str, 'InterpOpts']:
|
6304
|
+
kw = {}
|
6305
|
+
while s and (a := INTERP_OPT_ATTRS_BY_GLYPH.get(s[-1])):
|
6306
|
+
s, kw[a] = s[:-1], True
|
6307
|
+
return s, cls(**kw)
|
5975
6308
|
|
5976
6309
|
|
5977
6310
|
@dc.dataclass(frozen=True)
|
5978
|
-
class
|
5979
|
-
|
5980
|
-
|
5981
|
-
tag: str
|
5982
|
-
m: ObjMarshaler
|
6311
|
+
class InterpVersion:
|
6312
|
+
version: Version
|
6313
|
+
opts: InterpOpts
|
5983
6314
|
|
5984
|
-
|
5985
|
-
|
6315
|
+
def __str__(self) -> str:
|
6316
|
+
return str(self.version) + str(self.opts)
|
5986
6317
|
|
5987
6318
|
@classmethod
|
5988
|
-
def
|
6319
|
+
def parse(cls, s: str) -> 'InterpVersion':
|
6320
|
+
s, o = InterpOpts.parse_suffix(s)
|
6321
|
+
v = Version(s)
|
5989
6322
|
return cls(
|
5990
|
-
|
5991
|
-
|
6323
|
+
version=v,
|
6324
|
+
opts=o,
|
5992
6325
|
)
|
5993
6326
|
|
5994
|
-
|
5995
|
-
|
5996
|
-
|
5997
|
-
|
5998
|
-
|
5999
|
-
|
6000
|
-
impl = self.impls_by_tag[t]
|
6001
|
-
return impl.m.unmarshal(v, ctx)
|
6327
|
+
@classmethod
|
6328
|
+
def try_parse(cls, s: str) -> ta.Optional['InterpVersion']:
|
6329
|
+
try:
|
6330
|
+
return cls.parse(s)
|
6331
|
+
except (KeyError, InvalidVersion):
|
6332
|
+
return None
|
6002
6333
|
|
6003
6334
|
|
6004
6335
|
@dc.dataclass(frozen=True)
|
6005
|
-
class
|
6006
|
-
|
6336
|
+
class InterpSpecifier:
|
6337
|
+
specifier: Specifier
|
6338
|
+
opts: InterpOpts
|
6007
6339
|
|
6008
|
-
def
|
6009
|
-
return
|
6340
|
+
def __str__(self) -> str:
|
6341
|
+
return str(self.specifier) + str(self.opts)
|
6010
6342
|
|
6011
|
-
|
6012
|
-
|
6343
|
+
@classmethod
|
6344
|
+
def parse(cls, s: str) -> 'InterpSpecifier':
|
6345
|
+
s, o = InterpOpts.parse_suffix(s)
|
6346
|
+
if not any(s.startswith(o) for o in Specifier.OPERATORS):
|
6347
|
+
s = '~=' + s
|
6348
|
+
if s.count('.') < 2:
|
6349
|
+
s += '.0'
|
6350
|
+
return cls(
|
6351
|
+
specifier=Specifier(s),
|
6352
|
+
opts=o,
|
6353
|
+
)
|
6013
6354
|
|
6355
|
+
def contains(self, iv: InterpVersion) -> bool:
|
6356
|
+
return self.specifier.contains(iv.version) and self.opts == iv.opts
|
6014
6357
|
|
6015
|
-
|
6016
|
-
|
6017
|
-
return str(check.isinstance(o, decimal.Decimal))
|
6358
|
+
def __contains__(self, iv: InterpVersion) -> bool:
|
6359
|
+
return self.contains(iv)
|
6018
6360
|
|
6019
|
-
def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
6020
|
-
return decimal.Decimal(check.isinstance(v, str))
|
6021
6361
|
|
6362
|
+
@dc.dataclass(frozen=True)
|
6363
|
+
class Interp:
|
6364
|
+
exe: str
|
6365
|
+
version: InterpVersion
|
6022
6366
|
|
6023
|
-
class FractionObjMarshaler(ObjMarshaler):
|
6024
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
6025
|
-
fr = check.isinstance(o, fractions.Fraction)
|
6026
|
-
return [fr.numerator, fr.denominator]
|
6027
6367
|
|
6028
|
-
|
6029
|
-
|
6030
|
-
return fractions.Fraction(num, denom)
|
6368
|
+
########################################
|
6369
|
+
# ../../configs.py
|
6031
6370
|
|
6032
6371
|
|
6033
|
-
|
6034
|
-
|
6035
|
-
|
6372
|
+
def parse_config_file(
|
6373
|
+
name: str,
|
6374
|
+
f: ta.TextIO,
|
6375
|
+
) -> ConfigMapping:
|
6376
|
+
if name.endswith('.toml'):
|
6377
|
+
return toml_loads(f.read())
|
6036
6378
|
|
6037
|
-
|
6038
|
-
|
6379
|
+
elif any(name.endswith(e) for e in ('.yml', '.yaml')):
|
6380
|
+
yaml = __import__('yaml')
|
6381
|
+
return yaml.safe_load(f)
|
6039
6382
|
|
6383
|
+
elif name.endswith('.ini'):
|
6384
|
+
import configparser
|
6385
|
+
cp = configparser.ConfigParser()
|
6386
|
+
cp.read_file(f)
|
6387
|
+
config_dct: ta.Dict[str, ta.Any] = {}
|
6388
|
+
for sec in cp.sections():
|
6389
|
+
cd = config_dct
|
6390
|
+
for k in sec.split('.'):
|
6391
|
+
cd = cd.setdefault(k, {})
|
6392
|
+
cd.update(cp.items(sec))
|
6393
|
+
return config_dct
|
6040
6394
|
|
6041
|
-
|
6395
|
+
else:
|
6396
|
+
return json.loads(f.read())
|
6042
6397
|
|
6043
6398
|
|
6044
|
-
|
6045
|
-
|
6046
|
-
|
6047
|
-
|
6048
|
-
|
6049
|
-
|
6050
|
-
|
6051
|
-
|
6399
|
+
def read_config_file(
|
6400
|
+
path: str,
|
6401
|
+
cls: ta.Type[T],
|
6402
|
+
*,
|
6403
|
+
prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
|
6404
|
+
) -> T:
|
6405
|
+
with open(path) as cf:
|
6406
|
+
config_dct = parse_config_file(os.path.basename(path), cf)
|
6052
6407
|
|
6053
|
-
|
6054
|
-
|
6055
|
-
fractions.Fraction: FractionObjMarshaler(),
|
6056
|
-
uuid.UUID: UuidObjMarshaler(),
|
6057
|
-
}
|
6408
|
+
if prepare is not None:
|
6409
|
+
config_dct = prepare(config_dct)
|
6058
6410
|
|
6059
|
-
|
6060
|
-
**{t: t for t in (dict,)},
|
6061
|
-
**{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
|
6062
|
-
}
|
6411
|
+
return unmarshal_obj(config_dct, cls)
|
6063
6412
|
|
6064
|
-
_OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
|
6065
|
-
**{t: t for t in (list, tuple, set, frozenset)},
|
6066
|
-
collections.abc.Set: frozenset,
|
6067
|
-
collections.abc.MutableSet: set,
|
6068
|
-
collections.abc.Sequence: tuple,
|
6069
|
-
collections.abc.MutableSequence: list,
|
6070
|
-
}
|
6071
6413
|
|
6414
|
+
def build_config_named_children(
|
6415
|
+
o: ta.Union[
|
6416
|
+
ta.Sequence[ConfigMapping],
|
6417
|
+
ta.Mapping[str, ConfigMapping],
|
6418
|
+
None,
|
6419
|
+
],
|
6420
|
+
*,
|
6421
|
+
name_key: str = 'name',
|
6422
|
+
) -> ta.Optional[ta.Sequence[ConfigMapping]]:
|
6423
|
+
if o is None:
|
6424
|
+
return None
|
6072
6425
|
|
6073
|
-
|
6426
|
+
lst: ta.List[ConfigMapping] = []
|
6427
|
+
if isinstance(o, ta.Mapping):
|
6428
|
+
for k, v in o.items():
|
6429
|
+
check.isinstance(v, ta.Mapping)
|
6430
|
+
if name_key in v:
|
6431
|
+
n = v[name_key]
|
6432
|
+
if k != n:
|
6433
|
+
raise KeyError(f'Given names do not match: {n} != {k}')
|
6434
|
+
lst.append(v)
|
6435
|
+
else:
|
6436
|
+
lst.append({name_key: k, **v})
|
6074
6437
|
|
6438
|
+
else:
|
6439
|
+
check.not_isinstance(o, str)
|
6440
|
+
lst.extend(o)
|
6075
6441
|
|
6076
|
-
|
6077
|
-
|
6078
|
-
|
6079
|
-
|
6080
|
-
|
6442
|
+
seen = set()
|
6443
|
+
for d in lst:
|
6444
|
+
n = d['name']
|
6445
|
+
if n in d:
|
6446
|
+
raise KeyError(f'Duplicate name: {n}')
|
6447
|
+
seen.add(n)
|
6081
6448
|
|
6082
|
-
|
6083
|
-
generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES, # noqa
|
6084
|
-
generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES, # noqa
|
6085
|
-
) -> None:
|
6086
|
-
super().__init__()
|
6449
|
+
return lst
|
6087
6450
|
|
6088
|
-
self._default_options = default_options
|
6089
6451
|
|
6090
|
-
|
6091
|
-
|
6092
|
-
self._generic_iterable_types = generic_iterable_types
|
6452
|
+
########################################
|
6453
|
+
# ../commands/marshal.py
|
6093
6454
|
|
6094
|
-
self._lock = threading.RLock()
|
6095
|
-
self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
|
6096
|
-
self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
|
6097
6455
|
|
6098
|
-
|
6456
|
+
def install_command_marshaling(
|
6457
|
+
cmds: CommandNameMap,
|
6458
|
+
msh: ObjMarshalerManager,
|
6459
|
+
) -> None:
|
6460
|
+
for fn in [
|
6461
|
+
lambda c: c,
|
6462
|
+
lambda c: c.Output,
|
6463
|
+
]:
|
6464
|
+
msh.register_opj_marshaler(
|
6465
|
+
fn(Command),
|
6466
|
+
PolymorphicObjMarshaler.of([
|
6467
|
+
PolymorphicObjMarshaler.Impl(
|
6468
|
+
fn(cmd),
|
6469
|
+
name,
|
6470
|
+
msh.get_obj_marshaler(fn(cmd)),
|
6471
|
+
)
|
6472
|
+
for name, cmd in cmds.items()
|
6473
|
+
]),
|
6474
|
+
)
|
6099
6475
|
|
6100
|
-
def make_obj_marshaler(
|
6101
|
-
self,
|
6102
|
-
ty: ta.Any,
|
6103
|
-
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
6104
|
-
*,
|
6105
|
-
nonstrict_dataclasses: bool = False,
|
6106
|
-
) -> ObjMarshaler:
|
6107
|
-
if isinstance(ty, type):
|
6108
|
-
if abc.ABC in ty.__bases__:
|
6109
|
-
impls = [ity for ity in deep_subclasses(ty) if abc.ABC not in ity.__bases__] # type: ignore
|
6110
|
-
if all(ity.__qualname__.endswith(ty.__name__) for ity in impls):
|
6111
|
-
ins = {ity: snake_case(ity.__qualname__[:-len(ty.__name__)]) for ity in impls}
|
6112
|
-
else:
|
6113
|
-
ins = {ity: ity.__qualname__ for ity in impls}
|
6114
|
-
return PolymorphicObjMarshaler.of([
|
6115
|
-
PolymorphicObjMarshaler.Impl(
|
6116
|
-
ity,
|
6117
|
-
itn,
|
6118
|
-
rec(ity),
|
6119
|
-
)
|
6120
|
-
for ity, itn in ins.items()
|
6121
|
-
])
|
6122
6476
|
|
6123
|
-
|
6124
|
-
|
6477
|
+
########################################
|
6478
|
+
# ../commands/ping.py
|
6125
6479
|
|
6126
|
-
if dc.is_dataclass(ty):
|
6127
|
-
return DataclassObjMarshaler(
|
6128
|
-
ty,
|
6129
|
-
{f.name: rec(f.type) for f in dc.fields(ty)},
|
6130
|
-
nonstrict=nonstrict_dataclasses,
|
6131
|
-
)
|
6132
6480
|
|
6133
|
-
|
6134
|
-
try:
|
6135
|
-
mt = self._generic_mapping_types[ta.get_origin(ty)]
|
6136
|
-
except KeyError:
|
6137
|
-
pass
|
6138
|
-
else:
|
6139
|
-
k, v = ta.get_args(ty)
|
6140
|
-
return MappingObjMarshaler(mt, rec(k), rec(v))
|
6481
|
+
##
|
6141
6482
|
|
6142
|
-
try:
|
6143
|
-
st = self._generic_iterable_types[ta.get_origin(ty)]
|
6144
|
-
except KeyError:
|
6145
|
-
pass
|
6146
|
-
else:
|
6147
|
-
[e] = ta.get_args(ty)
|
6148
|
-
return IterableObjMarshaler(st, rec(e))
|
6149
6483
|
|
6150
|
-
|
6151
|
-
|
6484
|
+
@dc.dataclass(frozen=True)
|
6485
|
+
class PingCommand(Command['PingCommand.Output']):
|
6486
|
+
time: float = dc.field(default_factory=time.time)
|
6152
6487
|
|
6153
|
-
|
6488
|
+
@dc.dataclass(frozen=True)
|
6489
|
+
class Output(Command.Output):
|
6490
|
+
time: float
|
6154
6491
|
|
6155
|
-
#
|
6156
6492
|
|
6157
|
-
|
6158
|
-
|
6159
|
-
|
6160
|
-
raise KeyError(ty)
|
6161
|
-
self._obj_marshalers[ty] = m
|
6493
|
+
class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
|
6494
|
+
async def execute(self, cmd: PingCommand) -> PingCommand.Output:
|
6495
|
+
return PingCommand.Output(cmd.time)
|
6162
6496
|
|
6163
|
-
def get_obj_marshaler(
|
6164
|
-
self,
|
6165
|
-
ty: ta.Any,
|
6166
|
-
*,
|
6167
|
-
no_cache: bool = False,
|
6168
|
-
**kwargs: ta.Any,
|
6169
|
-
) -> ObjMarshaler:
|
6170
|
-
with self._lock:
|
6171
|
-
if not no_cache:
|
6172
|
-
try:
|
6173
|
-
return self._obj_marshalers[ty]
|
6174
|
-
except KeyError:
|
6175
|
-
pass
|
6176
6497
|
|
6177
|
-
|
6178
|
-
|
6179
|
-
except KeyError:
|
6180
|
-
pass
|
6498
|
+
########################################
|
6499
|
+
# ../commands/types.py
|
6181
6500
|
|
6182
|
-
rec = functools.partial(
|
6183
|
-
self.get_obj_marshaler,
|
6184
|
-
no_cache=no_cache,
|
6185
|
-
**kwargs,
|
6186
|
-
)
|
6187
6501
|
|
6188
|
-
|
6189
|
-
self._proxies[ty] = p
|
6190
|
-
try:
|
6191
|
-
m = self.make_obj_marshaler(ty, rec, **kwargs)
|
6192
|
-
finally:
|
6193
|
-
del self._proxies[ty]
|
6194
|
-
p.m = m
|
6502
|
+
CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
|
6195
6503
|
|
6196
|
-
if not no_cache:
|
6197
|
-
self._obj_marshalers[ty] = m
|
6198
|
-
return m
|
6199
6504
|
|
6200
|
-
|
6505
|
+
########################################
|
6506
|
+
# ../deploy/commands.py
|
6201
6507
|
|
6202
|
-
def _make_context(self, opts: ta.Optional[ObjMarshalOptions]) -> 'ObjMarshalContext':
|
6203
|
-
return ObjMarshalContext(
|
6204
|
-
options=opts or self._default_options,
|
6205
|
-
manager=self,
|
6206
|
-
)
|
6207
6508
|
|
6208
|
-
|
6209
|
-
self,
|
6210
|
-
o: ta.Any,
|
6211
|
-
ty: ta.Any = None,
|
6212
|
-
opts: ta.Optional[ObjMarshalOptions] = None,
|
6213
|
-
) -> ta.Any:
|
6214
|
-
m = self.get_obj_marshaler(ty if ty is not None else type(o))
|
6215
|
-
return m.marshal(o, self._make_context(opts))
|
6509
|
+
##
|
6216
6510
|
|
6217
|
-
def unmarshal_obj(
|
6218
|
-
self,
|
6219
|
-
o: ta.Any,
|
6220
|
-
ty: ta.Union[ta.Type[T], ta.Any],
|
6221
|
-
opts: ta.Optional[ObjMarshalOptions] = None,
|
6222
|
-
) -> T:
|
6223
|
-
m = self.get_obj_marshaler(ty)
|
6224
|
-
return m.unmarshal(o, self._make_context(opts))
|
6225
6511
|
|
6226
|
-
|
6227
|
-
|
6228
|
-
|
6229
|
-
|
6230
|
-
|
6231
|
-
) -> ta.Any:
|
6232
|
-
if ty is None:
|
6233
|
-
ty = type(o)
|
6234
|
-
m: ta.Any = self.marshal_obj(o, ty, opts)
|
6235
|
-
u: ta.Any = self.unmarshal_obj(m, ty, opts)
|
6236
|
-
return u
|
6512
|
+
@dc.dataclass(frozen=True)
|
6513
|
+
class DeployCommand(Command['DeployCommand.Output']):
|
6514
|
+
@dc.dataclass(frozen=True)
|
6515
|
+
class Output(Command.Output):
|
6516
|
+
pass
|
6237
6517
|
|
6238
6518
|
|
6239
|
-
|
6240
|
-
|
6241
|
-
|
6242
|
-
manager: ObjMarshalerManager
|
6519
|
+
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
6520
|
+
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
6521
|
+
log.info('Deploying!')
|
6243
6522
|
|
6523
|
+
return DeployCommand.Output()
|
6244
6524
|
|
6245
|
-
##
|
6246
6525
|
|
6526
|
+
########################################
|
6527
|
+
# ../marshal.py
|
6247
6528
|
|
6248
|
-
OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
|
6249
6529
|
|
6250
|
-
|
6251
|
-
|
6252
|
-
|
6253
|
-
marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
|
6254
|
-
unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj
|
6530
|
+
@dc.dataclass(frozen=True)
|
6531
|
+
class ObjMarshalerInstaller:
|
6532
|
+
fn: ta.Callable[[ObjMarshalerManager], None]
|
6255
6533
|
|
6256
6534
|
|
6257
|
-
|
6258
|
-
# ../../../omlish/lite/runtime.py
|
6535
|
+
ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMarshalerInstaller])
|
6259
6536
|
|
6260
6537
|
|
6261
|
-
|
6262
|
-
|
6263
|
-
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
6538
|
+
########################################
|
6539
|
+
# ../remote/channel.py
|
6264
6540
|
|
6265
6541
|
|
6266
|
-
|
6542
|
+
##
|
6267
6543
|
|
6268
6544
|
|
6269
|
-
|
6270
|
-
|
6271
|
-
|
6545
|
+
class RemoteChannel(abc.ABC):
|
6546
|
+
@abc.abstractmethod
|
6547
|
+
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
|
6548
|
+
raise NotImplementedError
|
6272
6549
|
|
6550
|
+
@abc.abstractmethod
|
6551
|
+
def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
|
6552
|
+
raise NotImplementedError
|
6273
6553
|
|
6274
|
-
|
6275
|
-
|
6554
|
+
def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
|
6555
|
+
pass
|
6276
6556
|
|
6277
6557
|
|
6278
|
-
|
6279
|
-
INTERP_OPT_GLYPHS_BY_ATTR: ta.Mapping[str, str] = collections.OrderedDict([
|
6280
|
-
('debug', 'd'),
|
6281
|
-
('threaded', 't'),
|
6282
|
-
])
|
6558
|
+
##
|
6283
6559
|
|
6284
|
-
INTERP_OPT_ATTRS_BY_GLYPH: ta.Mapping[str, str] = collections.OrderedDict(
|
6285
|
-
(g, a) for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items()
|
6286
|
-
)
|
6287
6560
|
|
6561
|
+
class RemoteChannelImpl(RemoteChannel):
|
6562
|
+
def __init__(
|
6563
|
+
self,
|
6564
|
+
input: asyncio.StreamReader, # noqa
|
6565
|
+
output: asyncio.StreamWriter,
|
6566
|
+
*,
|
6567
|
+
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
6568
|
+
) -> None:
|
6569
|
+
super().__init__()
|
6288
6570
|
|
6289
|
-
|
6290
|
-
|
6291
|
-
|
6292
|
-
debug: bool = False
|
6571
|
+
self._input = input
|
6572
|
+
self._output = output
|
6573
|
+
self._msh = msh
|
6293
6574
|
|
6294
|
-
|
6295
|
-
|
6575
|
+
self._input_lock = asyncio.Lock()
|
6576
|
+
self._output_lock = asyncio.Lock()
|
6296
6577
|
|
6297
|
-
|
6298
|
-
|
6299
|
-
return cls(**{INTERP_OPT_ATTRS_BY_GLYPH[g]: True for g in s})
|
6578
|
+
def set_marshaler(self, msh: ObjMarshalerManager) -> None:
|
6579
|
+
self._msh = msh
|
6300
6580
|
|
6301
|
-
|
6302
|
-
def parse_suffix(cls, s: str) -> ta.Tuple[str, 'InterpOpts']:
|
6303
|
-
kw = {}
|
6304
|
-
while s and (a := INTERP_OPT_ATTRS_BY_GLYPH.get(s[-1])):
|
6305
|
-
s, kw[a] = s[:-1], True
|
6306
|
-
return s, cls(**kw)
|
6581
|
+
#
|
6307
6582
|
|
6583
|
+
async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
6584
|
+
j = json_dumps_compact(self._msh.marshal_obj(o, ty))
|
6585
|
+
d = j.encode('utf-8')
|
6308
6586
|
|
6309
|
-
|
6310
|
-
|
6311
|
-
|
6312
|
-
opts: InterpOpts
|
6587
|
+
self._output.write(struct.pack('<I', len(d)))
|
6588
|
+
self._output.write(d)
|
6589
|
+
await self._output.drain()
|
6313
6590
|
|
6314
|
-
def
|
6315
|
-
|
6591
|
+
async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
6592
|
+
async with self._output_lock:
|
6593
|
+
return await self._send_obj(o, ty)
|
6316
6594
|
|
6317
|
-
|
6318
|
-
def parse(cls, s: str) -> 'InterpVersion':
|
6319
|
-
s, o = InterpOpts.parse_suffix(s)
|
6320
|
-
v = Version(s)
|
6321
|
-
return cls(
|
6322
|
-
version=v,
|
6323
|
-
opts=o,
|
6324
|
-
)
|
6595
|
+
#
|
6325
6596
|
|
6326
|
-
|
6327
|
-
|
6328
|
-
|
6329
|
-
return cls.parse(s)
|
6330
|
-
except (KeyError, InvalidVersion):
|
6597
|
+
async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
6598
|
+
d = await self._input.read(4)
|
6599
|
+
if not d:
|
6331
6600
|
return None
|
6601
|
+
if len(d) != 4:
|
6602
|
+
raise EOFError
|
6332
6603
|
|
6604
|
+
sz = struct.unpack('<I', d)[0]
|
6605
|
+
d = await self._input.read(sz)
|
6606
|
+
if len(d) != sz:
|
6607
|
+
raise EOFError
|
6333
6608
|
|
6334
|
-
|
6335
|
-
|
6336
|
-
specifier: Specifier
|
6337
|
-
opts: InterpOpts
|
6338
|
-
|
6339
|
-
def __str__(self) -> str:
|
6340
|
-
return str(self.specifier) + str(self.opts)
|
6609
|
+
j = json.loads(d.decode('utf-8'))
|
6610
|
+
return self._msh.unmarshal_obj(j, ty)
|
6341
6611
|
|
6342
|
-
|
6343
|
-
|
6344
|
-
|
6345
|
-
if not any(s.startswith(o) for o in Specifier.OPERATORS):
|
6346
|
-
s = '~=' + s
|
6347
|
-
if s.count('.') < 2:
|
6348
|
-
s += '.0'
|
6349
|
-
return cls(
|
6350
|
-
specifier=Specifier(s),
|
6351
|
-
opts=o,
|
6352
|
-
)
|
6612
|
+
async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
6613
|
+
async with self._input_lock:
|
6614
|
+
return await self._recv_obj(ty)
|
6353
6615
|
|
6354
|
-
def contains(self, iv: InterpVersion) -> bool:
|
6355
|
-
return self.specifier.contains(iv.version) and self.opts == iv.opts
|
6356
6616
|
|
6357
|
-
|
6358
|
-
|
6617
|
+
########################################
|
6618
|
+
# ../system/config.py
|
6359
6619
|
|
6360
6620
|
|
6361
6621
|
@dc.dataclass(frozen=True)
|
6362
|
-
class
|
6363
|
-
|
6364
|
-
version: InterpVersion
|
6622
|
+
class SystemConfig:
|
6623
|
+
platform: ta.Optional[Platform] = None
|
6365
6624
|
|
6366
6625
|
|
6367
6626
|
########################################
|
6368
|
-
#
|
6627
|
+
# ../../../omlish/logs/standard.py
|
6628
|
+
"""
|
6629
|
+
TODO:
|
6630
|
+
- structured
|
6631
|
+
- prefixed
|
6632
|
+
- debug
|
6633
|
+
"""
|
6369
6634
|
|
6370
6635
|
|
6371
|
-
|
6372
|
-
name: str,
|
6373
|
-
f: ta.TextIO,
|
6374
|
-
) -> ConfigMapping:
|
6375
|
-
if name.endswith('.toml'):
|
6376
|
-
return toml_loads(f.read())
|
6636
|
+
##
|
6377
6637
|
|
6378
|
-
elif any(name.endswith(e) for e in ('.yml', '.yaml')):
|
6379
|
-
yaml = __import__('yaml')
|
6380
|
-
return yaml.safe_load(f)
|
6381
6638
|
|
6382
|
-
|
6383
|
-
|
6384
|
-
|
6385
|
-
|
6386
|
-
|
6387
|
-
|
6388
|
-
|
6389
|
-
|
6390
|
-
|
6391
|
-
cd.update(cp.items(sec))
|
6392
|
-
return config_dct
|
6639
|
+
STANDARD_LOG_FORMAT_PARTS = [
|
6640
|
+
('asctime', '%(asctime)-15s'),
|
6641
|
+
('process', 'pid=%(process)-6s'),
|
6642
|
+
('thread', 'tid=%(thread)x'),
|
6643
|
+
('levelname', '%(levelname)s'),
|
6644
|
+
('name', '%(name)s'),
|
6645
|
+
('separator', '::'),
|
6646
|
+
('message', '%(message)s'),
|
6647
|
+
]
|
6393
6648
|
|
6394
|
-
else:
|
6395
|
-
return json.loads(f.read())
|
6396
6649
|
|
6650
|
+
class StandardLogFormatter(logging.Formatter):
|
6651
|
+
@staticmethod
|
6652
|
+
def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
|
6653
|
+
return ' '.join(v for k, v in parts)
|
6397
6654
|
|
6398
|
-
|
6399
|
-
path: str,
|
6400
|
-
cls: ta.Type[T],
|
6401
|
-
*,
|
6402
|
-
prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
|
6403
|
-
) -> T:
|
6404
|
-
with open(path) as cf:
|
6405
|
-
config_dct = parse_config_file(os.path.basename(path), cf)
|
6655
|
+
converter = datetime.datetime.fromtimestamp # type: ignore
|
6406
6656
|
|
6407
|
-
|
6408
|
-
|
6657
|
+
def formatTime(self, record, datefmt=None):
|
6658
|
+
ct = self.converter(record.created) # type: ignore
|
6659
|
+
if datefmt:
|
6660
|
+
return ct.strftime(datefmt) # noqa
|
6661
|
+
else:
|
6662
|
+
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
6663
|
+
return '%s.%03d' % (t, record.msecs) # noqa
|
6409
6664
|
|
6410
|
-
return unmarshal_obj(config_dct, cls)
|
6411
6665
|
|
6666
|
+
##
|
6412
6667
|
|
6413
|
-
def build_config_named_children(
|
6414
|
-
o: ta.Union[
|
6415
|
-
ta.Sequence[ConfigMapping],
|
6416
|
-
ta.Mapping[str, ConfigMapping],
|
6417
|
-
None,
|
6418
|
-
],
|
6419
|
-
*,
|
6420
|
-
name_key: str = 'name',
|
6421
|
-
) -> ta.Optional[ta.Sequence[ConfigMapping]]:
|
6422
|
-
if o is None:
|
6423
|
-
return None
|
6424
6668
|
|
6425
|
-
|
6426
|
-
|
6427
|
-
for k, v in o.items():
|
6428
|
-
check.isinstance(v, ta.Mapping)
|
6429
|
-
if name_key in v:
|
6430
|
-
n = v[name_key]
|
6431
|
-
if k != n:
|
6432
|
-
raise KeyError(f'Given names do not match: {n} != {k}')
|
6433
|
-
lst.append(v)
|
6434
|
-
else:
|
6435
|
-
lst.append({name_key: k, **v})
|
6669
|
+
class StandardLogHandler(ProxyLogHandler):
|
6670
|
+
pass
|
6436
6671
|
|
6437
|
-
else:
|
6438
|
-
check.not_isinstance(o, str)
|
6439
|
-
lst.extend(o)
|
6440
6672
|
|
6441
|
-
|
6442
|
-
for d in lst:
|
6443
|
-
n = d['name']
|
6444
|
-
if n in d:
|
6445
|
-
raise KeyError(f'Duplicate name: {n}')
|
6446
|
-
seen.add(n)
|
6673
|
+
##
|
6447
6674
|
|
6448
|
-
return lst
|
6449
6675
|
|
6676
|
+
@contextlib.contextmanager
|
6677
|
+
def _locking_logging_module_lock() -> ta.Iterator[None]:
|
6678
|
+
if hasattr(logging, '_acquireLock'):
|
6679
|
+
logging._acquireLock() # noqa
|
6680
|
+
try:
|
6681
|
+
yield
|
6682
|
+
finally:
|
6683
|
+
logging._releaseLock() # type: ignore # noqa
|
6450
6684
|
|
6451
|
-
|
6452
|
-
#
|
6685
|
+
elif hasattr(logging, '_lock'):
|
6686
|
+
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
6687
|
+
with logging._lock: # noqa
|
6688
|
+
yield
|
6453
6689
|
|
6454
|
-
|
6455
|
-
|
6456
|
-
cmds: CommandNameMap,
|
6457
|
-
msh: ObjMarshalerManager,
|
6458
|
-
) -> None:
|
6459
|
-
for fn in [
|
6460
|
-
lambda c: c,
|
6461
|
-
lambda c: c.Output,
|
6462
|
-
]:
|
6463
|
-
msh.register_opj_marshaler(
|
6464
|
-
fn(Command),
|
6465
|
-
PolymorphicObjMarshaler.of([
|
6466
|
-
PolymorphicObjMarshaler.Impl(
|
6467
|
-
fn(cmd),
|
6468
|
-
name,
|
6469
|
-
msh.get_obj_marshaler(fn(cmd)),
|
6470
|
-
)
|
6471
|
-
for name, cmd in cmds.items()
|
6472
|
-
]),
|
6473
|
-
)
|
6474
|
-
|
6475
|
-
|
6476
|
-
########################################
|
6477
|
-
# ../commands/ping.py
|
6478
|
-
|
6479
|
-
|
6480
|
-
##
|
6481
|
-
|
6482
|
-
|
6483
|
-
@dc.dataclass(frozen=True)
|
6484
|
-
class PingCommand(Command['PingCommand.Output']):
|
6485
|
-
time: float = dc.field(default_factory=time.time)
|
6486
|
-
|
6487
|
-
@dc.dataclass(frozen=True)
|
6488
|
-
class Output(Command.Output):
|
6489
|
-
time: float
|
6490
|
-
|
6491
|
-
|
6492
|
-
class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
|
6493
|
-
async def execute(self, cmd: PingCommand) -> PingCommand.Output:
|
6494
|
-
return PingCommand.Output(cmd.time)
|
6495
|
-
|
6496
|
-
|
6497
|
-
########################################
|
6498
|
-
# ../commands/types.py
|
6499
|
-
|
6500
|
-
|
6501
|
-
CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
|
6502
|
-
|
6503
|
-
|
6504
|
-
########################################
|
6505
|
-
# ../deploy/commands.py
|
6506
|
-
|
6507
|
-
|
6508
|
-
##
|
6509
|
-
|
6510
|
-
|
6511
|
-
@dc.dataclass(frozen=True)
|
6512
|
-
class DeployCommand(Command['DeployCommand.Output']):
|
6513
|
-
@dc.dataclass(frozen=True)
|
6514
|
-
class Output(Command.Output):
|
6515
|
-
pass
|
6516
|
-
|
6517
|
-
|
6518
|
-
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
6519
|
-
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
6520
|
-
log.info('Deploying!')
|
6521
|
-
|
6522
|
-
return DeployCommand.Output()
|
6523
|
-
|
6524
|
-
|
6525
|
-
########################################
|
6526
|
-
# ../marshal.py
|
6527
|
-
|
6528
|
-
|
6529
|
-
@dc.dataclass(frozen=True)
|
6530
|
-
class ObjMarshalerInstaller:
|
6531
|
-
fn: ta.Callable[[ObjMarshalerManager], None]
|
6532
|
-
|
6533
|
-
|
6534
|
-
ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMarshalerInstaller])
|
6535
|
-
|
6536
|
-
|
6537
|
-
########################################
|
6538
|
-
# ../remote/channel.py
|
6539
|
-
|
6540
|
-
|
6541
|
-
##
|
6542
|
-
|
6543
|
-
|
6544
|
-
class RemoteChannel(abc.ABC):
|
6545
|
-
@abc.abstractmethod
|
6546
|
-
def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
|
6547
|
-
raise NotImplementedError
|
6548
|
-
|
6549
|
-
@abc.abstractmethod
|
6550
|
-
def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
|
6551
|
-
raise NotImplementedError
|
6552
|
-
|
6553
|
-
def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
|
6554
|
-
pass
|
6555
|
-
|
6556
|
-
|
6557
|
-
##
|
6558
|
-
|
6559
|
-
|
6560
|
-
class RemoteChannelImpl(RemoteChannel):
|
6561
|
-
def __init__(
|
6562
|
-
self,
|
6563
|
-
input: asyncio.StreamReader, # noqa
|
6564
|
-
output: asyncio.StreamWriter,
|
6565
|
-
*,
|
6566
|
-
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
6567
|
-
) -> None:
|
6568
|
-
super().__init__()
|
6569
|
-
|
6570
|
-
self._input = input
|
6571
|
-
self._output = output
|
6572
|
-
self._msh = msh
|
6573
|
-
|
6574
|
-
self._input_lock = asyncio.Lock()
|
6575
|
-
self._output_lock = asyncio.Lock()
|
6576
|
-
|
6577
|
-
def set_marshaler(self, msh: ObjMarshalerManager) -> None:
|
6578
|
-
self._msh = msh
|
6579
|
-
|
6580
|
-
#
|
6581
|
-
|
6582
|
-
async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
6583
|
-
j = json_dumps_compact(self._msh.marshal_obj(o, ty))
|
6584
|
-
d = j.encode('utf-8')
|
6585
|
-
|
6586
|
-
self._output.write(struct.pack('<I', len(d)))
|
6587
|
-
self._output.write(d)
|
6588
|
-
await self._output.drain()
|
6589
|
-
|
6590
|
-
async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
|
6591
|
-
async with self._output_lock:
|
6592
|
-
return await self._send_obj(o, ty)
|
6593
|
-
|
6594
|
-
#
|
6595
|
-
|
6596
|
-
async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
6597
|
-
d = await self._input.read(4)
|
6598
|
-
if not d:
|
6599
|
-
return None
|
6600
|
-
if len(d) != 4:
|
6601
|
-
raise EOFError
|
6602
|
-
|
6603
|
-
sz = struct.unpack('<I', d)[0]
|
6604
|
-
d = await self._input.read(sz)
|
6605
|
-
if len(d) != sz:
|
6606
|
-
raise EOFError
|
6607
|
-
|
6608
|
-
j = json.loads(d.decode('utf-8'))
|
6609
|
-
return self._msh.unmarshal_obj(j, ty)
|
6610
|
-
|
6611
|
-
async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
|
6612
|
-
async with self._input_lock:
|
6613
|
-
return await self._recv_obj(ty)
|
6690
|
+
else:
|
6691
|
+
raise Exception("Can't find lock in logging module")
|
6614
6692
|
|
6615
6693
|
|
6616
|
-
|
6617
|
-
|
6618
|
-
|
6619
|
-
|
6620
|
-
|
6621
|
-
|
6622
|
-
|
6623
|
-
|
6624
|
-
|
6625
|
-
|
6626
|
-
|
6627
|
-
|
6628
|
-
class LinuxPlatform(Platform, abc.ABC):
|
6629
|
-
pass
|
6630
|
-
|
6631
|
-
|
6632
|
-
class UbuntuPlatform(LinuxPlatform):
|
6633
|
-
pass
|
6634
|
-
|
6635
|
-
|
6636
|
-
class AmazonLinuxPlatform(LinuxPlatform):
|
6637
|
-
pass
|
6638
|
-
|
6639
|
-
|
6640
|
-
class GenericLinuxPlatform(LinuxPlatform):
|
6641
|
-
pass
|
6642
|
-
|
6643
|
-
|
6644
|
-
class DarwinPlatform(Platform):
|
6645
|
-
pass
|
6694
|
+
def configure_standard_logging(
|
6695
|
+
level: ta.Union[int, str] = logging.INFO,
|
6696
|
+
*,
|
6697
|
+
json: bool = False,
|
6698
|
+
target: ta.Optional[logging.Logger] = None,
|
6699
|
+
force: bool = False,
|
6700
|
+
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
6701
|
+
) -> ta.Optional[StandardLogHandler]:
|
6702
|
+
with _locking_logging_module_lock():
|
6703
|
+
if target is None:
|
6704
|
+
target = logging.root
|
6646
6705
|
|
6706
|
+
#
|
6647
6707
|
|
6648
|
-
|
6649
|
-
|
6708
|
+
if not force:
|
6709
|
+
if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
|
6710
|
+
return None
|
6650
6711
|
|
6712
|
+
#
|
6651
6713
|
|
6652
|
-
|
6714
|
+
if handler_factory is not None:
|
6715
|
+
handler = handler_factory()
|
6716
|
+
else:
|
6717
|
+
handler = logging.StreamHandler()
|
6653
6718
|
|
6719
|
+
#
|
6654
6720
|
|
6655
|
-
|
6656
|
-
|
6721
|
+
formatter: logging.Formatter
|
6722
|
+
if json:
|
6723
|
+
formatter = JsonLogFormatter()
|
6724
|
+
else:
|
6725
|
+
formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
|
6726
|
+
handler.setFormatter(formatter)
|
6657
6727
|
|
6658
|
-
|
6659
|
-
if (osr := LinuxOsRelease.read()) is None:
|
6660
|
-
return GenericLinuxPlatform()
|
6728
|
+
#
|
6661
6729
|
|
6662
|
-
|
6663
|
-
return AmazonLinuxPlatform()
|
6730
|
+
handler.addFilter(TidLogFilter())
|
6664
6731
|
|
6665
|
-
|
6666
|
-
return UbuntuPlatform()
|
6732
|
+
#
|
6667
6733
|
|
6668
|
-
|
6669
|
-
return GenericLinuxPlatform()
|
6734
|
+
target.addHandler(handler)
|
6670
6735
|
|
6671
|
-
|
6672
|
-
return DarwinPlatform()
|
6736
|
+
#
|
6673
6737
|
|
6674
|
-
|
6675
|
-
|
6738
|
+
if level is not None:
|
6739
|
+
target.setLevel(level)
|
6676
6740
|
|
6741
|
+
#
|
6677
6742
|
|
6678
|
-
|
6679
|
-
def detect_system_platform() -> Platform:
|
6680
|
-
platform = _detect_system_platform()
|
6681
|
-
log.info('Detected platform: %r', platform)
|
6682
|
-
return platform
|
6743
|
+
return StandardLogHandler(handler)
|
6683
6744
|
|
6684
6745
|
|
6685
6746
|
########################################
|
6686
|
-
# ../../../omlish/
|
6747
|
+
# ../../../omlish/subprocesses.py
|
6687
6748
|
|
6688
6749
|
|
6689
6750
|
##
|
@@ -6734,8 +6795,8 @@ def subprocess_close(
|
|
6734
6795
|
##
|
6735
6796
|
|
6736
6797
|
|
6737
|
-
class
|
6738
|
-
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] =
|
6798
|
+
class BaseSubprocesses(abc.ABC): # noqa
|
6799
|
+
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
|
6739
6800
|
|
6740
6801
|
def __init__(
|
6741
6802
|
self,
|
@@ -6748,6 +6809,9 @@ class AbstractSubprocesses(abc.ABC): # noqa
|
|
6748
6809
|
self._log = log if log is not None else self.DEFAULT_LOGGER
|
6749
6810
|
self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
|
6750
6811
|
|
6812
|
+
def set_logger(self, log: ta.Optional[logging.Logger]) -> None:
|
6813
|
+
self._log = log
|
6814
|
+
|
6751
6815
|
#
|
6752
6816
|
|
6753
6817
|
def prepare_args(
|
@@ -6859,23 +6923,25 @@ class AbstractSubprocesses(abc.ABC): # noqa
|
|
6859
6923
|
##
|
6860
6924
|
|
6861
6925
|
|
6862
|
-
class
|
6926
|
+
class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
|
6927
|
+
@abc.abstractmethod
|
6863
6928
|
def check_call(
|
6864
6929
|
self,
|
6865
6930
|
*cmd: str,
|
6866
6931
|
stdout: ta.Any = sys.stderr,
|
6867
6932
|
**kwargs: ta.Any,
|
6868
6933
|
) -> None:
|
6869
|
-
|
6870
|
-
subprocess.check_call(cmd, **kwargs)
|
6934
|
+
raise NotImplementedError
|
6871
6935
|
|
6936
|
+
@abc.abstractmethod
|
6872
6937
|
def check_output(
|
6873
6938
|
self,
|
6874
6939
|
*cmd: str,
|
6875
6940
|
**kwargs: ta.Any,
|
6876
6941
|
) -> bytes:
|
6877
|
-
|
6878
|
-
|
6942
|
+
raise NotImplementedError
|
6943
|
+
|
6944
|
+
#
|
6879
6945
|
|
6880
6946
|
def check_output_str(
|
6881
6947
|
self,
|
@@ -6917,9 +6983,109 @@ class Subprocesses(AbstractSubprocesses):
|
|
6917
6983
|
return ret.decode().strip()
|
6918
6984
|
|
6919
6985
|
|
6986
|
+
##
|
6987
|
+
|
6988
|
+
|
6989
|
+
class Subprocesses(AbstractSubprocesses):
|
6990
|
+
def check_call(
|
6991
|
+
self,
|
6992
|
+
*cmd: str,
|
6993
|
+
stdout: ta.Any = sys.stderr,
|
6994
|
+
**kwargs: ta.Any,
|
6995
|
+
) -> None:
|
6996
|
+
with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
|
6997
|
+
subprocess.check_call(cmd, **kwargs)
|
6998
|
+
|
6999
|
+
def check_output(
|
7000
|
+
self,
|
7001
|
+
*cmd: str,
|
7002
|
+
**kwargs: ta.Any,
|
7003
|
+
) -> bytes:
|
7004
|
+
with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
|
7005
|
+
return subprocess.check_output(cmd, **kwargs)
|
7006
|
+
|
7007
|
+
|
6920
7008
|
subprocesses = Subprocesses()
|
6921
7009
|
|
6922
7010
|
|
7011
|
+
##
|
7012
|
+
|
7013
|
+
|
7014
|
+
class AbstractAsyncSubprocesses(BaseSubprocesses):
|
7015
|
+
@abc.abstractmethod
|
7016
|
+
async def check_call(
|
7017
|
+
self,
|
7018
|
+
*cmd: str,
|
7019
|
+
stdout: ta.Any = sys.stderr,
|
7020
|
+
**kwargs: ta.Any,
|
7021
|
+
) -> None:
|
7022
|
+
raise NotImplementedError
|
7023
|
+
|
7024
|
+
@abc.abstractmethod
|
7025
|
+
async def check_output(
|
7026
|
+
self,
|
7027
|
+
*cmd: str,
|
7028
|
+
**kwargs: ta.Any,
|
7029
|
+
) -> bytes:
|
7030
|
+
raise NotImplementedError
|
7031
|
+
|
7032
|
+
#
|
7033
|
+
|
7034
|
+
async def check_output_str(
|
7035
|
+
self,
|
7036
|
+
*cmd: str,
|
7037
|
+
**kwargs: ta.Any,
|
7038
|
+
) -> str:
|
7039
|
+
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
7040
|
+
|
7041
|
+
#
|
7042
|
+
|
7043
|
+
async def try_call(
|
7044
|
+
self,
|
7045
|
+
*cmd: str,
|
7046
|
+
**kwargs: ta.Any,
|
7047
|
+
) -> bool:
|
7048
|
+
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
7049
|
+
return False
|
7050
|
+
else:
|
7051
|
+
return True
|
7052
|
+
|
7053
|
+
async def try_output(
|
7054
|
+
self,
|
7055
|
+
*cmd: str,
|
7056
|
+
**kwargs: ta.Any,
|
7057
|
+
) -> ta.Optional[bytes]:
|
7058
|
+
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
7059
|
+
return None
|
7060
|
+
else:
|
7061
|
+
return ret
|
7062
|
+
|
7063
|
+
async def try_output_str(
|
7064
|
+
self,
|
7065
|
+
*cmd: str,
|
7066
|
+
**kwargs: ta.Any,
|
7067
|
+
) -> ta.Optional[str]:
|
7068
|
+
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
7069
|
+
return None
|
7070
|
+
else:
|
7071
|
+
return ret.decode().strip()
|
7072
|
+
|
7073
|
+
|
7074
|
+
########################################
|
7075
|
+
# ../bootstrap.py
|
7076
|
+
|
7077
|
+
|
7078
|
+
@dc.dataclass(frozen=True)
|
7079
|
+
class MainBootstrap:
|
7080
|
+
main_config: MainConfig = MainConfig()
|
7081
|
+
|
7082
|
+
deploy_config: DeployConfig = DeployConfig()
|
7083
|
+
|
7084
|
+
remote_config: RemoteConfig = RemoteConfig()
|
7085
|
+
|
7086
|
+
system_config: SystemConfig = SystemConfig()
|
7087
|
+
|
7088
|
+
|
6923
7089
|
########################################
|
6924
7090
|
# ../commands/local.py
|
6925
7091
|
|
@@ -7337,16 +7503,7 @@ class RemoteCommandExecutor(CommandExecutor):
|
|
7337
7503
|
|
7338
7504
|
|
7339
7505
|
########################################
|
7340
|
-
#
|
7341
|
-
|
7342
|
-
|
7343
|
-
@dc.dataclass(frozen=True)
|
7344
|
-
class SystemConfig:
|
7345
|
-
platform: ta.Optional[Platform] = None
|
7346
|
-
|
7347
|
-
|
7348
|
-
########################################
|
7349
|
-
# ../../../omlish/lite/asyncio/subprocesses.py
|
7506
|
+
# ../../../omlish/asyncs/asyncio/subprocesses.py
|
7350
7507
|
|
7351
7508
|
|
7352
7509
|
##
|
@@ -7357,6 +7514,8 @@ class AsyncioProcessCommunicator:
|
|
7357
7514
|
self,
|
7358
7515
|
proc: asyncio.subprocess.Process,
|
7359
7516
|
loop: ta.Optional[ta.Any] = None,
|
7517
|
+
*,
|
7518
|
+
log: ta.Optional[logging.Logger] = None,
|
7360
7519
|
) -> None:
|
7361
7520
|
super().__init__()
|
7362
7521
|
|
@@ -7365,6 +7524,7 @@ class AsyncioProcessCommunicator:
|
|
7365
7524
|
|
7366
7525
|
self._proc = proc
|
7367
7526
|
self._loop = loop
|
7527
|
+
self._log = log
|
7368
7528
|
|
7369
7529
|
self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check.isinstance(
|
7370
7530
|
proc._transport, # type: ignore # noqa
|
@@ -7380,19 +7540,19 @@ class AsyncioProcessCommunicator:
|
|
7380
7540
|
try:
|
7381
7541
|
if input is not None:
|
7382
7542
|
stdin.write(input)
|
7383
|
-
if self._debug:
|
7384
|
-
|
7543
|
+
if self._debug and self._log is not None:
|
7544
|
+
self._log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
|
7385
7545
|
|
7386
7546
|
await stdin.drain()
|
7387
7547
|
|
7388
7548
|
except (BrokenPipeError, ConnectionResetError) as exc:
|
7389
7549
|
# communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
|
7390
7550
|
# exceptions.
|
7391
|
-
if self._debug:
|
7392
|
-
|
7551
|
+
if self._debug and self._log is not None:
|
7552
|
+
self._log.debug('%r communicate: stdin got %r', self, exc)
|
7393
7553
|
|
7394
|
-
if self._debug:
|
7395
|
-
|
7554
|
+
if self._debug and self._log is not None:
|
7555
|
+
self._log.debug('%r communicate: close stdin', self)
|
7396
7556
|
|
7397
7557
|
stdin.close()
|
7398
7558
|
|
@@ -7408,15 +7568,15 @@ class AsyncioProcessCommunicator:
|
|
7408
7568
|
check.equal(fd, 1)
|
7409
7569
|
stream = check.not_none(self._proc.stdout)
|
7410
7570
|
|
7411
|
-
if self._debug:
|
7571
|
+
if self._debug and self._log is not None:
|
7412
7572
|
name = 'stdout' if fd == 1 else 'stderr'
|
7413
|
-
|
7573
|
+
self._log.debug('%r communicate: read %s', self, name)
|
7414
7574
|
|
7415
7575
|
output = await stream.read()
|
7416
7576
|
|
7417
|
-
if self._debug:
|
7577
|
+
if self._debug and self._log is not None:
|
7418
7578
|
name = 'stdout' if fd == 1 else 'stderr'
|
7419
|
-
|
7579
|
+
self._log.debug('%r communicate: close %s', self, name)
|
7420
7580
|
|
7421
7581
|
transport.close()
|
7422
7582
|
|
@@ -7465,7 +7625,7 @@ class AsyncioProcessCommunicator:
|
|
7465
7625
|
##
|
7466
7626
|
|
7467
7627
|
|
7468
|
-
class AsyncioSubprocesses(
|
7628
|
+
class AsyncioSubprocesses(AbstractAsyncSubprocesses):
|
7469
7629
|
async def communicate(
|
7470
7630
|
self,
|
7471
7631
|
proc: asyncio.subprocess.Process,
|
@@ -7562,45 +7722,6 @@ class AsyncioSubprocesses(AbstractSubprocesses):
|
|
7562
7722
|
with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs): # noqa
|
7563
7723
|
return check.not_none((await self.run(*cmd, **kwargs)).stdout)
|
7564
7724
|
|
7565
|
-
async def check_output_str(
|
7566
|
-
self,
|
7567
|
-
*cmd: str,
|
7568
|
-
**kwargs: ta.Any,
|
7569
|
-
) -> str:
|
7570
|
-
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
7571
|
-
|
7572
|
-
#
|
7573
|
-
|
7574
|
-
async def try_call(
|
7575
|
-
self,
|
7576
|
-
*cmd: str,
|
7577
|
-
**kwargs: ta.Any,
|
7578
|
-
) -> bool:
|
7579
|
-
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
7580
|
-
return False
|
7581
|
-
else:
|
7582
|
-
return True
|
7583
|
-
|
7584
|
-
async def try_output(
|
7585
|
-
self,
|
7586
|
-
*cmd: str,
|
7587
|
-
**kwargs: ta.Any,
|
7588
|
-
) -> ta.Optional[bytes]:
|
7589
|
-
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
7590
|
-
return None
|
7591
|
-
else:
|
7592
|
-
return ret
|
7593
|
-
|
7594
|
-
async def try_output_str(
|
7595
|
-
self,
|
7596
|
-
*cmd: str,
|
7597
|
-
**kwargs: ta.Any,
|
7598
|
-
) -> ta.Optional[str]:
|
7599
|
-
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
7600
|
-
return None
|
7601
|
-
else:
|
7602
|
-
return ret.decode().strip()
|
7603
|
-
|
7604
7725
|
|
7605
7726
|
asyncio_subprocesses = AsyncioSubprocesses()
|
7606
7727
|
|
@@ -7700,21 +7821,6 @@ class InterpInspector:
|
|
7700
7821
|
INTERP_INSPECTOR = InterpInspector()
|
7701
7822
|
|
7702
7823
|
|
7703
|
-
########################################
|
7704
|
-
# ../bootstrap.py
|
7705
|
-
|
7706
|
-
|
7707
|
-
@dc.dataclass(frozen=True)
|
7708
|
-
class MainBootstrap:
|
7709
|
-
main_config: MainConfig = MainConfig()
|
7710
|
-
|
7711
|
-
deploy_config: DeployConfig = DeployConfig()
|
7712
|
-
|
7713
|
-
remote_config: RemoteConfig = RemoteConfig()
|
7714
|
-
|
7715
|
-
system_config: SystemConfig = SystemConfig()
|
7716
|
-
|
7717
|
-
|
7718
7824
|
########################################
|
7719
7825
|
# ../commands/subprocess.py
|
7720
7826
|
|
@@ -7801,39 +7907,22 @@ github.com/wrmsr/omlish@rev
|
|
7801
7907
|
##
|
7802
7908
|
|
7803
7909
|
|
7804
|
-
@dc.dataclass(frozen=True)
|
7805
|
-
class DeployGitRepo:
|
7806
|
-
host: ta.Optional[str] = None
|
7807
|
-
username: ta.Optional[str] = None
|
7808
|
-
path: ta.Optional[str] = None
|
7809
|
-
|
7810
|
-
def __post_init__(self) -> None:
|
7811
|
-
check.not_in('..', check.non_empty_str(self.host))
|
7812
|
-
check.not_in('.', check.non_empty_str(self.path))
|
7813
|
-
|
7814
|
-
|
7815
|
-
@dc.dataclass(frozen=True)
|
7816
|
-
class DeployGitSpec:
|
7817
|
-
repo: DeployGitRepo
|
7818
|
-
rev: DeployRev
|
7819
|
-
|
7820
|
-
|
7821
|
-
##
|
7822
|
-
|
7823
|
-
|
7824
7910
|
class DeployGitManager(DeployPathOwner):
|
7825
7911
|
def __init__(
|
7826
7912
|
self,
|
7827
7913
|
*,
|
7828
|
-
deploy_home: DeployHome,
|
7914
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
7829
7915
|
) -> None:
|
7830
7916
|
super().__init__()
|
7831
7917
|
|
7832
7918
|
self._deploy_home = deploy_home
|
7833
|
-
self._dir = os.path.join(deploy_home, 'git')
|
7834
7919
|
|
7835
7920
|
self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
|
7836
7921
|
|
7922
|
+
@cached_nullary
|
7923
|
+
def _dir(self) -> str:
|
7924
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'git')
|
7925
|
+
|
7837
7926
|
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
7838
7927
|
return {
|
7839
7928
|
DeployPath.parse('git'),
|
@@ -7850,7 +7939,7 @@ class DeployGitManager(DeployPathOwner):
|
|
7850
7939
|
self._git = git
|
7851
7940
|
self._repo = repo
|
7852
7941
|
self._dir = os.path.join(
|
7853
|
-
self._git._dir, # noqa
|
7942
|
+
self._git._dir(), # noqa
|
7854
7943
|
check.non_empty_str(repo.host),
|
7855
7944
|
check.non_empty_str(repo.path),
|
7856
7945
|
)
|
@@ -7907,8 +7996,8 @@ class DeployGitManager(DeployPathOwner):
|
|
7907
7996
|
repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
|
7908
7997
|
return repo_dir
|
7909
7998
|
|
7910
|
-
async def checkout(self,
|
7911
|
-
await self.get_repo_dir(
|
7999
|
+
async def checkout(self, repo: DeployGitRepo, rev: DeployRev, dst_dir: str) -> None:
|
8000
|
+
await self.get_repo_dir(repo).checkout(rev, dst_dir)
|
7912
8001
|
|
7913
8002
|
|
7914
8003
|
########################################
|
@@ -7924,12 +8013,15 @@ class DeployVenvManager(DeployPathOwner):
|
|
7924
8013
|
def __init__(
|
7925
8014
|
self,
|
7926
8015
|
*,
|
7927
|
-
deploy_home: DeployHome,
|
8016
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
7928
8017
|
) -> None:
|
7929
8018
|
super().__init__()
|
7930
8019
|
|
7931
8020
|
self._deploy_home = deploy_home
|
7932
|
-
|
8021
|
+
|
8022
|
+
@cached_nullary
|
8023
|
+
def _dir(self) -> str:
|
8024
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
|
7933
8025
|
|
7934
8026
|
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
7935
8027
|
return {
|
@@ -7966,66 +8058,208 @@ class DeployVenvManager(DeployPathOwner):
|
|
7966
8058
|
|
7967
8059
|
async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
|
7968
8060
|
await self.setup_venv(
|
7969
|
-
os.path.join(self._deploy_home, 'apps', app_tag.app, app_tag.tag),
|
7970
|
-
os.path.join(self.
|
8061
|
+
os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
|
8062
|
+
os.path.join(self._dir(), app_tag.app, app_tag.tag),
|
7971
8063
|
)
|
7972
8064
|
|
7973
8065
|
|
7974
8066
|
########################################
|
7975
|
-
# ../remote/
|
8067
|
+
# ../remote/_main.py
|
7976
8068
|
|
7977
8069
|
|
7978
8070
|
##
|
7979
8071
|
|
7980
8072
|
|
7981
|
-
class
|
7982
|
-
|
7983
|
-
|
7984
|
-
|
7985
|
-
shell_quote: bool = False
|
8073
|
+
class _RemoteExecutionLogHandler(logging.Handler):
|
8074
|
+
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
8075
|
+
super().__init__()
|
8076
|
+
self._fn = fn
|
7986
8077
|
|
7987
|
-
|
7988
|
-
|
8078
|
+
def emit(self, record):
|
8079
|
+
msg = self.format(record)
|
8080
|
+
self._fn(msg)
|
7989
8081
|
|
7990
|
-
stderr: ta.Optional[str] = None # SubprocessChannelOption
|
7991
8082
|
|
7992
|
-
|
7993
|
-
class Spawned:
|
7994
|
-
stdin: asyncio.StreamWriter
|
7995
|
-
stdout: asyncio.StreamReader
|
7996
|
-
stderr: ta.Optional[asyncio.StreamReader]
|
8083
|
+
##
|
7997
8084
|
|
7998
|
-
|
7999
|
-
|
8085
|
+
|
8086
|
+
class _RemoteExecutionMain:
|
8087
|
+
def __init__(
|
8000
8088
|
self,
|
8001
|
-
|
8002
|
-
|
8003
|
-
|
8004
|
-
timeout: ta.Optional[float] = None,
|
8005
|
-
debug: bool = False,
|
8006
|
-
) -> ta.AsyncContextManager[Spawned]:
|
8007
|
-
raise NotImplementedError
|
8089
|
+
chan: RemoteChannel,
|
8090
|
+
) -> None:
|
8091
|
+
super().__init__()
|
8008
8092
|
|
8093
|
+
self._chan = chan
|
8009
8094
|
|
8010
|
-
|
8095
|
+
self.__bootstrap: ta.Optional[MainBootstrap] = None
|
8096
|
+
self.__injector: ta.Optional[Injector] = None
|
8011
8097
|
|
8098
|
+
@property
|
8099
|
+
def _bootstrap(self) -> MainBootstrap:
|
8100
|
+
return check.not_none(self.__bootstrap)
|
8012
8101
|
|
8013
|
-
|
8014
|
-
|
8015
|
-
|
8016
|
-
shell: bool
|
8102
|
+
@property
|
8103
|
+
def _injector(self) -> Injector:
|
8104
|
+
return check.not_none(self.__injector)
|
8017
8105
|
|
8018
|
-
|
8019
|
-
|
8020
|
-
|
8021
|
-
|
8022
|
-
|
8023
|
-
|
8024
|
-
|
8025
|
-
|
8026
|
-
|
8027
|
-
|
8028
|
-
|
8106
|
+
#
|
8107
|
+
|
8108
|
+
def _timebomb_main(
|
8109
|
+
self,
|
8110
|
+
delay_s: float,
|
8111
|
+
*,
|
8112
|
+
sig: int = signal.SIGINT,
|
8113
|
+
code: int = 1,
|
8114
|
+
) -> None:
|
8115
|
+
time.sleep(delay_s)
|
8116
|
+
|
8117
|
+
if (pgid := os.getpgid(0)) == os.getpid():
|
8118
|
+
os.killpg(pgid, sig)
|
8119
|
+
|
8120
|
+
os._exit(code) # noqa
|
8121
|
+
|
8122
|
+
@cached_nullary
|
8123
|
+
def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
|
8124
|
+
if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
|
8125
|
+
return None
|
8126
|
+
|
8127
|
+
thr = threading.Thread(
|
8128
|
+
target=functools.partial(self._timebomb_main, tbd),
|
8129
|
+
name=f'{self.__class__.__name__}.timebomb',
|
8130
|
+
daemon=True,
|
8131
|
+
)
|
8132
|
+
|
8133
|
+
thr.start()
|
8134
|
+
|
8135
|
+
log.debug('Started timebomb thread: %r', thr)
|
8136
|
+
|
8137
|
+
return thr
|
8138
|
+
|
8139
|
+
#
|
8140
|
+
|
8141
|
+
@cached_nullary
|
8142
|
+
def _log_handler(self) -> _RemoteLogHandler:
|
8143
|
+
return _RemoteLogHandler(self._chan)
|
8144
|
+
|
8145
|
+
#
|
8146
|
+
|
8147
|
+
async def _setup(self) -> None:
|
8148
|
+
check.none(self.__bootstrap)
|
8149
|
+
check.none(self.__injector)
|
8150
|
+
|
8151
|
+
# Bootstrap
|
8152
|
+
|
8153
|
+
self.__bootstrap = check.not_none(await self._chan.recv_obj(MainBootstrap))
|
8154
|
+
|
8155
|
+
if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
|
8156
|
+
pycharm_debug_connect(prd)
|
8157
|
+
|
8158
|
+
self.__injector = main_bootstrap(self._bootstrap)
|
8159
|
+
|
8160
|
+
self._chan.set_marshaler(self._injector[ObjMarshalerManager])
|
8161
|
+
|
8162
|
+
# Post-bootstrap
|
8163
|
+
|
8164
|
+
if self._bootstrap.remote_config.set_pgid:
|
8165
|
+
if os.getpgid(0) != os.getpid():
|
8166
|
+
log.debug('Setting pgid')
|
8167
|
+
os.setpgid(0, 0)
|
8168
|
+
|
8169
|
+
if (ds := self._bootstrap.remote_config.deathsig) is not None:
|
8170
|
+
log.debug('Setting deathsig: %s', ds)
|
8171
|
+
set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
|
8172
|
+
|
8173
|
+
self._timebomb_thread()
|
8174
|
+
|
8175
|
+
if self._bootstrap.remote_config.forward_logging:
|
8176
|
+
log.debug('Installing log forwarder')
|
8177
|
+
logging.root.addHandler(self._log_handler())
|
8178
|
+
|
8179
|
+
#
|
8180
|
+
|
8181
|
+
async def run(self) -> None:
|
8182
|
+
await self._setup()
|
8183
|
+
|
8184
|
+
executor = self._injector[LocalCommandExecutor]
|
8185
|
+
|
8186
|
+
handler = _RemoteCommandHandler(self._chan, executor)
|
8187
|
+
|
8188
|
+
await handler.run()
|
8189
|
+
|
8190
|
+
|
8191
|
+
def _remote_execution_main() -> None:
|
8192
|
+
rt = pyremote_bootstrap_finalize() # noqa
|
8193
|
+
|
8194
|
+
async def inner() -> None:
|
8195
|
+
input = await asyncio_open_stream_reader(rt.input) # noqa
|
8196
|
+
output = await asyncio_open_stream_writer(rt.output)
|
8197
|
+
|
8198
|
+
chan = RemoteChannelImpl(
|
8199
|
+
input,
|
8200
|
+
output,
|
8201
|
+
)
|
8202
|
+
|
8203
|
+
await _RemoteExecutionMain(chan).run()
|
8204
|
+
|
8205
|
+
asyncio.run(inner())
|
8206
|
+
|
8207
|
+
|
8208
|
+
########################################
|
8209
|
+
# ../remote/spawning.py
|
8210
|
+
|
8211
|
+
|
8212
|
+
##
|
8213
|
+
|
8214
|
+
|
8215
|
+
class RemoteSpawning(abc.ABC):
|
8216
|
+
@dc.dataclass(frozen=True)
|
8217
|
+
class Target:
|
8218
|
+
shell: ta.Optional[str] = None
|
8219
|
+
shell_quote: bool = False
|
8220
|
+
|
8221
|
+
DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
|
8222
|
+
python: str = DEFAULT_PYTHON
|
8223
|
+
|
8224
|
+
stderr: ta.Optional[str] = None # SubprocessChannelOption
|
8225
|
+
|
8226
|
+
@dc.dataclass(frozen=True)
|
8227
|
+
class Spawned:
|
8228
|
+
stdin: asyncio.StreamWriter
|
8229
|
+
stdout: asyncio.StreamReader
|
8230
|
+
stderr: ta.Optional[asyncio.StreamReader]
|
8231
|
+
|
8232
|
+
@abc.abstractmethod
|
8233
|
+
def spawn(
|
8234
|
+
self,
|
8235
|
+
tgt: Target,
|
8236
|
+
src: str,
|
8237
|
+
*,
|
8238
|
+
timeout: ta.Optional[float] = None,
|
8239
|
+
debug: bool = False,
|
8240
|
+
) -> ta.AsyncContextManager[Spawned]:
|
8241
|
+
raise NotImplementedError
|
8242
|
+
|
8243
|
+
|
8244
|
+
##
|
8245
|
+
|
8246
|
+
|
8247
|
+
class SubprocessRemoteSpawning(RemoteSpawning):
|
8248
|
+
class _PreparedCmd(ta.NamedTuple): # noqa
|
8249
|
+
cmd: ta.Sequence[str]
|
8250
|
+
shell: bool
|
8251
|
+
|
8252
|
+
def _prepare_cmd(
|
8253
|
+
self,
|
8254
|
+
tgt: RemoteSpawning.Target,
|
8255
|
+
src: str,
|
8256
|
+
) -> _PreparedCmd:
|
8257
|
+
if tgt.shell is not None:
|
8258
|
+
sh_src = f'{tgt.python} -c {shlex.quote(src)}'
|
8259
|
+
if tgt.shell_quote:
|
8260
|
+
sh_src = shlex.quote(sh_src)
|
8261
|
+
sh_cmd = f'{tgt.shell} {sh_src}'
|
8262
|
+
return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
|
8029
8263
|
|
8030
8264
|
else:
|
8031
8265
|
return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
|
@@ -8366,14 +8600,14 @@ def make_deploy_tag(
|
|
8366
8600
|
now = datetime.datetime.utcnow() # noqa
|
8367
8601
|
now_fmt = '%Y%m%dT%H%M%S'
|
8368
8602
|
now_str = now.strftime(now_fmt)
|
8369
|
-
return DeployTag('-'.join([
|
8603
|
+
return DeployTag('-'.join([now_str, rev]))
|
8370
8604
|
|
8371
8605
|
|
8372
8606
|
class DeployAppManager(DeployPathOwner):
|
8373
8607
|
def __init__(
|
8374
8608
|
self,
|
8375
8609
|
*,
|
8376
|
-
deploy_home: DeployHome,
|
8610
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
8377
8611
|
git: DeployGitManager,
|
8378
8612
|
venvs: DeployVenvManager,
|
8379
8613
|
) -> None:
|
@@ -8383,7 +8617,9 @@ class DeployAppManager(DeployPathOwner):
|
|
8383
8617
|
self._git = git
|
8384
8618
|
self._venvs = venvs
|
8385
8619
|
|
8386
|
-
|
8620
|
+
@cached_nullary
|
8621
|
+
def _dir(self) -> str:
|
8622
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
|
8387
8623
|
|
8388
8624
|
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
8389
8625
|
return {
|
@@ -8392,20 +8628,16 @@ class DeployAppManager(DeployPathOwner):
|
|
8392
8628
|
|
8393
8629
|
async def prepare_app(
|
8394
8630
|
self,
|
8395
|
-
|
8396
|
-
rev: DeployRev,
|
8397
|
-
repo: DeployGitRepo,
|
8631
|
+
spec: DeploySpec,
|
8398
8632
|
):
|
8399
|
-
app_tag = DeployAppTag(app, make_deploy_tag(rev))
|
8400
|
-
app_dir = os.path.join(self._dir, app, app_tag.tag)
|
8633
|
+
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev))
|
8634
|
+
app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
|
8401
8635
|
|
8402
8636
|
#
|
8403
8637
|
|
8404
8638
|
await self._git.checkout(
|
8405
|
-
|
8406
|
-
|
8407
|
-
rev=rev,
|
8408
|
-
),
|
8639
|
+
spec.repo,
|
8640
|
+
spec.rev,
|
8409
8641
|
app_dir,
|
8410
8642
|
)
|
8411
8643
|
|
@@ -8415,145 +8647,122 @@ class DeployAppManager(DeployPathOwner):
|
|
8415
8647
|
|
8416
8648
|
|
8417
8649
|
########################################
|
8418
|
-
# ../remote/
|
8419
|
-
|
8420
|
-
|
8421
|
-
##
|
8422
|
-
|
8423
|
-
|
8424
|
-
class _RemoteExecutionLogHandler(logging.Handler):
|
8425
|
-
def __init__(self, fn: ta.Callable[[str], None]) -> None:
|
8426
|
-
super().__init__()
|
8427
|
-
self._fn = fn
|
8428
|
-
|
8429
|
-
def emit(self, record):
|
8430
|
-
msg = self.format(record)
|
8431
|
-
self._fn(msg)
|
8650
|
+
# ../remote/connection.py
|
8432
8651
|
|
8433
8652
|
|
8434
8653
|
##
|
8435
8654
|
|
8436
8655
|
|
8437
|
-
class
|
8656
|
+
class PyremoteRemoteExecutionConnector:
|
8438
8657
|
def __init__(
|
8439
8658
|
self,
|
8440
|
-
|
8659
|
+
*,
|
8660
|
+
spawning: RemoteSpawning,
|
8661
|
+
msh: ObjMarshalerManager,
|
8662
|
+
payload_file: ta.Optional[RemoteExecutionPayloadFile] = None,
|
8441
8663
|
) -> None:
|
8442
8664
|
super().__init__()
|
8443
8665
|
|
8444
|
-
self.
|
8666
|
+
self._spawning = spawning
|
8667
|
+
self._msh = msh
|
8668
|
+
self._payload_file = payload_file
|
8445
8669
|
|
8446
|
-
|
8447
|
-
self.__injector: ta.Optional[Injector] = None
|
8670
|
+
#
|
8448
8671
|
|
8449
|
-
@
|
8450
|
-
def
|
8451
|
-
return
|
8672
|
+
@cached_nullary
|
8673
|
+
def _payload_src(self) -> str:
|
8674
|
+
return get_remote_payload_src(file=self._payload_file)
|
8452
8675
|
|
8453
|
-
@
|
8454
|
-
def
|
8455
|
-
return
|
8676
|
+
@cached_nullary
|
8677
|
+
def _remote_src(self) -> ta.Sequence[str]:
|
8678
|
+
return [
|
8679
|
+
self._payload_src(),
|
8680
|
+
'_remote_execution_main()',
|
8681
|
+
]
|
8682
|
+
|
8683
|
+
@cached_nullary
|
8684
|
+
def _spawn_src(self) -> str:
|
8685
|
+
return pyremote_build_bootstrap_cmd(__package__ or 'manage')
|
8456
8686
|
|
8457
8687
|
#
|
8458
8688
|
|
8459
|
-
|
8689
|
+
@contextlib.asynccontextmanager
|
8690
|
+
async def connect(
|
8460
8691
|
self,
|
8461
|
-
|
8462
|
-
|
8463
|
-
|
8464
|
-
|
8465
|
-
|
8466
|
-
time.sleep(delay_s)
|
8692
|
+
tgt: RemoteSpawning.Target,
|
8693
|
+
bs: MainBootstrap,
|
8694
|
+
) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
|
8695
|
+
spawn_src = self._spawn_src()
|
8696
|
+
remote_src = self._remote_src()
|
8467
8697
|
|
8468
|
-
|
8469
|
-
|
8698
|
+
async with self._spawning.spawn(
|
8699
|
+
tgt,
|
8700
|
+
spawn_src,
|
8701
|
+
debug=bs.main_config.debug,
|
8702
|
+
) as proc:
|
8703
|
+
res = await PyremoteBootstrapDriver( # noqa
|
8704
|
+
remote_src,
|
8705
|
+
PyremoteBootstrapOptions(
|
8706
|
+
debug=bs.main_config.debug,
|
8707
|
+
),
|
8708
|
+
).async_run(
|
8709
|
+
proc.stdout,
|
8710
|
+
proc.stdin,
|
8711
|
+
)
|
8470
8712
|
|
8471
|
-
|
8713
|
+
chan = RemoteChannelImpl(
|
8714
|
+
proc.stdout,
|
8715
|
+
proc.stdin,
|
8716
|
+
msh=self._msh,
|
8717
|
+
)
|
8472
8718
|
|
8473
|
-
|
8474
|
-
def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
|
8475
|
-
if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
|
8476
|
-
return None
|
8719
|
+
await chan.send_obj(bs)
|
8477
8720
|
|
8478
|
-
|
8479
|
-
|
8480
|
-
|
8481
|
-
daemon=True,
|
8482
|
-
)
|
8721
|
+
rce: RemoteCommandExecutor
|
8722
|
+
async with aclosing(RemoteCommandExecutor(chan)) as rce:
|
8723
|
+
await rce.start()
|
8483
8724
|
|
8484
|
-
|
8485
|
-
|
8486
|
-
log.debug('Started timebomb thread: %r', thr)
|
8487
|
-
|
8488
|
-
return thr
|
8489
|
-
|
8490
|
-
#
|
8491
|
-
|
8492
|
-
@cached_nullary
|
8493
|
-
def _log_handler(self) -> _RemoteLogHandler:
|
8494
|
-
return _RemoteLogHandler(self._chan)
|
8495
|
-
|
8496
|
-
#
|
8497
|
-
|
8498
|
-
async def _setup(self) -> None:
|
8499
|
-
check.none(self.__bootstrap)
|
8500
|
-
check.none(self.__injector)
|
8501
|
-
|
8502
|
-
# Bootstrap
|
8503
|
-
|
8504
|
-
self.__bootstrap = check.not_none(await self._chan.recv_obj(MainBootstrap))
|
8505
|
-
|
8506
|
-
if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
|
8507
|
-
pycharm_debug_connect(prd)
|
8508
|
-
|
8509
|
-
self.__injector = main_bootstrap(self._bootstrap)
|
8510
|
-
|
8511
|
-
self._chan.set_marshaler(self._injector[ObjMarshalerManager])
|
8512
|
-
|
8513
|
-
# Post-bootstrap
|
8514
|
-
|
8515
|
-
if self._bootstrap.remote_config.set_pgid:
|
8516
|
-
if os.getpgid(0) != os.getpid():
|
8517
|
-
log.debug('Setting pgid')
|
8518
|
-
os.setpgid(0, 0)
|
8519
|
-
|
8520
|
-
if (ds := self._bootstrap.remote_config.deathsig) is not None:
|
8521
|
-
log.debug('Setting deathsig: %s', ds)
|
8522
|
-
set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
|
8523
|
-
|
8524
|
-
self._timebomb_thread()
|
8525
|
-
|
8526
|
-
if self._bootstrap.remote_config.forward_logging:
|
8527
|
-
log.debug('Installing log forwarder')
|
8528
|
-
logging.root.addHandler(self._log_handler())
|
8725
|
+
yield rce
|
8529
8726
|
|
8530
|
-
#
|
8531
8727
|
|
8532
|
-
|
8533
|
-
await self._setup()
|
8534
|
-
|
8535
|
-
executor = self._injector[LocalCommandExecutor]
|
8728
|
+
##
|
8536
8729
|
|
8537
|
-
handler = _RemoteCommandHandler(self._chan, executor)
|
8538
8730
|
|
8539
|
-
|
8731
|
+
class InProcessRemoteExecutionConnector:
|
8732
|
+
def __init__(
|
8733
|
+
self,
|
8734
|
+
*,
|
8735
|
+
msh: ObjMarshalerManager,
|
8736
|
+
local_executor: LocalCommandExecutor,
|
8737
|
+
) -> None:
|
8738
|
+
super().__init__()
|
8540
8739
|
|
8740
|
+
self._msh = msh
|
8741
|
+
self._local_executor = local_executor
|
8541
8742
|
|
8542
|
-
|
8543
|
-
|
8743
|
+
@contextlib.asynccontextmanager
|
8744
|
+
async def connect(self) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
|
8745
|
+
r0, w0 = asyncio_create_bytes_channel()
|
8746
|
+
r1, w1 = asyncio_create_bytes_channel()
|
8544
8747
|
|
8545
|
-
|
8546
|
-
|
8547
|
-
output = await asyncio_open_stream_writer(rt.output)
|
8748
|
+
remote_chan = RemoteChannelImpl(r0, w1, msh=self._msh)
|
8749
|
+
local_chan = RemoteChannelImpl(r1, w0, msh=self._msh)
|
8548
8750
|
|
8549
|
-
|
8550
|
-
|
8551
|
-
|
8751
|
+
rch = _RemoteCommandHandler(
|
8752
|
+
remote_chan,
|
8753
|
+
self._local_executor,
|
8552
8754
|
)
|
8755
|
+
rch_task = asyncio.create_task(rch.run()) # noqa
|
8756
|
+
try:
|
8757
|
+
rce: RemoteCommandExecutor
|
8758
|
+
async with aclosing(RemoteCommandExecutor(local_chan)) as rce:
|
8759
|
+
await rce.start()
|
8553
8760
|
|
8554
|
-
|
8761
|
+
yield rce
|
8555
8762
|
|
8556
|
-
|
8763
|
+
finally:
|
8764
|
+
rch.stop()
|
8765
|
+
await rch_task
|
8557
8766
|
|
8558
8767
|
|
8559
8768
|
########################################
|
@@ -8899,521 +9108,256 @@ class PyenvVersionInstaller:
|
|
8899
9108
|
self._pyenv.exe(),
|
8900
9109
|
'install',
|
8901
9110
|
*conf_args,
|
8902
|
-
]
|
8903
|
-
|
8904
|
-
await asyncio_subprocesses.check_call(
|
8905
|
-
*full_args,
|
8906
|
-
env=env,
|
8907
|
-
)
|
8908
|
-
|
8909
|
-
exe = os.path.join(await self.install_dir(), 'bin', 'python')
|
8910
|
-
if not os.path.isfile(exe):
|
8911
|
-
raise RuntimeError(f'Interpreter not found: {exe}')
|
8912
|
-
return exe
|
8913
|
-
|
8914
|
-
|
8915
|
-
##
|
8916
|
-
|
8917
|
-
|
8918
|
-
class PyenvInterpProvider(InterpProvider):
|
8919
|
-
def __init__(
|
8920
|
-
self,
|
8921
|
-
pyenv: Pyenv = Pyenv(),
|
8922
|
-
|
8923
|
-
inspect: bool = False,
|
8924
|
-
inspector: InterpInspector = INTERP_INSPECTOR,
|
8925
|
-
|
8926
|
-
*,
|
8927
|
-
|
8928
|
-
try_update: bool = False,
|
8929
|
-
) -> None:
|
8930
|
-
super().__init__()
|
8931
|
-
|
8932
|
-
self._pyenv = pyenv
|
8933
|
-
|
8934
|
-
self._inspect = inspect
|
8935
|
-
self._inspector = inspector
|
8936
|
-
|
8937
|
-
self._try_update = try_update
|
8938
|
-
|
8939
|
-
#
|
8940
|
-
|
8941
|
-
@staticmethod
|
8942
|
-
def guess_version(s: str) -> ta.Optional[InterpVersion]:
|
8943
|
-
def strip_sfx(s: str, sfx: str) -> ta.Tuple[str, bool]:
|
8944
|
-
if s.endswith(sfx):
|
8945
|
-
return s[:-len(sfx)], True
|
8946
|
-
return s, False
|
8947
|
-
ok = {}
|
8948
|
-
s, ok['debug'] = strip_sfx(s, '-debug')
|
8949
|
-
s, ok['threaded'] = strip_sfx(s, 't')
|
8950
|
-
try:
|
8951
|
-
v = Version(s)
|
8952
|
-
except InvalidVersion:
|
8953
|
-
return None
|
8954
|
-
return InterpVersion(v, InterpOpts(**ok))
|
8955
|
-
|
8956
|
-
class Installed(ta.NamedTuple):
|
8957
|
-
name: str
|
8958
|
-
exe: str
|
8959
|
-
version: InterpVersion
|
8960
|
-
|
8961
|
-
async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
8962
|
-
iv: ta.Optional[InterpVersion]
|
8963
|
-
if self._inspect:
|
8964
|
-
try:
|
8965
|
-
iv = check.not_none(await self._inspector.inspect(ep)).iv
|
8966
|
-
except Exception as e: # noqa
|
8967
|
-
return None
|
8968
|
-
else:
|
8969
|
-
iv = self.guess_version(vn)
|
8970
|
-
if iv is None:
|
8971
|
-
return None
|
8972
|
-
return PyenvInterpProvider.Installed(
|
8973
|
-
name=vn,
|
8974
|
-
exe=ep,
|
8975
|
-
version=iv,
|
8976
|
-
)
|
8977
|
-
|
8978
|
-
async def installed(self) -> ta.Sequence[Installed]:
|
8979
|
-
ret: ta.List[PyenvInterpProvider.Installed] = []
|
8980
|
-
for vn, ep in await self._pyenv.version_exes():
|
8981
|
-
if (i := await self._make_installed(vn, ep)) is None:
|
8982
|
-
log.debug('Invalid pyenv version: %s', vn)
|
8983
|
-
continue
|
8984
|
-
ret.append(i)
|
8985
|
-
return ret
|
8986
|
-
|
8987
|
-
#
|
8988
|
-
|
8989
|
-
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
8990
|
-
return [i.version for i in await self.installed()]
|
8991
|
-
|
8992
|
-
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
8993
|
-
for i in await self.installed():
|
8994
|
-
if i.version == version:
|
8995
|
-
return Interp(
|
8996
|
-
exe=i.exe,
|
8997
|
-
version=i.version,
|
8998
|
-
)
|
8999
|
-
raise KeyError(version)
|
9000
|
-
|
9001
|
-
#
|
9002
|
-
|
9003
|
-
async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9004
|
-
lst = []
|
9005
|
-
|
9006
|
-
for vs in await self._pyenv.installable_versions():
|
9007
|
-
if (iv := self.guess_version(vs)) is None:
|
9008
|
-
continue
|
9009
|
-
if iv.opts.debug:
|
9010
|
-
raise Exception('Pyenv installable versions not expected to have debug suffix')
|
9011
|
-
for d in [False, True]:
|
9012
|
-
lst.append(dc.replace(iv, opts=dc.replace(iv.opts, debug=d)))
|
9013
|
-
|
9014
|
-
return lst
|
9015
|
-
|
9016
|
-
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9017
|
-
lst = await self._get_installable_versions(spec)
|
9018
|
-
|
9019
|
-
if self._try_update and not any(v in spec for v in lst):
|
9020
|
-
if self._pyenv.update():
|
9021
|
-
lst = await self._get_installable_versions(spec)
|
9022
|
-
|
9023
|
-
return lst
|
9024
|
-
|
9025
|
-
async def install_version(self, version: InterpVersion) -> Interp:
|
9026
|
-
inst_version = str(version.version)
|
9027
|
-
inst_opts = version.opts
|
9028
|
-
if inst_opts.threaded:
|
9029
|
-
inst_version += 't'
|
9030
|
-
inst_opts = dc.replace(inst_opts, threaded=False)
|
9031
|
-
|
9032
|
-
installer = PyenvVersionInstaller(
|
9033
|
-
inst_version,
|
9034
|
-
interp_opts=inst_opts,
|
9035
|
-
)
|
9036
|
-
|
9037
|
-
exe = await installer.install()
|
9038
|
-
return Interp(exe, version)
|
9039
|
-
|
9040
|
-
|
9041
|
-
########################################
|
9042
|
-
# ../../../omdev/interp/system.py
|
9043
|
-
"""
|
9044
|
-
TODO:
|
9045
|
-
- python, python3, python3.12, ...
|
9046
|
-
- check if path py's are venvs: sys.prefix != sys.base_prefix
|
9047
|
-
"""
|
9048
|
-
|
9049
|
-
|
9050
|
-
##
|
9051
|
-
|
9052
|
-
|
9053
|
-
@dc.dataclass(frozen=True)
|
9054
|
-
class SystemInterpProvider(InterpProvider):
|
9055
|
-
cmd: str = 'python3'
|
9056
|
-
path: ta.Optional[str] = None
|
9057
|
-
|
9058
|
-
inspect: bool = False
|
9059
|
-
inspector: InterpInspector = INTERP_INSPECTOR
|
9060
|
-
|
9061
|
-
#
|
9062
|
-
|
9063
|
-
@staticmethod
|
9064
|
-
def _re_which(
|
9065
|
-
pat: re.Pattern,
|
9066
|
-
*,
|
9067
|
-
mode: int = os.F_OK | os.X_OK,
|
9068
|
-
path: ta.Optional[str] = None,
|
9069
|
-
) -> ta.List[str]:
|
9070
|
-
if path is None:
|
9071
|
-
path = os.environ.get('PATH', None)
|
9072
|
-
if path is None:
|
9073
|
-
try:
|
9074
|
-
path = os.confstr('CS_PATH')
|
9075
|
-
except (AttributeError, ValueError):
|
9076
|
-
path = os.defpath
|
9077
|
-
|
9078
|
-
if not path:
|
9079
|
-
return []
|
9080
|
-
|
9081
|
-
path = os.fsdecode(path)
|
9082
|
-
pathlst = path.split(os.pathsep)
|
9083
|
-
|
9084
|
-
def _access_check(fn: str, mode: int) -> bool:
|
9085
|
-
return os.path.exists(fn) and os.access(fn, mode)
|
9086
|
-
|
9087
|
-
out = []
|
9088
|
-
seen = set()
|
9089
|
-
for d in pathlst:
|
9090
|
-
normdir = os.path.normcase(d)
|
9091
|
-
if normdir not in seen:
|
9092
|
-
seen.add(normdir)
|
9093
|
-
if not _access_check(normdir, mode):
|
9094
|
-
continue
|
9095
|
-
for thefile in os.listdir(d):
|
9096
|
-
name = os.path.join(d, thefile)
|
9097
|
-
if not (
|
9098
|
-
os.path.isfile(name) and
|
9099
|
-
pat.fullmatch(thefile) and
|
9100
|
-
_access_check(name, mode)
|
9101
|
-
):
|
9102
|
-
continue
|
9103
|
-
out.append(name)
|
9104
|
-
|
9105
|
-
return out
|
9106
|
-
|
9107
|
-
@cached_nullary
|
9108
|
-
def exes(self) -> ta.List[str]:
|
9109
|
-
return self._re_which(
|
9110
|
-
re.compile(r'python3(\.\d+)?'),
|
9111
|
-
path=self.path,
|
9112
|
-
)
|
9113
|
-
|
9114
|
-
#
|
9115
|
-
|
9116
|
-
async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
9117
|
-
if not self.inspect:
|
9118
|
-
s = os.path.basename(exe)
|
9119
|
-
if s.startswith('python'):
|
9120
|
-
s = s[len('python'):]
|
9121
|
-
if '.' in s:
|
9122
|
-
try:
|
9123
|
-
return InterpVersion.parse(s)
|
9124
|
-
except InvalidVersion:
|
9125
|
-
pass
|
9126
|
-
ii = await self.inspector.inspect(exe)
|
9127
|
-
return ii.iv if ii is not None else None
|
9128
|
-
|
9129
|
-
async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
9130
|
-
lst = []
|
9131
|
-
for e in self.exes():
|
9132
|
-
if (ev := await self.get_exe_version(e)) is None:
|
9133
|
-
log.debug('Invalid system version: %s', e)
|
9134
|
-
continue
|
9135
|
-
lst.append((e, ev))
|
9136
|
-
return lst
|
9137
|
-
|
9138
|
-
#
|
9139
|
-
|
9140
|
-
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9141
|
-
return [ev for e, ev in await self.exe_versions()]
|
9142
|
-
|
9143
|
-
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
9144
|
-
for e, ev in await self.exe_versions():
|
9145
|
-
if ev != version:
|
9146
|
-
continue
|
9147
|
-
return Interp(
|
9148
|
-
exe=e,
|
9149
|
-
version=ev,
|
9150
|
-
)
|
9151
|
-
raise KeyError(version)
|
9152
|
-
|
9153
|
-
|
9154
|
-
########################################
|
9155
|
-
# ../remote/connection.py
|
9156
|
-
|
9157
|
-
|
9158
|
-
##
|
9159
|
-
|
9160
|
-
|
9161
|
-
class PyremoteRemoteExecutionConnector:
|
9162
|
-
def __init__(
|
9163
|
-
self,
|
9164
|
-
*,
|
9165
|
-
spawning: RemoteSpawning,
|
9166
|
-
msh: ObjMarshalerManager,
|
9167
|
-
payload_file: ta.Optional[RemoteExecutionPayloadFile] = None,
|
9168
|
-
) -> None:
|
9169
|
-
super().__init__()
|
9170
|
-
|
9171
|
-
self._spawning = spawning
|
9172
|
-
self._msh = msh
|
9173
|
-
self._payload_file = payload_file
|
9174
|
-
|
9175
|
-
#
|
9176
|
-
|
9177
|
-
@cached_nullary
|
9178
|
-
def _payload_src(self) -> str:
|
9179
|
-
return get_remote_payload_src(file=self._payload_file)
|
9180
|
-
|
9181
|
-
@cached_nullary
|
9182
|
-
def _remote_src(self) -> ta.Sequence[str]:
|
9183
|
-
return [
|
9184
|
-
self._payload_src(),
|
9185
|
-
'_remote_execution_main()',
|
9186
|
-
]
|
9187
|
-
|
9188
|
-
@cached_nullary
|
9189
|
-
def _spawn_src(self) -> str:
|
9190
|
-
return pyremote_build_bootstrap_cmd(__package__ or 'manage')
|
9191
|
-
|
9192
|
-
#
|
9193
|
-
|
9194
|
-
@contextlib.asynccontextmanager
|
9195
|
-
async def connect(
|
9196
|
-
self,
|
9197
|
-
tgt: RemoteSpawning.Target,
|
9198
|
-
bs: MainBootstrap,
|
9199
|
-
) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
|
9200
|
-
spawn_src = self._spawn_src()
|
9201
|
-
remote_src = self._remote_src()
|
9202
|
-
|
9203
|
-
async with self._spawning.spawn(
|
9204
|
-
tgt,
|
9205
|
-
spawn_src,
|
9206
|
-
debug=bs.main_config.debug,
|
9207
|
-
) as proc:
|
9208
|
-
res = await PyremoteBootstrapDriver( # noqa
|
9209
|
-
remote_src,
|
9210
|
-
PyremoteBootstrapOptions(
|
9211
|
-
debug=bs.main_config.debug,
|
9212
|
-
),
|
9213
|
-
).async_run(
|
9214
|
-
proc.stdout,
|
9215
|
-
proc.stdin,
|
9216
|
-
)
|
9217
|
-
|
9218
|
-
chan = RemoteChannelImpl(
|
9219
|
-
proc.stdout,
|
9220
|
-
proc.stdin,
|
9221
|
-
msh=self._msh,
|
9222
|
-
)
|
9223
|
-
|
9224
|
-
await chan.send_obj(bs)
|
9225
|
-
|
9226
|
-
rce: RemoteCommandExecutor
|
9227
|
-
async with aclosing(RemoteCommandExecutor(chan)) as rce:
|
9228
|
-
await rce.start()
|
9111
|
+
]
|
9229
9112
|
|
9230
|
-
|
9113
|
+
await asyncio_subprocesses.check_call(
|
9114
|
+
*full_args,
|
9115
|
+
env=env,
|
9116
|
+
)
|
9117
|
+
|
9118
|
+
exe = os.path.join(await self.install_dir(), 'bin', 'python')
|
9119
|
+
if not os.path.isfile(exe):
|
9120
|
+
raise RuntimeError(f'Interpreter not found: {exe}')
|
9121
|
+
return exe
|
9231
9122
|
|
9232
9123
|
|
9233
9124
|
##
|
9234
9125
|
|
9235
9126
|
|
9236
|
-
class
|
9127
|
+
class PyenvInterpProvider(InterpProvider):
|
9237
9128
|
def __init__(
|
9238
9129
|
self,
|
9130
|
+
pyenv: Pyenv = Pyenv(),
|
9131
|
+
|
9132
|
+
inspect: bool = False,
|
9133
|
+
inspector: InterpInspector = INTERP_INSPECTOR,
|
9134
|
+
|
9239
9135
|
*,
|
9240
|
-
|
9241
|
-
|
9136
|
+
|
9137
|
+
try_update: bool = False,
|
9242
9138
|
) -> None:
|
9243
9139
|
super().__init__()
|
9244
9140
|
|
9245
|
-
self.
|
9246
|
-
self._local_executor = local_executor
|
9141
|
+
self._pyenv = pyenv
|
9247
9142
|
|
9248
|
-
|
9249
|
-
|
9250
|
-
r0, w0 = asyncio_create_bytes_channel()
|
9251
|
-
r1, w1 = asyncio_create_bytes_channel()
|
9143
|
+
self._inspect = inspect
|
9144
|
+
self._inspector = inspector
|
9252
9145
|
|
9253
|
-
|
9254
|
-
local_chan = RemoteChannelImpl(r1, w0, msh=self._msh)
|
9146
|
+
self._try_update = try_update
|
9255
9147
|
|
9256
|
-
|
9257
|
-
|
9258
|
-
|
9259
|
-
|
9260
|
-
|
9148
|
+
#
|
9149
|
+
|
9150
|
+
@staticmethod
|
9151
|
+
def guess_version(s: str) -> ta.Optional[InterpVersion]:
|
9152
|
+
def strip_sfx(s: str, sfx: str) -> ta.Tuple[str, bool]:
|
9153
|
+
if s.endswith(sfx):
|
9154
|
+
return s[:-len(sfx)], True
|
9155
|
+
return s, False
|
9156
|
+
ok = {}
|
9157
|
+
s, ok['debug'] = strip_sfx(s, '-debug')
|
9158
|
+
s, ok['threaded'] = strip_sfx(s, 't')
|
9261
9159
|
try:
|
9262
|
-
|
9263
|
-
|
9264
|
-
|
9160
|
+
v = Version(s)
|
9161
|
+
except InvalidVersion:
|
9162
|
+
return None
|
9163
|
+
return InterpVersion(v, InterpOpts(**ok))
|
9265
9164
|
|
9266
|
-
|
9165
|
+
class Installed(ta.NamedTuple):
|
9166
|
+
name: str
|
9167
|
+
exe: str
|
9168
|
+
version: InterpVersion
|
9267
9169
|
|
9268
|
-
|
9269
|
-
|
9270
|
-
|
9170
|
+
async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
|
9171
|
+
iv: ta.Optional[InterpVersion]
|
9172
|
+
if self._inspect:
|
9173
|
+
try:
|
9174
|
+
iv = check.not_none(await self._inspector.inspect(ep)).iv
|
9175
|
+
except Exception as e: # noqa
|
9176
|
+
return None
|
9177
|
+
else:
|
9178
|
+
iv = self.guess_version(vn)
|
9179
|
+
if iv is None:
|
9180
|
+
return None
|
9181
|
+
return PyenvInterpProvider.Installed(
|
9182
|
+
name=vn,
|
9183
|
+
exe=ep,
|
9184
|
+
version=iv,
|
9185
|
+
)
|
9271
9186
|
|
9187
|
+
async def installed(self) -> ta.Sequence[Installed]:
|
9188
|
+
ret: ta.List[PyenvInterpProvider.Installed] = []
|
9189
|
+
for vn, ep in await self._pyenv.version_exes():
|
9190
|
+
if (i := await self._make_installed(vn, ep)) is None:
|
9191
|
+
log.debug('Invalid pyenv version: %s', vn)
|
9192
|
+
continue
|
9193
|
+
ret.append(i)
|
9194
|
+
return ret
|
9272
9195
|
|
9273
|
-
|
9274
|
-
# ../system/inject.py
|
9196
|
+
#
|
9275
9197
|
|
9198
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9199
|
+
return [i.version for i in await self.installed()]
|
9276
9200
|
|
9277
|
-
def
|
9278
|
-
|
9279
|
-
|
9280
|
-
|
9281
|
-
|
9282
|
-
|
9283
|
-
|
9201
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
9202
|
+
for i in await self.installed():
|
9203
|
+
if i.version == version:
|
9204
|
+
return Interp(
|
9205
|
+
exe=i.exe,
|
9206
|
+
version=i.version,
|
9207
|
+
)
|
9208
|
+
raise KeyError(version)
|
9284
9209
|
|
9285
9210
|
#
|
9286
9211
|
|
9287
|
-
|
9288
|
-
|
9212
|
+
async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9213
|
+
lst = []
|
9289
9214
|
|
9290
|
-
|
9215
|
+
for vs in await self._pyenv.installable_versions():
|
9216
|
+
if (iv := self.guess_version(vs)) is None:
|
9217
|
+
continue
|
9218
|
+
if iv.opts.debug:
|
9219
|
+
raise Exception('Pyenv installable versions not expected to have debug suffix')
|
9220
|
+
for d in [False, True]:
|
9221
|
+
lst.append(dc.replace(iv, opts=dc.replace(iv.opts, debug=d)))
|
9291
9222
|
|
9292
|
-
|
9293
|
-
lst.extend([
|
9294
|
-
inj.bind(YumSystemPackageManager, singleton=True),
|
9295
|
-
inj.bind(SystemPackageManager, to_key=YumSystemPackageManager),
|
9296
|
-
])
|
9223
|
+
return lst
|
9297
9224
|
|
9298
|
-
|
9299
|
-
lst.
|
9300
|
-
inj.bind(AptSystemPackageManager, singleton=True),
|
9301
|
-
inj.bind(SystemPackageManager, to_key=AptSystemPackageManager),
|
9302
|
-
])
|
9225
|
+
async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9226
|
+
lst = await self._get_installable_versions(spec)
|
9303
9227
|
|
9304
|
-
|
9305
|
-
|
9306
|
-
|
9307
|
-
inj.bind(SystemPackageManager, to_key=BrewSystemPackageManager),
|
9308
|
-
])
|
9228
|
+
if self._try_update and not any(v in spec for v in lst):
|
9229
|
+
if self._pyenv.update():
|
9230
|
+
lst = await self._get_installable_versions(spec)
|
9309
9231
|
|
9310
|
-
|
9232
|
+
return lst
|
9311
9233
|
|
9312
|
-
|
9313
|
-
|
9314
|
-
|
9234
|
+
async def install_version(self, version: InterpVersion) -> Interp:
|
9235
|
+
inst_version = str(version.version)
|
9236
|
+
inst_opts = version.opts
|
9237
|
+
if inst_opts.threaded:
|
9238
|
+
inst_version += 't'
|
9239
|
+
inst_opts = dc.replace(inst_opts, threaded=False)
|
9315
9240
|
|
9316
|
-
|
9241
|
+
installer = PyenvVersionInstaller(
|
9242
|
+
inst_version,
|
9243
|
+
interp_opts=inst_opts,
|
9244
|
+
)
|
9317
9245
|
|
9318
|
-
|
9246
|
+
exe = await installer.install()
|
9247
|
+
return Interp(exe, version)
|
9319
9248
|
|
9320
9249
|
|
9321
9250
|
########################################
|
9322
|
-
# ../../../omdev/interp/
|
9323
|
-
|
9324
|
-
|
9325
|
-
|
9326
|
-
|
9327
|
-
|
9251
|
+
# ../../../omdev/interp/system.py
|
9252
|
+
"""
|
9253
|
+
TODO:
|
9254
|
+
- python, python3, python3.12, ...
|
9255
|
+
- check if path py's are venvs: sys.prefix != sys.base_prefix
|
9256
|
+
"""
|
9328
9257
|
|
9329
9258
|
|
9330
|
-
|
9331
|
-
def __init__(
|
9332
|
-
self,
|
9333
|
-
providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
|
9334
|
-
) -> None:
|
9335
|
-
super().__init__()
|
9259
|
+
##
|
9336
9260
|
|
9337
|
-
self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
|
9338
9261
|
|
9339
|
-
|
9340
|
-
|
9341
|
-
|
9342
|
-
|
9343
|
-
for si in await p.get_installed_versions(spec)
|
9344
|
-
if spec.contains(si)
|
9345
|
-
]
|
9262
|
+
@dc.dataclass(frozen=True)
|
9263
|
+
class SystemInterpProvider(InterpProvider):
|
9264
|
+
cmd: str = 'python3'
|
9265
|
+
path: ta.Optional[str] = None
|
9346
9266
|
|
9347
|
-
|
9348
|
-
|
9349
|
-
return None
|
9267
|
+
inspect: bool = False
|
9268
|
+
inspector: InterpInspector = INTERP_INSPECTOR
|
9350
9269
|
|
9351
|
-
|
9352
|
-
bp = list(self._providers.values())[bi]
|
9353
|
-
return (bp, bv)
|
9270
|
+
#
|
9354
9271
|
|
9355
|
-
|
9356
|
-
|
9357
|
-
|
9272
|
+
@staticmethod
|
9273
|
+
def _re_which(
|
9274
|
+
pat: re.Pattern,
|
9358
9275
|
*,
|
9359
|
-
|
9360
|
-
|
9361
|
-
|
9362
|
-
if
|
9363
|
-
|
9364
|
-
|
9276
|
+
mode: int = os.F_OK | os.X_OK,
|
9277
|
+
path: ta.Optional[str] = None,
|
9278
|
+
) -> ta.List[str]:
|
9279
|
+
if path is None:
|
9280
|
+
path = os.environ.get('PATH', None)
|
9281
|
+
if path is None:
|
9282
|
+
try:
|
9283
|
+
path = os.confstr('CS_PATH')
|
9284
|
+
except (AttributeError, ValueError):
|
9285
|
+
path = os.defpath
|
9365
9286
|
|
9366
|
-
if not
|
9367
|
-
return
|
9287
|
+
if not path:
|
9288
|
+
return []
|
9368
9289
|
|
9369
|
-
|
9290
|
+
path = os.fsdecode(path)
|
9291
|
+
pathlst = path.split(os.pathsep)
|
9370
9292
|
|
9371
|
-
|
9372
|
-
|
9373
|
-
|
9374
|
-
|
9375
|
-
|
9376
|
-
|
9293
|
+
def _access_check(fn: str, mode: int) -> bool:
|
9294
|
+
return os.path.exists(fn) and os.access(fn, mode)
|
9295
|
+
|
9296
|
+
out = []
|
9297
|
+
seen = set()
|
9298
|
+
for d in pathlst:
|
9299
|
+
normdir = os.path.normcase(d)
|
9300
|
+
if normdir not in seen:
|
9301
|
+
seen.add(normdir)
|
9302
|
+
if not _access_check(normdir, mode):
|
9303
|
+
continue
|
9304
|
+
for thefile in os.listdir(d):
|
9305
|
+
name = os.path.join(d, thefile)
|
9306
|
+
if not (
|
9307
|
+
os.path.isfile(name) and
|
9308
|
+
pat.fullmatch(thefile) and
|
9309
|
+
_access_check(name, mode)
|
9310
|
+
):
|
9311
|
+
continue
|
9312
|
+
out.append(name)
|
9377
9313
|
|
9378
|
-
|
9379
|
-
return await tp.install_version(bv)
|
9314
|
+
return out
|
9380
9315
|
|
9381
|
-
|
9382
|
-
|
9383
|
-
|
9384
|
-
|
9385
|
-
|
9386
|
-
|
9387
|
-
if spec.contains(si)
|
9388
|
-
]
|
9389
|
-
if lst:
|
9390
|
-
print(f' {n}')
|
9391
|
-
for si in lst:
|
9392
|
-
print(f' {si}')
|
9316
|
+
@cached_nullary
|
9317
|
+
def exes(self) -> ta.List[str]:
|
9318
|
+
return self._re_which(
|
9319
|
+
re.compile(r'python3(\.\d+)?'),
|
9320
|
+
path=self.path,
|
9321
|
+
)
|
9393
9322
|
|
9394
|
-
|
9323
|
+
#
|
9395
9324
|
|
9396
|
-
|
9397
|
-
|
9398
|
-
|
9399
|
-
|
9400
|
-
|
9401
|
-
|
9402
|
-
|
9403
|
-
|
9404
|
-
|
9405
|
-
|
9406
|
-
|
9325
|
+
async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
|
9326
|
+
if not self.inspect:
|
9327
|
+
s = os.path.basename(exe)
|
9328
|
+
if s.startswith('python'):
|
9329
|
+
s = s[len('python'):]
|
9330
|
+
if '.' in s:
|
9331
|
+
try:
|
9332
|
+
return InterpVersion.parse(s)
|
9333
|
+
except InvalidVersion:
|
9334
|
+
pass
|
9335
|
+
ii = await self.inspector.inspect(exe)
|
9336
|
+
return ii.iv if ii is not None else None
|
9407
9337
|
|
9338
|
+
async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
|
9339
|
+
lst = []
|
9340
|
+
for e in self.exes():
|
9341
|
+
if (ev := await self.get_exe_version(e)) is None:
|
9342
|
+
log.debug('Invalid system version: %s', e)
|
9343
|
+
continue
|
9344
|
+
lst.append((e, ev))
|
9345
|
+
return lst
|
9408
9346
|
|
9409
|
-
|
9410
|
-
# pyenv is preferred to system interpreters as it tends to have more support for things like tkinter
|
9411
|
-
PyenvInterpProvider(try_update=True),
|
9347
|
+
#
|
9412
9348
|
|
9413
|
-
|
9349
|
+
async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
|
9350
|
+
return [ev for e, ev in await self.exe_versions()]
|
9414
9351
|
|
9415
|
-
|
9416
|
-
|
9352
|
+
async def get_installed_version(self, version: InterpVersion) -> Interp:
|
9353
|
+
for e, ev in await self.exe_versions():
|
9354
|
+
if ev != version:
|
9355
|
+
continue
|
9356
|
+
return Interp(
|
9357
|
+
exe=e,
|
9358
|
+
version=ev,
|
9359
|
+
)
|
9360
|
+
raise KeyError(version)
|
9417
9361
|
|
9418
9362
|
|
9419
9363
|
########################################
|
@@ -9444,6 +9388,54 @@ def bind_remote(
|
|
9444
9388
|
return inj.as_bindings(*lst)
|
9445
9389
|
|
9446
9390
|
|
9391
|
+
########################################
|
9392
|
+
# ../system/inject.py
|
9393
|
+
|
9394
|
+
|
9395
|
+
def bind_system(
|
9396
|
+
*,
|
9397
|
+
system_config: SystemConfig,
|
9398
|
+
) -> InjectorBindings:
|
9399
|
+
lst: ta.List[InjectorBindingOrBindings] = [
|
9400
|
+
inj.bind(system_config),
|
9401
|
+
]
|
9402
|
+
|
9403
|
+
#
|
9404
|
+
|
9405
|
+
platform = system_config.platform or detect_system_platform()
|
9406
|
+
lst.append(inj.bind(platform, key=Platform))
|
9407
|
+
|
9408
|
+
#
|
9409
|
+
|
9410
|
+
if isinstance(platform, AmazonLinuxPlatform):
|
9411
|
+
lst.extend([
|
9412
|
+
inj.bind(YumSystemPackageManager, singleton=True),
|
9413
|
+
inj.bind(SystemPackageManager, to_key=YumSystemPackageManager),
|
9414
|
+
])
|
9415
|
+
|
9416
|
+
elif isinstance(platform, LinuxPlatform):
|
9417
|
+
lst.extend([
|
9418
|
+
inj.bind(AptSystemPackageManager, singleton=True),
|
9419
|
+
inj.bind(SystemPackageManager, to_key=AptSystemPackageManager),
|
9420
|
+
])
|
9421
|
+
|
9422
|
+
elif isinstance(platform, DarwinPlatform):
|
9423
|
+
lst.extend([
|
9424
|
+
inj.bind(BrewSystemPackageManager, singleton=True),
|
9425
|
+
inj.bind(SystemPackageManager, to_key=BrewSystemPackageManager),
|
9426
|
+
])
|
9427
|
+
|
9428
|
+
#
|
9429
|
+
|
9430
|
+
lst.extend([
|
9431
|
+
bind_command(CheckSystemPackageCommand, CheckSystemPackageCommandExecutor),
|
9432
|
+
])
|
9433
|
+
|
9434
|
+
#
|
9435
|
+
|
9436
|
+
return inj.as_bindings(*lst)
|
9437
|
+
|
9438
|
+
|
9447
9439
|
########################################
|
9448
9440
|
# ../targets/connection.py
|
9449
9441
|
|
@@ -9579,33 +9571,101 @@ class SshManageTargetConnector(ManageTargetConnector):
|
|
9579
9571
|
|
9580
9572
|
|
9581
9573
|
########################################
|
9582
|
-
#
|
9574
|
+
# ../../../omdev/interp/resolvers.py
|
9583
9575
|
|
9584
9576
|
|
9585
|
-
|
9577
|
+
INTERP_PROVIDER_TYPES_BY_NAME: ta.Mapping[str, ta.Type[InterpProvider]] = {
|
9578
|
+
cls.name: cls for cls in deep_subclasses(InterpProvider) if abc.ABC not in cls.__bases__ # type: ignore
|
9579
|
+
}
|
9586
9580
|
|
9587
9581
|
|
9588
|
-
|
9589
|
-
|
9590
|
-
|
9591
|
-
|
9582
|
+
class InterpResolver:
|
9583
|
+
def __init__(
|
9584
|
+
self,
|
9585
|
+
providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
|
9586
|
+
) -> None:
|
9587
|
+
super().__init__()
|
9592
9588
|
|
9593
|
-
|
9594
|
-
class Output(Command.Output):
|
9595
|
-
exe: str
|
9596
|
-
version: str
|
9597
|
-
opts: InterpOpts
|
9589
|
+
self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
|
9598
9590
|
|
9591
|
+
async def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
|
9592
|
+
lst = [
|
9593
|
+
(i, si)
|
9594
|
+
for i, p in enumerate(self._providers.values())
|
9595
|
+
for si in await p.get_installed_versions(spec)
|
9596
|
+
if spec.contains(si)
|
9597
|
+
]
|
9599
9598
|
|
9600
|
-
|
9601
|
-
|
9602
|
-
|
9603
|
-
|
9604
|
-
|
9605
|
-
|
9606
|
-
|
9607
|
-
|
9599
|
+
slst = sorted(lst, key=lambda t: (-t[0], t[1].version))
|
9600
|
+
if not slst:
|
9601
|
+
return None
|
9602
|
+
|
9603
|
+
bi, bv = slst[-1]
|
9604
|
+
bp = list(self._providers.values())[bi]
|
9605
|
+
return (bp, bv)
|
9606
|
+
|
9607
|
+
async def resolve(
|
9608
|
+
self,
|
9609
|
+
spec: InterpSpecifier,
|
9610
|
+
*,
|
9611
|
+
install: bool = False,
|
9612
|
+
) -> ta.Optional[Interp]:
|
9613
|
+
tup = await self._resolve_installed(spec)
|
9614
|
+
if tup is not None:
|
9615
|
+
bp, bv = tup
|
9616
|
+
return await bp.get_installed_version(bv)
|
9617
|
+
|
9618
|
+
if not install:
|
9619
|
+
return None
|
9620
|
+
|
9621
|
+
tp = list(self._providers.values())[0] # noqa
|
9622
|
+
|
9623
|
+
sv = sorted(
|
9624
|
+
[s for s in await tp.get_installable_versions(spec) if s in spec],
|
9625
|
+
key=lambda s: s.version,
|
9608
9626
|
)
|
9627
|
+
if not sv:
|
9628
|
+
return None
|
9629
|
+
|
9630
|
+
bv = sv[-1]
|
9631
|
+
return await tp.install_version(bv)
|
9632
|
+
|
9633
|
+
async def list(self, spec: InterpSpecifier) -> None:
|
9634
|
+
print('installed:')
|
9635
|
+
for n, p in self._providers.items():
|
9636
|
+
lst = [
|
9637
|
+
si
|
9638
|
+
for si in await p.get_installed_versions(spec)
|
9639
|
+
if spec.contains(si)
|
9640
|
+
]
|
9641
|
+
if lst:
|
9642
|
+
print(f' {n}')
|
9643
|
+
for si in lst:
|
9644
|
+
print(f' {si}')
|
9645
|
+
|
9646
|
+
print()
|
9647
|
+
|
9648
|
+
print('installable:')
|
9649
|
+
for n, p in self._providers.items():
|
9650
|
+
lst = [
|
9651
|
+
si
|
9652
|
+
for si in await p.get_installable_versions(spec)
|
9653
|
+
if spec.contains(si)
|
9654
|
+
]
|
9655
|
+
if lst:
|
9656
|
+
print(f' {n}')
|
9657
|
+
for si in lst:
|
9658
|
+
print(f' {si}')
|
9659
|
+
|
9660
|
+
|
9661
|
+
DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
|
9662
|
+
# pyenv is preferred to system interpreters as it tends to have more support for things like tkinter
|
9663
|
+
PyenvInterpProvider(try_update=True),
|
9664
|
+
|
9665
|
+
RunningInterpProvider(),
|
9666
|
+
|
9667
|
+
SystemInterpProvider(),
|
9668
|
+
]])
|
9609
9669
|
|
9610
9670
|
|
9611
9671
|
########################################
|
@@ -9637,6 +9697,36 @@ def bind_targets() -> InjectorBindings:
|
|
9637
9697
|
return inj.as_bindings(*lst)
|
9638
9698
|
|
9639
9699
|
|
9700
|
+
########################################
|
9701
|
+
# ../deploy/interp.py
|
9702
|
+
|
9703
|
+
|
9704
|
+
##
|
9705
|
+
|
9706
|
+
|
9707
|
+
@dc.dataclass(frozen=True)
|
9708
|
+
class InterpCommand(Command['InterpCommand.Output']):
|
9709
|
+
spec: str
|
9710
|
+
install: bool = False
|
9711
|
+
|
9712
|
+
@dc.dataclass(frozen=True)
|
9713
|
+
class Output(Command.Output):
|
9714
|
+
exe: str
|
9715
|
+
version: str
|
9716
|
+
opts: InterpOpts
|
9717
|
+
|
9718
|
+
|
9719
|
+
class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
|
9720
|
+
async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
|
9721
|
+
i = InterpSpecifier.parse(check.not_none(cmd.spec))
|
9722
|
+
o = check.not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
|
9723
|
+
return InterpCommand.Output(
|
9724
|
+
exe=o.exe,
|
9725
|
+
version=str(o.version.version),
|
9726
|
+
opts=o.version.opts,
|
9727
|
+
)
|
9728
|
+
|
9729
|
+
|
9640
9730
|
########################################
|
9641
9731
|
# ../deploy/inject.py
|
9642
9732
|
|