ominfra 0.0.0.dev191__py3-none-any.whl → 0.0.0.dev193__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/configs.py +1 -28
- ominfra/manage/deploy/apps.py +10 -0
- ominfra/manage/deploy/conf/manager.py +10 -4
- ominfra/manage/deploy/conf/specs.py +2 -2
- ominfra/manage/deploy/deploy.py +12 -5
- ominfra/manage/deploy/git.py +3 -1
- ominfra/manage/deploy/specs.py +2 -0
- ominfra/manage/deploy/systemd.py +2 -1
- ominfra/scripts/journald2aws.py +2 -27
- ominfra/scripts/manage.py +1495 -1448
- ominfra/scripts/supervisor.py +1056 -1072
- ominfra/supervisor/configs.py +6 -0
- ominfra/supervisor/http.py +1 -1
- ominfra/supervisor/inject.py +11 -8
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/METADATA +4 -4
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/RECORD +20 -20
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -16,6 +16,7 @@ import asyncio.subprocess
|
|
16
16
|
import base64
|
17
17
|
import collections
|
18
18
|
import collections.abc
|
19
|
+
import configparser
|
19
20
|
import contextlib
|
20
21
|
import contextvars
|
21
22
|
import ctypes as ct
|
@@ -73,17 +74,20 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
|
|
73
74
|
VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
|
74
75
|
VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
|
75
76
|
|
76
|
-
# ../../omdev/toml/parser.py
|
77
|
-
TomlParseFloat = ta.Callable[[str], ta.Any]
|
78
|
-
TomlKey = ta.Tuple[str, ...]
|
79
|
-
TomlPos = int # ta.TypeAlias
|
80
|
-
|
81
77
|
# deploy/paths/types.py
|
82
78
|
DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
|
83
79
|
|
84
80
|
# ../../omlish/asyncs/asyncio/timeouts.py
|
85
81
|
AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
|
86
82
|
|
83
|
+
# ../../omlish/formats/ini/sections.py
|
84
|
+
IniSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
|
85
|
+
|
86
|
+
# ../../omlish/formats/toml/parser.py
|
87
|
+
TomlParseFloat = ta.Callable[[str], ta.Any]
|
88
|
+
TomlKey = ta.Tuple[str, ...]
|
89
|
+
TomlPos = int # ta.TypeAlias
|
90
|
+
|
87
91
|
# ../../omlish/lite/cached.py
|
88
92
|
T = ta.TypeVar('T')
|
89
93
|
CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
@@ -129,7 +133,6 @@ AtomicPathSwapState = ta.Literal['open', 'committed', 'aborted'] # ta.TypeAlias
|
|
129
133
|
|
130
134
|
# ../configs.py
|
131
135
|
ConfigMapping = ta.Mapping[str, ta.Any]
|
132
|
-
IniConfigSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
|
133
136
|
|
134
137
|
# ../../omlish/subprocesses.py
|
135
138
|
SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
|
@@ -549,1482 +552,1523 @@ def canonicalize_version(
|
|
549
552
|
|
550
553
|
|
551
554
|
########################################
|
552
|
-
#
|
553
|
-
# SPDX-License-Identifier: MIT
|
554
|
-
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
555
|
-
# Licensed to PSF under a Contributor Agreement.
|
556
|
-
#
|
557
|
-
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
558
|
-
# --------------------------------------------
|
559
|
-
#
|
560
|
-
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
561
|
-
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
562
|
-
# documentation.
|
563
|
-
#
|
564
|
-
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
565
|
-
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
566
|
-
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
567
|
-
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
568
|
-
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
569
|
-
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
570
|
-
#
|
571
|
-
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
572
|
-
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
573
|
-
# any such work a brief summary of the changes made to Python.
|
574
|
-
#
|
575
|
-
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
576
|
-
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
577
|
-
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
578
|
-
# RIGHTS.
|
579
|
-
#
|
580
|
-
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
581
|
-
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
582
|
-
# ADVISED OF THE POSSIBILITY THEREOF.
|
583
|
-
#
|
584
|
-
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
585
|
-
#
|
586
|
-
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
587
|
-
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
588
|
-
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
589
|
-
#
|
590
|
-
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
591
|
-
# License Agreement.
|
592
|
-
#
|
593
|
-
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
555
|
+
# ../config.py
|
594
556
|
|
595
557
|
|
596
|
-
|
558
|
+
@dc.dataclass(frozen=True)
|
559
|
+
class MainConfig:
|
560
|
+
log_level: ta.Optional[str] = 'INFO'
|
597
561
|
|
562
|
+
debug: bool = False
|
598
563
|
|
599
|
-
_TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
|
600
564
|
|
601
|
-
|
602
|
-
|
603
|
-
0
|
604
|
-
(?:
|
605
|
-
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
606
|
-
|
|
607
|
-
b[01](?:_?[01])* # bin
|
608
|
-
|
|
609
|
-
o[0-7](?:_?[0-7])* # oct
|
610
|
-
)
|
611
|
-
|
|
612
|
-
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
613
|
-
(?P<floatpart>
|
614
|
-
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
615
|
-
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
616
|
-
)
|
617
|
-
""",
|
618
|
-
flags=re.VERBOSE,
|
619
|
-
)
|
620
|
-
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
621
|
-
TOML_RE_DATETIME = re.compile(
|
622
|
-
rf"""
|
623
|
-
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
624
|
-
(?:
|
625
|
-
[Tt ]
|
626
|
-
{_TOML_TIME_RE_STR}
|
627
|
-
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
628
|
-
)?
|
629
|
-
""",
|
630
|
-
flags=re.VERBOSE,
|
631
|
-
)
|
565
|
+
########################################
|
566
|
+
# ../deploy/config.py
|
632
567
|
|
633
568
|
|
634
|
-
|
635
|
-
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
569
|
+
##
|
636
570
|
|
637
|
-
Raises ValueError if the match does not correspond to a valid date or datetime.
|
638
|
-
"""
|
639
|
-
(
|
640
|
-
year_str,
|
641
|
-
month_str,
|
642
|
-
day_str,
|
643
|
-
hour_str,
|
644
|
-
minute_str,
|
645
|
-
sec_str,
|
646
|
-
micros_str,
|
647
|
-
zulu_time,
|
648
|
-
offset_sign_str,
|
649
|
-
offset_hour_str,
|
650
|
-
offset_minute_str,
|
651
|
-
) = match.groups()
|
652
|
-
year, month, day = int(year_str), int(month_str), int(day_str)
|
653
|
-
if hour_str is None:
|
654
|
-
return datetime.date(year, month, day)
|
655
|
-
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
656
|
-
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
657
|
-
if offset_sign_str:
|
658
|
-
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
659
|
-
offset_hour_str, offset_minute_str, offset_sign_str,
|
660
|
-
)
|
661
|
-
elif zulu_time:
|
662
|
-
tz = datetime.UTC
|
663
|
-
else: # local date-time
|
664
|
-
tz = None
|
665
|
-
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
666
571
|
|
572
|
+
@dc.dataclass(frozen=True)
|
573
|
+
class DeployConfig:
|
574
|
+
pass
|
667
575
|
|
668
|
-
@functools.lru_cache() # noqa
|
669
|
-
def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
|
670
|
-
sign = 1 if sign_str == '+' else -1
|
671
|
-
return datetime.timezone(
|
672
|
-
datetime.timedelta(
|
673
|
-
hours=sign * int(hour_str),
|
674
|
-
minutes=sign * int(minute_str),
|
675
|
-
),
|
676
|
-
)
|
677
576
|
|
577
|
+
########################################
|
578
|
+
# ../deploy/paths/types.py
|
678
579
|
|
679
|
-
def toml_match_to_localtime(match: re.Match) -> datetime.time:
|
680
|
-
hour_str, minute_str, sec_str, micros_str = match.groups()
|
681
|
-
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
682
|
-
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
683
580
|
|
581
|
+
##
|
684
582
|
|
685
|
-
def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
|
686
|
-
if match.group('floatpart'):
|
687
|
-
return parse_float(match.group())
|
688
|
-
return int(match.group(), 0)
|
689
583
|
|
584
|
+
########################################
|
585
|
+
# ../deploy/types.py
|
690
586
|
|
691
|
-
TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
|
692
587
|
|
693
|
-
|
694
|
-
# functions.
|
695
|
-
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
696
|
-
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
588
|
+
##
|
697
589
|
|
698
|
-
TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
699
|
-
TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
700
590
|
|
701
|
-
|
591
|
+
DeployHome = ta.NewType('DeployHome', str)
|
702
592
|
|
703
|
-
|
704
|
-
TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
|
705
|
-
TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
|
706
|
-
TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
|
707
|
-
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
593
|
+
DeployRev = ta.NewType('DeployRev', str)
|
708
594
|
|
709
|
-
TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
|
710
|
-
{
|
711
|
-
'\\b': '\u0008', # backspace
|
712
|
-
'\\t': '\u0009', # tab
|
713
|
-
'\\n': '\u000A', # linefeed
|
714
|
-
'\\f': '\u000C', # form feed
|
715
|
-
'\\r': '\u000D', # carriage return
|
716
|
-
'\\"': '\u0022', # quote
|
717
|
-
'\\\\': '\u005C', # backslash
|
718
|
-
},
|
719
|
-
)
|
720
595
|
|
596
|
+
########################################
|
597
|
+
# ../../pyremote.py
|
598
|
+
"""
|
599
|
+
Basically this: https://mitogen.networkgenomics.com/howitworks.html
|
721
600
|
|
722
|
-
|
723
|
-
|
601
|
+
TODO:
|
602
|
+
- log: ta.Optional[logging.Logger] = None + log.debug's
|
603
|
+
"""
|
724
604
|
|
725
605
|
|
726
|
-
|
727
|
-
"""Parse TOML from a binary file object."""
|
728
|
-
b = fp.read()
|
729
|
-
try:
|
730
|
-
s = b.decode()
|
731
|
-
except AttributeError:
|
732
|
-
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
733
|
-
return toml_loads(s, parse_float=parse_float)
|
606
|
+
##
|
734
607
|
|
735
608
|
|
736
|
-
|
737
|
-
|
609
|
+
@dc.dataclass(frozen=True)
|
610
|
+
class PyremoteBootstrapOptions:
|
611
|
+
debug: bool = False
|
738
612
|
|
739
|
-
|
740
|
-
|
741
|
-
src = s.replace('\r\n', '\n')
|
742
|
-
except (AttributeError, TypeError):
|
743
|
-
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
744
|
-
pos = 0
|
745
|
-
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
746
|
-
header: TomlKey = ()
|
747
|
-
parse_float = toml_make_safe_parse_float(parse_float)
|
613
|
+
DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
|
614
|
+
main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
|
748
615
|
|
749
|
-
# Parse one statement at a time (typically means one line in TOML source)
|
750
|
-
while True:
|
751
|
-
# 1. Skip line leading whitespace
|
752
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
753
616
|
|
754
|
-
|
755
|
-
# - end of file
|
756
|
-
# - end of line
|
757
|
-
# - comment
|
758
|
-
# - key/value pair
|
759
|
-
# - append dict to list (and move to its namespace)
|
760
|
-
# - create dict (and move to its namespace)
|
761
|
-
# Skip trailing whitespace when applicable.
|
762
|
-
try:
|
763
|
-
char = src[pos]
|
764
|
-
except IndexError:
|
765
|
-
break
|
766
|
-
if char == '\n':
|
767
|
-
pos += 1
|
768
|
-
continue
|
769
|
-
if char in TOML_KEY_INITIAL_CHARS:
|
770
|
-
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
771
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
772
|
-
elif char == '[':
|
773
|
-
try:
|
774
|
-
second_char: ta.Optional[str] = src[pos + 1]
|
775
|
-
except IndexError:
|
776
|
-
second_char = None
|
777
|
-
out.flags.finalize_pending()
|
778
|
-
if second_char == '[':
|
779
|
-
pos, header = toml_create_list_rule(src, pos, out)
|
780
|
-
else:
|
781
|
-
pos, header = toml_create_dict_rule(src, pos, out)
|
782
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
783
|
-
elif char != '#':
|
784
|
-
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
617
|
+
##
|
785
618
|
|
786
|
-
# 3. Skip comment
|
787
|
-
pos = toml_skip_comment(src, pos)
|
788
619
|
|
789
|
-
|
790
|
-
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
620
|
+
@dc.dataclass(frozen=True)
|
621
|
+
class PyremoteEnvInfo:
|
622
|
+
sys_base_prefix: str
|
623
|
+
sys_byteorder: str
|
624
|
+
sys_defaultencoding: str
|
625
|
+
sys_exec_prefix: str
|
626
|
+
sys_executable: str
|
627
|
+
sys_implementation_name: str
|
628
|
+
sys_path: ta.List[str]
|
629
|
+
sys_platform: str
|
630
|
+
sys_prefix: str
|
631
|
+
sys_version: str
|
632
|
+
sys_version_info: ta.List[ta.Union[int, str]]
|
799
633
|
|
800
|
-
|
634
|
+
platform_architecture: ta.List[str]
|
635
|
+
platform_machine: str
|
636
|
+
platform_platform: str
|
637
|
+
platform_processor: str
|
638
|
+
platform_system: str
|
639
|
+
platform_release: str
|
640
|
+
platform_version: str
|
801
641
|
|
642
|
+
site_userbase: str
|
802
643
|
|
803
|
-
|
804
|
-
|
644
|
+
os_cwd: str
|
645
|
+
os_gid: int
|
646
|
+
os_loadavg: ta.List[float]
|
647
|
+
os_login: ta.Optional[str]
|
648
|
+
os_pgrp: int
|
649
|
+
os_pid: int
|
650
|
+
os_ppid: int
|
651
|
+
os_uid: int
|
805
652
|
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
|
653
|
+
pw_name: str
|
654
|
+
pw_uid: int
|
655
|
+
pw_gid: int
|
656
|
+
pw_gecos: str
|
657
|
+
pw_dir: str
|
658
|
+
pw_shell: str
|
810
659
|
|
811
|
-
|
812
|
-
self._flags: ta.Dict[str, dict] = {}
|
813
|
-
self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
|
660
|
+
env_path: ta.Optional[str]
|
814
661
|
|
815
|
-
def add_pending(self, key: TomlKey, flag: int) -> None:
|
816
|
-
self._pending_flags.add((key, flag))
|
817
662
|
|
818
|
-
|
819
|
-
|
820
|
-
self.set(key, flag, recursive=False)
|
821
|
-
self._pending_flags.clear()
|
663
|
+
def _get_pyremote_env_info() -> PyremoteEnvInfo:
|
664
|
+
os_uid = os.getuid()
|
822
665
|
|
823
|
-
|
824
|
-
cont = self._flags
|
825
|
-
for k in key[:-1]:
|
826
|
-
if k not in cont:
|
827
|
-
return
|
828
|
-
cont = cont[k]['nested']
|
829
|
-
cont.pop(key[-1], None)
|
666
|
+
pw = pwd.getpwuid(os_uid)
|
830
667
|
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
|
835
|
-
|
836
|
-
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
837
|
-
cont = cont[k]['nested']
|
838
|
-
if key_stem not in cont:
|
839
|
-
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
840
|
-
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
668
|
+
os_login: ta.Optional[str]
|
669
|
+
try:
|
670
|
+
os_login = os.getlogin()
|
671
|
+
except OSError:
|
672
|
+
os_login = None
|
841
673
|
|
842
|
-
|
843
|
-
|
844
|
-
|
845
|
-
|
846
|
-
|
847
|
-
|
848
|
-
|
849
|
-
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
if key_stem in cont:
|
855
|
-
cont = cont[key_stem]
|
856
|
-
return flag in cont['flags'] or flag in cont['recursive_flags']
|
857
|
-
return False
|
674
|
+
return PyremoteEnvInfo(
|
675
|
+
sys_base_prefix=sys.base_prefix,
|
676
|
+
sys_byteorder=sys.byteorder,
|
677
|
+
sys_defaultencoding=sys.getdefaultencoding(),
|
678
|
+
sys_exec_prefix=sys.exec_prefix,
|
679
|
+
sys_executable=sys.executable,
|
680
|
+
sys_implementation_name=sys.implementation.name,
|
681
|
+
sys_path=sys.path,
|
682
|
+
sys_platform=sys.platform,
|
683
|
+
sys_prefix=sys.prefix,
|
684
|
+
sys_version=sys.version,
|
685
|
+
sys_version_info=list(sys.version_info),
|
858
686
|
|
687
|
+
platform_architecture=list(platform.architecture()),
|
688
|
+
platform_machine=platform.machine(),
|
689
|
+
platform_platform=platform.platform(),
|
690
|
+
platform_processor=platform.processor(),
|
691
|
+
platform_system=platform.system(),
|
692
|
+
platform_release=platform.release(),
|
693
|
+
platform_version=platform.version(),
|
859
694
|
|
860
|
-
|
861
|
-
def __init__(self) -> None:
|
862
|
-
# The parsed content of the TOML document
|
863
|
-
self.dict: ta.Dict[str, ta.Any] = {}
|
695
|
+
site_userbase=site.getuserbase(),
|
864
696
|
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
if k not in cont:
|
874
|
-
cont[k] = {}
|
875
|
-
cont = cont[k]
|
876
|
-
if access_lists and isinstance(cont, list):
|
877
|
-
cont = cont[-1]
|
878
|
-
if not isinstance(cont, dict):
|
879
|
-
raise KeyError('There is no nest behind this key')
|
880
|
-
return cont
|
697
|
+
os_cwd=os.getcwd(),
|
698
|
+
os_gid=os.getgid(),
|
699
|
+
os_loadavg=list(os.getloadavg()),
|
700
|
+
os_login=os_login,
|
701
|
+
os_pgrp=os.getpgrp(),
|
702
|
+
os_pid=os.getpid(),
|
703
|
+
os_ppid=os.getppid(),
|
704
|
+
os_uid=os_uid,
|
881
705
|
|
882
|
-
|
883
|
-
|
884
|
-
|
885
|
-
|
886
|
-
|
887
|
-
|
888
|
-
raise KeyError('An object other than list found behind this key')
|
889
|
-
list_.append({})
|
890
|
-
else:
|
891
|
-
cont[last_key] = [{}]
|
706
|
+
pw_name=pw.pw_name,
|
707
|
+
pw_uid=pw.pw_uid,
|
708
|
+
pw_gid=pw.pw_gid,
|
709
|
+
pw_gecos=pw.pw_gecos,
|
710
|
+
pw_dir=pw.pw_dir,
|
711
|
+
pw_shell=pw.pw_shell,
|
892
712
|
|
713
|
+
env_path=os.environ.get('PATH'),
|
714
|
+
)
|
893
715
|
|
894
|
-
class TomlOutput(ta.NamedTuple):
|
895
|
-
data: TomlNestedDict
|
896
|
-
flags: TomlFlags
|
897
716
|
|
717
|
+
##
|
898
718
|
|
899
|
-
def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
|
900
|
-
try:
|
901
|
-
while src[pos] in chars:
|
902
|
-
pos += 1
|
903
|
-
except IndexError:
|
904
|
-
pass
|
905
|
-
return pos
|
906
719
|
|
720
|
+
_PYREMOTE_BOOTSTRAP_INPUT_FD = 100
|
721
|
+
_PYREMOTE_BOOTSTRAP_SRC_FD = 101
|
907
722
|
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
|
913
|
-
error_on: ta.FrozenSet[str],
|
914
|
-
error_on_eof: bool,
|
915
|
-
) -> TomlPos:
|
916
|
-
try:
|
917
|
-
new_pos = src.index(expect, pos)
|
918
|
-
except ValueError:
|
919
|
-
new_pos = len(src)
|
920
|
-
if error_on_eof:
|
921
|
-
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
723
|
+
_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
|
724
|
+
_PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
|
725
|
+
_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
|
726
|
+
_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
|
727
|
+
_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
|
922
728
|
|
923
|
-
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
return new_pos
|
729
|
+
_PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
|
730
|
+
_PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
|
731
|
+
_PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
|
732
|
+
_PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
|
928
733
|
|
734
|
+
_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
|
929
735
|
|
930
|
-
|
931
|
-
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
|
937
|
-
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
938
|
-
)
|
939
|
-
return pos
|
736
|
+
_PYREMOTE_BOOTSTRAP_IMPORTS = [
|
737
|
+
'base64',
|
738
|
+
'os',
|
739
|
+
'struct',
|
740
|
+
'sys',
|
741
|
+
'zlib',
|
742
|
+
]
|
940
743
|
|
941
744
|
|
942
|
-
def
|
943
|
-
|
944
|
-
|
945
|
-
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
946
|
-
pos = toml_skip_comment(src, pos)
|
947
|
-
if pos == pos_before_skip:
|
948
|
-
return pos
|
745
|
+
def _pyremote_bootstrap_main(context_name: str) -> None:
|
746
|
+
# Get pid
|
747
|
+
pid = os.getpid()
|
949
748
|
|
749
|
+
# Two copies of payload src to be sent to parent
|
750
|
+
r0, w0 = os.pipe()
|
751
|
+
r1, w1 = os.pipe()
|
950
752
|
|
951
|
-
|
952
|
-
|
953
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
954
|
-
pos, key = toml_parse_key(src, pos)
|
753
|
+
if (cp := os.fork()):
|
754
|
+
# Parent process
|
955
755
|
|
956
|
-
|
957
|
-
|
958
|
-
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
959
|
-
try:
|
960
|
-
out.data.get_or_create_nest(key)
|
961
|
-
except KeyError:
|
962
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
756
|
+
# Dup original stdin to comm_fd for use as comm channel
|
757
|
+
os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
|
963
758
|
|
964
|
-
|
965
|
-
|
966
|
-
return pos + 1, key
|
759
|
+
# Overwrite stdin (fed to python repl) with first copy of src
|
760
|
+
os.dup2(r0, 0)
|
967
761
|
|
762
|
+
# Dup second copy of src to src_fd to recover after launch
|
763
|
+
os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
|
968
764
|
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
pos, key = toml_parse_key(src, pos)
|
973
|
-
|
974
|
-
if out.flags.is_(key, TomlFlags.FROZEN):
|
975
|
-
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
976
|
-
# Free the namespace now that it points to another empty list item...
|
977
|
-
out.flags.unset_all(key)
|
978
|
-
# ...but this key precisely is still prohibited from table declaration
|
979
|
-
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
980
|
-
try:
|
981
|
-
out.data.append_nest_to_list(key)
|
982
|
-
except KeyError:
|
983
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
765
|
+
# Close remaining fd's
|
766
|
+
for f in [r0, w0, r1, w1]:
|
767
|
+
os.close(f)
|
984
768
|
|
985
|
-
|
986
|
-
|
987
|
-
|
769
|
+
# Save vars
|
770
|
+
env = os.environ
|
771
|
+
exe = sys.executable
|
772
|
+
env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
|
773
|
+
env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
|
774
|
+
env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
|
988
775
|
|
776
|
+
# Start repl reading stdin from r0
|
777
|
+
os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
|
989
778
|
|
990
|
-
|
991
|
-
|
992
|
-
pos: TomlPos,
|
993
|
-
out: TomlOutput,
|
994
|
-
header: TomlKey,
|
995
|
-
parse_float: TomlParseFloat,
|
996
|
-
) -> TomlPos:
|
997
|
-
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
998
|
-
key_parent, key_stem = key[:-1], key[-1]
|
999
|
-
abs_key_parent = header + key_parent
|
779
|
+
else:
|
780
|
+
# Child process
|
1000
781
|
|
1001
|
-
|
1002
|
-
|
1003
|
-
# Check that dotted key syntax does not redefine an existing table
|
1004
|
-
if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
|
1005
|
-
raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
|
1006
|
-
# Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
|
1007
|
-
# table sections.
|
1008
|
-
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
782
|
+
# Write first ack
|
783
|
+
os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
|
1009
784
|
|
1010
|
-
|
1011
|
-
|
1012
|
-
src,
|
1013
|
-
pos,
|
1014
|
-
f'Cannot mutate immutable namespace {abs_key_parent}',
|
1015
|
-
)
|
785
|
+
# Write pid
|
786
|
+
os.write(1, struct.pack('<Q', pid))
|
1016
787
|
|
1017
|
-
|
1018
|
-
|
1019
|
-
|
1020
|
-
|
1021
|
-
|
1022
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
1023
|
-
# Mark inline table and array namespaces recursively immutable
|
1024
|
-
if isinstance(value, (dict, list)):
|
1025
|
-
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
1026
|
-
nest[key_stem] = value
|
1027
|
-
return pos
|
788
|
+
# Read payload src from stdin
|
789
|
+
payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
|
790
|
+
if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
|
791
|
+
raise EOFError
|
792
|
+
payload_src = zlib.decompress(payload_z)
|
1028
793
|
|
794
|
+
# Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
|
795
|
+
# fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
796
|
+
for w in [w0, w1]:
|
797
|
+
fp = os.fdopen(w, 'wb', 0)
|
798
|
+
fp.write(payload_src)
|
799
|
+
fp.close()
|
1029
800
|
|
1030
|
-
|
1031
|
-
|
1032
|
-
pos: TomlPos,
|
1033
|
-
parse_float: TomlParseFloat,
|
1034
|
-
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
1035
|
-
pos, key = toml_parse_key(src, pos)
|
1036
|
-
try:
|
1037
|
-
char: ta.Optional[str] = src[pos]
|
1038
|
-
except IndexError:
|
1039
|
-
char = None
|
1040
|
-
if char != '=':
|
1041
|
-
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
1042
|
-
pos += 1
|
1043
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1044
|
-
pos, value = toml_parse_value(src, pos, parse_float)
|
1045
|
-
return pos, key, value
|
801
|
+
# Write second ack
|
802
|
+
os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
|
1046
803
|
|
804
|
+
# Exit child
|
805
|
+
sys.exit(0)
|
1047
806
|
|
1048
|
-
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
1049
|
-
pos, key_part = toml_parse_key_part(src, pos)
|
1050
|
-
key: TomlKey = (key_part,)
|
1051
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1052
|
-
while True:
|
1053
|
-
try:
|
1054
|
-
char: ta.Optional[str] = src[pos]
|
1055
|
-
except IndexError:
|
1056
|
-
char = None
|
1057
|
-
if char != '.':
|
1058
|
-
return pos, key
|
1059
|
-
pos += 1
|
1060
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1061
|
-
pos, key_part = toml_parse_key_part(src, pos)
|
1062
|
-
key += (key_part,)
|
1063
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1064
807
|
|
808
|
+
##
|
1065
809
|
|
1066
|
-
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1067
|
-
try:
|
1068
|
-
char: ta.Optional[str] = src[pos]
|
1069
|
-
except IndexError:
|
1070
|
-
char = None
|
1071
|
-
if char in TOML_BARE_KEY_CHARS:
|
1072
|
-
start_pos = pos
|
1073
|
-
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
1074
|
-
return pos, src[start_pos:pos]
|
1075
|
-
if char == "'":
|
1076
|
-
return toml_parse_literal_str(src, pos)
|
1077
|
-
if char == '"':
|
1078
|
-
return toml_parse_one_line_basic_str(src, pos)
|
1079
|
-
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
1080
810
|
|
811
|
+
def pyremote_build_bootstrap_cmd(context_name: str) -> str:
|
812
|
+
if any(c in context_name for c in '\'"'):
|
813
|
+
raise NameError(context_name)
|
1081
814
|
|
1082
|
-
|
1083
|
-
|
1084
|
-
|
815
|
+
import inspect
|
816
|
+
import textwrap
|
817
|
+
bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
|
1085
818
|
|
819
|
+
for gl in [
|
820
|
+
'_PYREMOTE_BOOTSTRAP_INPUT_FD',
|
821
|
+
'_PYREMOTE_BOOTSTRAP_SRC_FD',
|
1086
822
|
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
823
|
+
'_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
|
824
|
+
'_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
|
825
|
+
'_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
|
1090
826
|
|
1091
|
-
|
1092
|
-
|
1093
|
-
return pos + 1, array
|
1094
|
-
while True:
|
1095
|
-
pos, val = toml_parse_value(src, pos, parse_float)
|
1096
|
-
array.append(val)
|
1097
|
-
pos = toml_skip_comments_and_array_ws(src, pos)
|
827
|
+
'_PYREMOTE_BOOTSTRAP_ACK0',
|
828
|
+
'_PYREMOTE_BOOTSTRAP_ACK1',
|
1098
829
|
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1102
|
-
if c != ',':
|
1103
|
-
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
1104
|
-
pos += 1
|
830
|
+
'_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
|
831
|
+
]:
|
832
|
+
bs_src = bs_src.replace(gl, repr(globals()[gl]))
|
1105
833
|
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
834
|
+
bs_src = '\n'.join(
|
835
|
+
cl
|
836
|
+
for l in bs_src.splitlines()
|
837
|
+
if (cl := (l.split('#')[0]).rstrip())
|
838
|
+
if cl.strip()
|
839
|
+
)
|
1109
840
|
|
841
|
+
bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
|
842
|
+
bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
|
843
|
+
if b'"' in bs_z85:
|
844
|
+
raise ValueError(bs_z85)
|
1110
845
|
|
1111
|
-
|
1112
|
-
|
1113
|
-
|
1114
|
-
|
846
|
+
stmts = [
|
847
|
+
f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
|
848
|
+
f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
|
849
|
+
f'_pyremote_bootstrap_main("{context_name}")',
|
850
|
+
]
|
1115
851
|
|
1116
|
-
|
1117
|
-
|
1118
|
-
return pos + 1, nested_dict.dict
|
1119
|
-
while True:
|
1120
|
-
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1121
|
-
key_parent, key_stem = key[:-1], key[-1]
|
1122
|
-
if flags.is_(key, TomlFlags.FROZEN):
|
1123
|
-
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1124
|
-
try:
|
1125
|
-
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
1126
|
-
except KeyError:
|
1127
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1128
|
-
if key_stem in nest:
|
1129
|
-
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
1130
|
-
nest[key_stem] = value
|
1131
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1132
|
-
c = src[pos:pos + 1]
|
1133
|
-
if c == '}':
|
1134
|
-
return pos + 1, nested_dict.dict
|
1135
|
-
if c != ',':
|
1136
|
-
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
1137
|
-
if isinstance(value, (dict, list)):
|
1138
|
-
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
1139
|
-
pos += 1
|
1140
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
852
|
+
cmd = '; '.join(stmts)
|
853
|
+
return cmd
|
1141
854
|
|
1142
855
|
|
1143
|
-
|
1144
|
-
src: str,
|
1145
|
-
pos: TomlPos,
|
1146
|
-
*,
|
1147
|
-
multiline: bool = False,
|
1148
|
-
) -> ta.Tuple[TomlPos, str]:
|
1149
|
-
escape_id = src[pos:pos + 2]
|
1150
|
-
pos += 2
|
1151
|
-
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
1152
|
-
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
1153
|
-
# newline.
|
1154
|
-
if escape_id != '\\\n':
|
1155
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1156
|
-
try:
|
1157
|
-
char = src[pos]
|
1158
|
-
except IndexError:
|
1159
|
-
return pos, ''
|
1160
|
-
if char != '\n':
|
1161
|
-
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
1162
|
-
pos += 1
|
1163
|
-
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1164
|
-
return pos, ''
|
1165
|
-
if escape_id == '\\u':
|
1166
|
-
return toml_parse_hex_char(src, pos, 4)
|
1167
|
-
if escape_id == '\\U':
|
1168
|
-
return toml_parse_hex_char(src, pos, 8)
|
1169
|
-
try:
|
1170
|
-
return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
|
1171
|
-
except KeyError:
|
1172
|
-
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
856
|
+
##
|
1173
857
|
|
1174
858
|
|
1175
|
-
|
1176
|
-
|
859
|
+
@dc.dataclass(frozen=True)
|
860
|
+
class PyremotePayloadRuntime:
|
861
|
+
input: ta.BinaryIO
|
862
|
+
output: ta.BinaryIO
|
863
|
+
context_name: str
|
864
|
+
payload_src: str
|
865
|
+
options: PyremoteBootstrapOptions
|
866
|
+
env_info: PyremoteEnvInfo
|
1177
867
|
|
1178
868
|
|
1179
|
-
def
|
1180
|
-
|
1181
|
-
if
|
1182
|
-
|
1183
|
-
|
1184
|
-
|
1185
|
-
|
1186
|
-
|
1187
|
-
|
869
|
+
def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
870
|
+
# If src file var is not present we need to do initial finalization
|
871
|
+
if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
|
872
|
+
# Read second copy of payload src
|
873
|
+
r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
|
874
|
+
payload_src = r1.read().decode('utf-8')
|
875
|
+
r1.close()
|
876
|
+
|
877
|
+
# Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
|
878
|
+
# a pipe at once.
|
879
|
+
os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
|
1188
880
|
|
881
|
+
# Read options
|
882
|
+
options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
|
883
|
+
if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
|
884
|
+
raise EOFError
|
885
|
+
options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
|
1189
886
|
|
1190
|
-
|
1191
|
-
|
1192
|
-
|
1193
|
-
|
1194
|
-
|
1195
|
-
|
1196
|
-
|
887
|
+
# If debugging, re-exec as file
|
888
|
+
if options.debug:
|
889
|
+
# Write temp source file
|
890
|
+
import tempfile
|
891
|
+
tfd, tfn = tempfile.mkstemp('-pyremote.py')
|
892
|
+
os.write(tfd, payload_src.encode('utf-8'))
|
893
|
+
os.close(tfd)
|
1197
894
|
|
895
|
+
# Set vars
|
896
|
+
os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
|
897
|
+
os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
|
1198
898
|
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
899
|
+
# Re-exec temp file
|
900
|
+
exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
|
901
|
+
context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
|
902
|
+
os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
|
1203
903
|
|
1204
|
-
if literal:
|
1205
|
-
delim = "'"
|
1206
|
-
end_pos = toml_skip_until(
|
1207
|
-
src,
|
1208
|
-
pos,
|
1209
|
-
"'''",
|
1210
|
-
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
1211
|
-
error_on_eof=True,
|
1212
|
-
)
|
1213
|
-
result = src[pos:end_pos]
|
1214
|
-
pos = end_pos + 3
|
1215
904
|
else:
|
1216
|
-
|
1217
|
-
|
905
|
+
# Load options json var
|
906
|
+
options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
|
907
|
+
options = PyremoteBootstrapOptions(**json.loads(options_json_str))
|
1218
908
|
|
1219
|
-
|
1220
|
-
|
1221
|
-
|
1222
|
-
pos += 1
|
1223
|
-
if not src.startswith(delim, pos):
|
1224
|
-
return pos, result + delim
|
1225
|
-
pos += 1
|
1226
|
-
return pos, result + (delim * 2)
|
909
|
+
# Read temp source file
|
910
|
+
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
911
|
+
payload_src = sf.read()
|
1227
912
|
|
913
|
+
# Restore vars
|
914
|
+
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
915
|
+
context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
|
1228
916
|
|
1229
|
-
|
1230
|
-
|
1231
|
-
error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1232
|
-
parse_escapes = toml_parse_basic_str_escape_multiline
|
1233
|
-
else:
|
1234
|
-
error_on = TOML_ILLEGAL_BASIC_STR_CHARS
|
1235
|
-
parse_escapes = toml_parse_basic_str_escape
|
1236
|
-
result = ''
|
1237
|
-
start_pos = pos
|
1238
|
-
while True:
|
1239
|
-
try:
|
1240
|
-
char = src[pos]
|
1241
|
-
except IndexError:
|
1242
|
-
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
1243
|
-
if char == '"':
|
1244
|
-
if not multiline:
|
1245
|
-
return pos + 1, result + src[start_pos:pos]
|
1246
|
-
if src.startswith('"""', pos):
|
1247
|
-
return pos + 3, result + src[start_pos:pos]
|
1248
|
-
pos += 1
|
1249
|
-
continue
|
1250
|
-
if char == '\\':
|
1251
|
-
result += src[start_pos:pos]
|
1252
|
-
pos, parsed_escape = parse_escapes(src, pos)
|
1253
|
-
result += parsed_escape
|
1254
|
-
start_pos = pos
|
1255
|
-
continue
|
1256
|
-
if char in error_on:
|
1257
|
-
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
1258
|
-
pos += 1
|
917
|
+
# Write third ack
|
918
|
+
os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
|
1259
919
|
|
920
|
+
# Write env info
|
921
|
+
env_info = _get_pyremote_env_info()
|
922
|
+
env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
|
923
|
+
os.write(1, struct.pack('<I', len(env_info_json)))
|
924
|
+
os.write(1, env_info_json.encode('utf-8'))
|
1260
925
|
|
1261
|
-
|
1262
|
-
|
1263
|
-
|
1264
|
-
|
1265
|
-
)
|
1266
|
-
try:
|
1267
|
-
char: ta.Optional[str] = src[pos]
|
1268
|
-
except IndexError:
|
1269
|
-
char = None
|
926
|
+
# Setup IO
|
927
|
+
input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
|
928
|
+
output = os.fdopen(os.dup(1), 'wb', 0) # noqa
|
929
|
+
os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
|
930
|
+
os.close(nfd)
|
1270
931
|
|
1271
|
-
|
932
|
+
if (mn := options.main_name_override) is not None:
|
933
|
+
# Inspections like typing.get_type_hints need an entry in sys.modules.
|
934
|
+
sys.modules[mn] = sys.modules['__main__']
|
1272
935
|
|
1273
|
-
#
|
1274
|
-
|
1275
|
-
if src.startswith('"""', pos):
|
1276
|
-
return toml_parse_multiline_str(src, pos, literal=False)
|
1277
|
-
return toml_parse_one_line_basic_str(src, pos)
|
936
|
+
# Write fourth ack
|
937
|
+
output.write(_PYREMOTE_BOOTSTRAP_ACK3)
|
1278
938
|
|
1279
|
-
#
|
1280
|
-
|
1281
|
-
|
1282
|
-
|
1283
|
-
|
939
|
+
# Return
|
940
|
+
return PyremotePayloadRuntime(
|
941
|
+
input=input,
|
942
|
+
output=output,
|
943
|
+
context_name=context_name,
|
944
|
+
payload_src=payload_src,
|
945
|
+
options=options,
|
946
|
+
env_info=env_info,
|
947
|
+
)
|
1284
948
|
|
1285
|
-
# Booleans
|
1286
|
-
if char == 't':
|
1287
|
-
if src.startswith('true', pos):
|
1288
|
-
return pos + 4, True
|
1289
|
-
if char == 'f':
|
1290
|
-
if src.startswith('false', pos):
|
1291
|
-
return pos + 5, False
|
1292
949
|
|
1293
|
-
|
1294
|
-
if char == '[':
|
1295
|
-
return toml_parse_array(src, pos, parse_float)
|
950
|
+
##
|
1296
951
|
|
1297
|
-
# Inline tables
|
1298
|
-
if char == '{':
|
1299
|
-
return toml_parse_inline_table(src, pos, parse_float)
|
1300
952
|
|
1301
|
-
|
1302
|
-
|
1303
|
-
|
1304
|
-
|
1305
|
-
|
1306
|
-
|
1307
|
-
|
1308
|
-
return datetime_match.end(), datetime_obj
|
1309
|
-
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
1310
|
-
if localtime_match:
|
1311
|
-
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
953
|
+
class PyremoteBootstrapDriver:
|
954
|
+
def __init__(
|
955
|
+
self,
|
956
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
957
|
+
options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
|
958
|
+
) -> None:
|
959
|
+
super().__init__()
|
1312
960
|
|
1313
|
-
|
1314
|
-
|
1315
|
-
number_match = TOML_RE_NUMBER.match(src, pos)
|
1316
|
-
if number_match:
|
1317
|
-
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
961
|
+
self._payload_src = payload_src
|
962
|
+
self._options = options
|
1318
963
|
|
1319
|
-
|
1320
|
-
|
1321
|
-
if first_three in {'inf', 'nan'}:
|
1322
|
-
return pos + 3, parse_float(first_three)
|
1323
|
-
first_four = src[pos:pos + 4]
|
1324
|
-
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
1325
|
-
return pos + 4, parse_float(first_four)
|
964
|
+
self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
|
965
|
+
self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
|
1326
966
|
|
1327
|
-
|
967
|
+
self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
|
968
|
+
#
|
1328
969
|
|
970
|
+
@classmethod
|
971
|
+
def _prepare_payload_src(
|
972
|
+
cls,
|
973
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
974
|
+
options: PyremoteBootstrapOptions,
|
975
|
+
) -> str:
|
976
|
+
parts: ta.List[str]
|
977
|
+
if isinstance(payload_src, str):
|
978
|
+
parts = [payload_src]
|
979
|
+
else:
|
980
|
+
parts = list(payload_src)
|
1329
981
|
|
1330
|
-
|
1331
|
-
|
982
|
+
if (mn := options.main_name_override) is not None:
|
983
|
+
parts.insert(0, f'__name__ = {mn!r}')
|
1332
984
|
|
1333
|
-
|
1334
|
-
|
1335
|
-
return 'end of document'
|
1336
|
-
line = src.count('\n', 0, pos) + 1
|
1337
|
-
if line == 1:
|
1338
|
-
column = pos + 1
|
985
|
+
if len(parts) == 1:
|
986
|
+
return parts[0]
|
1339
987
|
else:
|
1340
|
-
|
1341
|
-
return f'line {line}, column {column}'
|
988
|
+
return '\n\n'.join(parts)
|
1342
989
|
|
1343
|
-
|
990
|
+
#
|
1344
991
|
|
992
|
+
@dc.dataclass(frozen=True)
|
993
|
+
class Read:
|
994
|
+
sz: int
|
1345
995
|
|
1346
|
-
|
1347
|
-
|
996
|
+
@dc.dataclass(frozen=True)
|
997
|
+
class Write:
|
998
|
+
d: bytes
|
1348
999
|
|
1000
|
+
class ProtocolError(Exception):
|
1001
|
+
pass
|
1349
1002
|
|
1350
|
-
|
1351
|
-
|
1003
|
+
@dc.dataclass(frozen=True)
|
1004
|
+
class Result:
|
1005
|
+
pid: int
|
1006
|
+
env_info: PyremoteEnvInfo
|
1352
1007
|
|
1353
|
-
|
1354
|
-
|
1355
|
-
|
1356
|
-
# The default `float` callable never returns illegal types. Optimize it.
|
1357
|
-
if parse_float is float:
|
1358
|
-
return float
|
1008
|
+
def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
|
1009
|
+
# Read first ack (after fork)
|
1010
|
+
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
|
1359
1011
|
|
1360
|
-
|
1361
|
-
|
1362
|
-
|
1363
|
-
raise ValueError('parse_float must not return dicts or lists') # noqa
|
1364
|
-
return float_value
|
1012
|
+
# Read pid
|
1013
|
+
d = yield from self._read(8)
|
1014
|
+
pid = struct.unpack('<Q', d)[0]
|
1365
1015
|
|
1366
|
-
|
1016
|
+
# Write payload src
|
1017
|
+
yield from self._write(struct.pack('<I', len(self._payload_z)))
|
1018
|
+
yield from self._write(self._payload_z)
|
1367
1019
|
|
1020
|
+
# Read second ack (after writing src copies)
|
1021
|
+
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
|
1368
1022
|
|
1369
|
-
|
1370
|
-
|
1023
|
+
# Write options
|
1024
|
+
yield from self._write(struct.pack('<I', len(self._options_json)))
|
1025
|
+
yield from self._write(self._options_json)
|
1371
1026
|
|
1027
|
+
# Read third ack (after reaping child process)
|
1028
|
+
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
|
1372
1029
|
|
1373
|
-
|
1374
|
-
|
1375
|
-
|
1030
|
+
# Read env info
|
1031
|
+
d = yield from self._read(4)
|
1032
|
+
env_info_json_len = struct.unpack('<I', d)[0]
|
1033
|
+
d = yield from self._read(env_info_json_len)
|
1034
|
+
env_info_json = d.decode('utf-8')
|
1035
|
+
env_info = PyremoteEnvInfo(**json.loads(env_info_json))
|
1376
1036
|
|
1377
|
-
|
1037
|
+
# Read fourth ack (after finalization completed)
|
1038
|
+
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
|
1378
1039
|
|
1040
|
+
# Return
|
1041
|
+
return self.Result(
|
1042
|
+
pid=pid,
|
1043
|
+
env_info=env_info,
|
1044
|
+
)
|
1379
1045
|
|
1380
|
-
|
1381
|
-
|
1046
|
+
def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
|
1047
|
+
d = yield self.Read(sz)
|
1048
|
+
if not isinstance(d, bytes):
|
1049
|
+
raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
|
1050
|
+
if len(d) != sz:
|
1051
|
+
raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
|
1052
|
+
return d
|
1382
1053
|
|
1054
|
+
def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
|
1055
|
+
d = yield from self._read(len(e))
|
1056
|
+
if d != e:
|
1057
|
+
raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
|
1383
1058
|
|
1384
|
-
|
1059
|
+
def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
|
1060
|
+
i = yield self.Write(d)
|
1061
|
+
if i is not None:
|
1062
|
+
raise self.ProtocolError('Unexpected input after write')
|
1385
1063
|
|
1064
|
+
#
|
1386
1065
|
|
1387
|
-
|
1388
|
-
|
1389
|
-
pass
|
1066
|
+
def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
|
1067
|
+
gen = self.gen()
|
1390
1068
|
|
1069
|
+
gi: ta.Optional[bytes] = None
|
1070
|
+
while True:
|
1071
|
+
try:
|
1072
|
+
if gi is not None:
|
1073
|
+
go = gen.send(gi)
|
1074
|
+
else:
|
1075
|
+
go = next(gen)
|
1076
|
+
except StopIteration as e:
|
1077
|
+
return e.value
|
1391
1078
|
|
1392
|
-
|
1393
|
-
|
1079
|
+
if isinstance(go, self.Read):
|
1080
|
+
if len(gi := input.read(go.sz)) != go.sz:
|
1081
|
+
raise EOFError
|
1082
|
+
elif isinstance(go, self.Write):
|
1083
|
+
gi = None
|
1084
|
+
output.write(go.d)
|
1085
|
+
output.flush()
|
1086
|
+
else:
|
1087
|
+
raise TypeError(go)
|
1394
1088
|
|
1089
|
+
async def async_run(
|
1090
|
+
self,
|
1091
|
+
input: ta.Any, # asyncio.StreamWriter # noqa
|
1092
|
+
output: ta.Any, # asyncio.StreamReader
|
1093
|
+
) -> Result:
|
1094
|
+
gen = self.gen()
|
1395
1095
|
|
1396
|
-
|
1096
|
+
gi: ta.Optional[bytes] = None
|
1097
|
+
while True:
|
1098
|
+
try:
|
1099
|
+
if gi is not None:
|
1100
|
+
go = gen.send(gi)
|
1101
|
+
else:
|
1102
|
+
go = next(gen)
|
1103
|
+
except StopIteration as e:
|
1104
|
+
return e.value
|
1105
|
+
|
1106
|
+
if isinstance(go, self.Read):
|
1107
|
+
if len(gi := await input.read(go.sz)) != go.sz:
|
1108
|
+
raise EOFError
|
1109
|
+
elif isinstance(go, self.Write):
|
1110
|
+
gi = None
|
1111
|
+
output.write(go.d)
|
1112
|
+
await output.drain()
|
1113
|
+
else:
|
1114
|
+
raise TypeError(go)
|
1397
1115
|
|
1398
1116
|
|
1399
1117
|
########################################
|
1400
|
-
#
|
1118
|
+
# ../../../omlish/asyncs/asyncio/channels.py
|
1401
1119
|
|
1402
1120
|
|
1403
|
-
|
1121
|
+
class AsyncioBytesChannelTransport(asyncio.Transport):
|
1122
|
+
def __init__(self, reader: asyncio.StreamReader) -> None:
|
1123
|
+
super().__init__()
|
1404
1124
|
|
1125
|
+
self.reader = reader
|
1126
|
+
self.closed: asyncio.Future = asyncio.Future()
|
1405
1127
|
|
1406
|
-
|
1128
|
+
# @ta.override
|
1129
|
+
def write(self, data: bytes) -> None:
|
1130
|
+
self.reader.feed_data(data)
|
1407
1131
|
|
1408
|
-
|
1132
|
+
# @ta.override
|
1133
|
+
def close(self) -> None:
|
1134
|
+
self.reader.feed_eof()
|
1135
|
+
if not self.closed.done():
|
1136
|
+
self.closed.set_result(True)
|
1409
1137
|
|
1138
|
+
# @ta.override
|
1139
|
+
def is_closing(self) -> bool:
|
1140
|
+
return self.closed.done()
|
1410
1141
|
|
1411
|
-
########################################
|
1412
|
-
# ../../pyremote.py
|
1413
|
-
"""
|
1414
|
-
Basically this: https://mitogen.networkgenomics.com/howitworks.html
|
1415
1142
|
|
1416
|
-
|
1417
|
-
|
1418
|
-
|
1143
|
+
def asyncio_create_bytes_channel(
|
1144
|
+
loop: ta.Any = None,
|
1145
|
+
) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
|
1146
|
+
if loop is None:
|
1147
|
+
loop = asyncio.get_running_loop()
|
1419
1148
|
|
1149
|
+
reader = asyncio.StreamReader()
|
1150
|
+
protocol = asyncio.StreamReaderProtocol(reader)
|
1151
|
+
transport = AsyncioBytesChannelTransport(reader)
|
1152
|
+
writer = asyncio.StreamWriter(transport, protocol, reader, loop)
|
1420
1153
|
|
1421
|
-
|
1154
|
+
return reader, writer
|
1422
1155
|
|
1423
1156
|
|
1424
|
-
|
1425
|
-
|
1426
|
-
debug: bool = False
|
1157
|
+
########################################
|
1158
|
+
# ../../../omlish/asyncs/asyncio/streams.py
|
1427
1159
|
|
1428
|
-
DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
|
1429
|
-
main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
|
1430
1160
|
|
1161
|
+
ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
|
1431
1162
|
|
1432
|
-
##
|
1433
1163
|
|
1164
|
+
async def asyncio_open_stream_reader(
|
1165
|
+
f: ta.IO,
|
1166
|
+
loop: ta.Any = None,
|
1167
|
+
*,
|
1168
|
+
limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
|
1169
|
+
) -> asyncio.StreamReader:
|
1170
|
+
if loop is None:
|
1171
|
+
loop = asyncio.get_running_loop()
|
1434
1172
|
|
1435
|
-
|
1436
|
-
|
1437
|
-
|
1438
|
-
|
1439
|
-
|
1440
|
-
sys_exec_prefix: str
|
1441
|
-
sys_executable: str
|
1442
|
-
sys_implementation_name: str
|
1443
|
-
sys_path: ta.List[str]
|
1444
|
-
sys_platform: str
|
1445
|
-
sys_prefix: str
|
1446
|
-
sys_version: str
|
1447
|
-
sys_version_info: ta.List[ta.Union[int, str]]
|
1173
|
+
reader = asyncio.StreamReader(limit=limit, loop=loop)
|
1174
|
+
await loop.connect_read_pipe(
|
1175
|
+
lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
|
1176
|
+
f,
|
1177
|
+
)
|
1448
1178
|
|
1449
|
-
|
1450
|
-
platform_machine: str
|
1451
|
-
platform_platform: str
|
1452
|
-
platform_processor: str
|
1453
|
-
platform_system: str
|
1454
|
-
platform_release: str
|
1455
|
-
platform_version: str
|
1179
|
+
return reader
|
1456
1180
|
|
1457
|
-
site_userbase: str
|
1458
1181
|
|
1459
|
-
|
1460
|
-
|
1461
|
-
|
1462
|
-
|
1463
|
-
|
1464
|
-
|
1465
|
-
os_ppid: int
|
1466
|
-
os_uid: int
|
1182
|
+
async def asyncio_open_stream_writer(
|
1183
|
+
f: ta.IO,
|
1184
|
+
loop: ta.Any = None,
|
1185
|
+
) -> asyncio.StreamWriter:
|
1186
|
+
if loop is None:
|
1187
|
+
loop = asyncio.get_running_loop()
|
1467
1188
|
|
1468
|
-
|
1469
|
-
|
1470
|
-
|
1471
|
-
|
1472
|
-
pw_dir: str
|
1473
|
-
pw_shell: str
|
1189
|
+
writer_transport, writer_protocol = await loop.connect_write_pipe(
|
1190
|
+
lambda: asyncio.streams.FlowControlMixin(loop=loop),
|
1191
|
+
f,
|
1192
|
+
)
|
1474
1193
|
|
1475
|
-
|
1194
|
+
return asyncio.streams.StreamWriter(
|
1195
|
+
writer_transport,
|
1196
|
+
writer_protocol,
|
1197
|
+
None,
|
1198
|
+
loop,
|
1199
|
+
)
|
1476
1200
|
|
1477
1201
|
|
1478
|
-
|
1479
|
-
|
1202
|
+
########################################
|
1203
|
+
# ../../../omlish/asyncs/asyncio/timeouts.py
|
1480
1204
|
|
1481
|
-
pw = pwd.getpwuid(os_uid)
|
1482
1205
|
|
1483
|
-
|
1484
|
-
|
1485
|
-
|
1486
|
-
|
1487
|
-
|
1206
|
+
def asyncio_maybe_timeout(
|
1207
|
+
fut: AwaitableT,
|
1208
|
+
timeout: ta.Optional[float] = None,
|
1209
|
+
) -> AwaitableT:
|
1210
|
+
if timeout is not None:
|
1211
|
+
fut = asyncio.wait_for(fut, timeout) # type: ignore
|
1212
|
+
return fut
|
1488
1213
|
|
1489
|
-
return PyremoteEnvInfo(
|
1490
|
-
sys_base_prefix=sys.base_prefix,
|
1491
|
-
sys_byteorder=sys.byteorder,
|
1492
|
-
sys_defaultencoding=sys.getdefaultencoding(),
|
1493
|
-
sys_exec_prefix=sys.exec_prefix,
|
1494
|
-
sys_executable=sys.executable,
|
1495
|
-
sys_implementation_name=sys.implementation.name,
|
1496
|
-
sys_path=sys.path,
|
1497
|
-
sys_platform=sys.platform,
|
1498
|
-
sys_prefix=sys.prefix,
|
1499
|
-
sys_version=sys.version,
|
1500
|
-
sys_version_info=list(sys.version_info),
|
1501
1214
|
|
1502
|
-
|
1503
|
-
|
1504
|
-
platform_platform=platform.platform(),
|
1505
|
-
platform_processor=platform.processor(),
|
1506
|
-
platform_system=platform.system(),
|
1507
|
-
platform_release=platform.release(),
|
1508
|
-
platform_version=platform.version(),
|
1215
|
+
########################################
|
1216
|
+
# ../../../omlish/formats/ini/sections.py
|
1509
1217
|
|
1510
|
-
site_userbase=site.getuserbase(),
|
1511
1218
|
|
1512
|
-
|
1513
|
-
os_gid=os.getgid(),
|
1514
|
-
os_loadavg=list(os.getloadavg()),
|
1515
|
-
os_login=os_login,
|
1516
|
-
os_pgrp=os.getpgrp(),
|
1517
|
-
os_pid=os.getpid(),
|
1518
|
-
os_ppid=os.getppid(),
|
1519
|
-
os_uid=os_uid,
|
1219
|
+
##
|
1520
1220
|
|
1521
|
-
pw_name=pw.pw_name,
|
1522
|
-
pw_uid=pw.pw_uid,
|
1523
|
-
pw_gid=pw.pw_gid,
|
1524
|
-
pw_gecos=pw.pw_gecos,
|
1525
|
-
pw_dir=pw.pw_dir,
|
1526
|
-
pw_shell=pw.pw_shell,
|
1527
1221
|
|
1528
|
-
|
1529
|
-
|
1222
|
+
def extract_ini_sections(cp: configparser.ConfigParser) -> IniSectionSettingsMap:
|
1223
|
+
config_dct: ta.Dict[str, ta.Any] = {}
|
1224
|
+
for sec in cp.sections():
|
1225
|
+
cd = config_dct
|
1226
|
+
for k in sec.split('.'):
|
1227
|
+
cd = cd.setdefault(k, {})
|
1228
|
+
cd.update(cp.items(sec))
|
1229
|
+
return config_dct
|
1530
1230
|
|
1531
1231
|
|
1532
1232
|
##
|
1533
1233
|
|
1534
1234
|
|
1535
|
-
|
1536
|
-
|
1537
|
-
|
1538
|
-
|
1539
|
-
_PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
|
1540
|
-
_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
|
1541
|
-
_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
|
1542
|
-
_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
|
1543
|
-
|
1544
|
-
_PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
|
1545
|
-
_PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
|
1546
|
-
_PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
|
1547
|
-
_PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
|
1548
|
-
|
1549
|
-
_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
|
1235
|
+
def render_ini_sections(
|
1236
|
+
settings_by_section: IniSectionSettingsMap,
|
1237
|
+
) -> str:
|
1238
|
+
out = io.StringIO()
|
1550
1239
|
|
1551
|
-
|
1552
|
-
|
1553
|
-
|
1554
|
-
'struct',
|
1555
|
-
'sys',
|
1556
|
-
'zlib',
|
1557
|
-
]
|
1240
|
+
for i, (section, settings) in enumerate(settings_by_section.items()):
|
1241
|
+
if i:
|
1242
|
+
out.write('\n')
|
1558
1243
|
|
1244
|
+
out.write(f'[{section}]\n')
|
1559
1245
|
|
1560
|
-
|
1561
|
-
|
1562
|
-
|
1246
|
+
for k, v in settings.items():
|
1247
|
+
if isinstance(v, str):
|
1248
|
+
out.write(f'{k}={v}\n')
|
1249
|
+
else:
|
1250
|
+
for vv in v:
|
1251
|
+
out.write(f'{k}={vv}\n')
|
1563
1252
|
|
1564
|
-
|
1565
|
-
r0, w0 = os.pipe()
|
1566
|
-
r1, w1 = os.pipe()
|
1253
|
+
return out.getvalue()
|
1567
1254
|
|
1568
|
-
if (cp := os.fork()):
|
1569
|
-
# Parent process
|
1570
1255
|
|
1571
|
-
|
1572
|
-
|
1256
|
+
########################################
|
1257
|
+
# ../../../omlish/formats/toml/parser.py
|
1258
|
+
# SPDX-License-Identifier: MIT
|
1259
|
+
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
1260
|
+
# Licensed to PSF under a Contributor Agreement.
|
1261
|
+
#
|
1262
|
+
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
1263
|
+
# --------------------------------------------
|
1264
|
+
#
|
1265
|
+
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
1266
|
+
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
1267
|
+
# documentation.
|
1268
|
+
#
|
1269
|
+
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
1270
|
+
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
1271
|
+
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
1272
|
+
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
1273
|
+
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
1274
|
+
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
1275
|
+
#
|
1276
|
+
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
1277
|
+
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
1278
|
+
# any such work a brief summary of the changes made to Python.
|
1279
|
+
#
|
1280
|
+
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
1281
|
+
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
1282
|
+
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
1283
|
+
# RIGHTS.
|
1284
|
+
#
|
1285
|
+
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
1286
|
+
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
1287
|
+
# ADVISED OF THE POSSIBILITY THEREOF.
|
1288
|
+
#
|
1289
|
+
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
1290
|
+
#
|
1291
|
+
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
1292
|
+
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
1293
|
+
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
1294
|
+
#
|
1295
|
+
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
1296
|
+
# License Agreement.
|
1297
|
+
#
|
1298
|
+
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
1573
1299
|
|
1574
|
-
# Overwrite stdin (fed to python repl) with first copy of src
|
1575
|
-
os.dup2(r0, 0)
|
1576
1300
|
|
1577
|
-
|
1578
|
-
os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
|
1301
|
+
##
|
1579
1302
|
|
1580
|
-
# Close remaining fd's
|
1581
|
-
for f in [r0, w0, r1, w1]:
|
1582
|
-
os.close(f)
|
1583
1303
|
|
1584
|
-
|
1585
|
-
env = os.environ
|
1586
|
-
exe = sys.executable
|
1587
|
-
env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
|
1588
|
-
env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
|
1589
|
-
env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
|
1304
|
+
_TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
|
1590
1305
|
|
1591
|
-
|
1592
|
-
|
1306
|
+
TOML_RE_NUMBER = re.compile(
|
1307
|
+
r"""
|
1308
|
+
0
|
1309
|
+
(?:
|
1310
|
+
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
1311
|
+
|
|
1312
|
+
b[01](?:_?[01])* # bin
|
1313
|
+
|
|
1314
|
+
o[0-7](?:_?[0-7])* # oct
|
1315
|
+
)
|
1316
|
+
|
|
1317
|
+
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
1318
|
+
(?P<floatpart>
|
1319
|
+
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
1320
|
+
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
1321
|
+
)
|
1322
|
+
""",
|
1323
|
+
flags=re.VERBOSE,
|
1324
|
+
)
|
1325
|
+
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
1326
|
+
TOML_RE_DATETIME = re.compile(
|
1327
|
+
rf"""
|
1328
|
+
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
1329
|
+
(?:
|
1330
|
+
[Tt ]
|
1331
|
+
{_TOML_TIME_RE_STR}
|
1332
|
+
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
1333
|
+
)?
|
1334
|
+
""",
|
1335
|
+
flags=re.VERBOSE,
|
1336
|
+
)
|
1593
1337
|
|
1594
|
-
else:
|
1595
|
-
# Child process
|
1596
1338
|
|
1597
|
-
|
1598
|
-
|
1339
|
+
def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
|
1340
|
+
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
1599
1341
|
|
1600
|
-
|
1601
|
-
|
1342
|
+
Raises ValueError if the match does not correspond to a valid date or datetime.
|
1343
|
+
"""
|
1344
|
+
(
|
1345
|
+
year_str,
|
1346
|
+
month_str,
|
1347
|
+
day_str,
|
1348
|
+
hour_str,
|
1349
|
+
minute_str,
|
1350
|
+
sec_str,
|
1351
|
+
micros_str,
|
1352
|
+
zulu_time,
|
1353
|
+
offset_sign_str,
|
1354
|
+
offset_hour_str,
|
1355
|
+
offset_minute_str,
|
1356
|
+
) = match.groups()
|
1357
|
+
year, month, day = int(year_str), int(month_str), int(day_str)
|
1358
|
+
if hour_str is None:
|
1359
|
+
return datetime.date(year, month, day)
|
1360
|
+
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
1361
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
1362
|
+
if offset_sign_str:
|
1363
|
+
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
1364
|
+
offset_hour_str, offset_minute_str, offset_sign_str,
|
1365
|
+
)
|
1366
|
+
elif zulu_time:
|
1367
|
+
tz = datetime.UTC
|
1368
|
+
else: # local date-time
|
1369
|
+
tz = None
|
1370
|
+
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
1602
1371
|
|
1603
|
-
# Read payload src from stdin
|
1604
|
-
payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
|
1605
|
-
if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
|
1606
|
-
raise EOFError
|
1607
|
-
payload_src = zlib.decompress(payload_z)
|
1608
1372
|
|
1609
|
-
|
1610
|
-
|
1611
|
-
|
1612
|
-
|
1613
|
-
|
1614
|
-
|
1373
|
+
@functools.lru_cache() # noqa
|
1374
|
+
def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
|
1375
|
+
sign = 1 if sign_str == '+' else -1
|
1376
|
+
return datetime.timezone(
|
1377
|
+
datetime.timedelta(
|
1378
|
+
hours=sign * int(hour_str),
|
1379
|
+
minutes=sign * int(minute_str),
|
1380
|
+
),
|
1381
|
+
)
|
1615
1382
|
|
1616
|
-
# Write second ack
|
1617
|
-
os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
|
1618
1383
|
|
1619
|
-
|
1620
|
-
|
1384
|
+
def toml_match_to_localtime(match: re.Match) -> datetime.time:
|
1385
|
+
hour_str, minute_str, sec_str, micros_str = match.groups()
|
1386
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
1387
|
+
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
1621
1388
|
|
1622
1389
|
|
1623
|
-
|
1390
|
+
def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
|
1391
|
+
if match.group('floatpart'):
|
1392
|
+
return parse_float(match.group())
|
1393
|
+
return int(match.group(), 0)
|
1624
1394
|
|
1625
1395
|
|
1626
|
-
|
1627
|
-
if any(c in context_name for c in '\'"'):
|
1628
|
-
raise NameError(context_name)
|
1396
|
+
TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
|
1629
1397
|
|
1630
|
-
|
1631
|
-
|
1632
|
-
|
1398
|
+
# Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
|
1399
|
+
# functions.
|
1400
|
+
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
1401
|
+
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
1633
1402
|
|
1634
|
-
|
1635
|
-
|
1636
|
-
'_PYREMOTE_BOOTSTRAP_SRC_FD',
|
1403
|
+
TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
1404
|
+
TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1637
1405
|
|
1638
|
-
|
1639
|
-
'_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
|
1640
|
-
'_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
|
1406
|
+
TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
1641
1407
|
|
1642
|
-
|
1643
|
-
|
1408
|
+
TOML_WS = frozenset(' \t')
|
1409
|
+
TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
|
1410
|
+
TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
|
1411
|
+
TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
|
1412
|
+
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
1644
1413
|
|
1645
|
-
|
1646
|
-
|
1647
|
-
|
1414
|
+
TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
|
1415
|
+
{
|
1416
|
+
'\\b': '\u0008', # backspace
|
1417
|
+
'\\t': '\u0009', # tab
|
1418
|
+
'\\n': '\u000A', # linefeed
|
1419
|
+
'\\f': '\u000C', # form feed
|
1420
|
+
'\\r': '\u000D', # carriage return
|
1421
|
+
'\\"': '\u0022', # quote
|
1422
|
+
'\\\\': '\u005C', # backslash
|
1423
|
+
},
|
1424
|
+
)
|
1648
1425
|
|
1649
|
-
bs_src = '\n'.join(
|
1650
|
-
cl
|
1651
|
-
for l in bs_src.splitlines()
|
1652
|
-
if (cl := (l.split('#')[0]).rstrip())
|
1653
|
-
if cl.strip()
|
1654
|
-
)
|
1655
1426
|
|
1656
|
-
|
1657
|
-
|
1658
|
-
if b'"' in bs_z85:
|
1659
|
-
raise ValueError(bs_z85)
|
1427
|
+
class TomlDecodeError(ValueError):
|
1428
|
+
"""An error raised if a document is not valid TOML."""
|
1660
1429
|
|
1661
|
-
stmts = [
|
1662
|
-
f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
|
1663
|
-
f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
|
1664
|
-
f'_pyremote_bootstrap_main("{context_name}")',
|
1665
|
-
]
|
1666
1430
|
|
1667
|
-
|
1668
|
-
|
1431
|
+
def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
|
1432
|
+
"""Parse TOML from a binary file object."""
|
1433
|
+
b = fp.read()
|
1434
|
+
try:
|
1435
|
+
s = b.decode()
|
1436
|
+
except AttributeError:
|
1437
|
+
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
1438
|
+
return toml_loads(s, parse_float=parse_float)
|
1669
1439
|
|
1670
1440
|
|
1671
|
-
|
1441
|
+
def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
|
1442
|
+
"""Parse TOML from a string."""
|
1672
1443
|
|
1444
|
+
# The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
|
1445
|
+
try:
|
1446
|
+
src = s.replace('\r\n', '\n')
|
1447
|
+
except (AttributeError, TypeError):
|
1448
|
+
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
1449
|
+
pos = 0
|
1450
|
+
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
1451
|
+
header: TomlKey = ()
|
1452
|
+
parse_float = toml_make_safe_parse_float(parse_float)
|
1673
1453
|
|
1674
|
-
|
1675
|
-
|
1676
|
-
|
1677
|
-
|
1678
|
-
context_name: str
|
1679
|
-
payload_src: str
|
1680
|
-
options: PyremoteBootstrapOptions
|
1681
|
-
env_info: PyremoteEnvInfo
|
1454
|
+
# Parse one statement at a time (typically means one line in TOML source)
|
1455
|
+
while True:
|
1456
|
+
# 1. Skip line leading whitespace
|
1457
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1682
1458
|
|
1459
|
+
# 2. Parse rules. Expect one of the following:
|
1460
|
+
# - end of file
|
1461
|
+
# - end of line
|
1462
|
+
# - comment
|
1463
|
+
# - key/value pair
|
1464
|
+
# - append dict to list (and move to its namespace)
|
1465
|
+
# - create dict (and move to its namespace)
|
1466
|
+
# Skip trailing whitespace when applicable.
|
1467
|
+
try:
|
1468
|
+
char = src[pos]
|
1469
|
+
except IndexError:
|
1470
|
+
break
|
1471
|
+
if char == '\n':
|
1472
|
+
pos += 1
|
1473
|
+
continue
|
1474
|
+
if char in TOML_KEY_INITIAL_CHARS:
|
1475
|
+
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
1476
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1477
|
+
elif char == '[':
|
1478
|
+
try:
|
1479
|
+
second_char: ta.Optional[str] = src[pos + 1]
|
1480
|
+
except IndexError:
|
1481
|
+
second_char = None
|
1482
|
+
out.flags.finalize_pending()
|
1483
|
+
if second_char == '[':
|
1484
|
+
pos, header = toml_create_list_rule(src, pos, out)
|
1485
|
+
else:
|
1486
|
+
pos, header = toml_create_dict_rule(src, pos, out)
|
1487
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1488
|
+
elif char != '#':
|
1489
|
+
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
1683
1490
|
|
1684
|
-
|
1685
|
-
|
1686
|
-
if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
|
1687
|
-
# Read second copy of payload src
|
1688
|
-
r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
|
1689
|
-
payload_src = r1.read().decode('utf-8')
|
1690
|
-
r1.close()
|
1491
|
+
# 3. Skip comment
|
1492
|
+
pos = toml_skip_comment(src, pos)
|
1691
1493
|
|
1692
|
-
#
|
1693
|
-
|
1694
|
-
|
1494
|
+
# 4. Expect end of line or end of file
|
1495
|
+
try:
|
1496
|
+
char = src[pos]
|
1497
|
+
except IndexError:
|
1498
|
+
break
|
1499
|
+
if char != '\n':
|
1500
|
+
raise toml_suffixed_err(
|
1501
|
+
src, pos, 'Expected newline or end of document after a statement',
|
1502
|
+
)
|
1503
|
+
pos += 1
|
1695
1504
|
|
1696
|
-
|
1697
|
-
options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
|
1698
|
-
if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
|
1699
|
-
raise EOFError
|
1700
|
-
options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
|
1505
|
+
return out.data.dict
|
1701
1506
|
|
1702
|
-
# If debugging, re-exec as file
|
1703
|
-
if options.debug:
|
1704
|
-
# Write temp source file
|
1705
|
-
import tempfile
|
1706
|
-
tfd, tfn = tempfile.mkstemp('-pyremote.py')
|
1707
|
-
os.write(tfd, payload_src.encode('utf-8'))
|
1708
|
-
os.close(tfd)
|
1709
1507
|
|
1710
|
-
|
1711
|
-
|
1712
|
-
os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
|
1508
|
+
class TomlFlags:
|
1509
|
+
"""Flags that map to parsed keys/namespaces."""
|
1713
1510
|
|
1714
|
-
|
1715
|
-
|
1716
|
-
|
1717
|
-
|
1511
|
+
# Marks an immutable namespace (inline array or inline table).
|
1512
|
+
FROZEN = 0
|
1513
|
+
# Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
|
1514
|
+
EXPLICIT_NEST = 1
|
1718
1515
|
|
1719
|
-
|
1720
|
-
|
1721
|
-
|
1722
|
-
options = PyremoteBootstrapOptions(**json.loads(options_json_str))
|
1516
|
+
def __init__(self) -> None:
|
1517
|
+
self._flags: ta.Dict[str, dict] = {}
|
1518
|
+
self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
|
1723
1519
|
|
1724
|
-
|
1725
|
-
|
1726
|
-
payload_src = sf.read()
|
1520
|
+
def add_pending(self, key: TomlKey, flag: int) -> None:
|
1521
|
+
self._pending_flags.add((key, flag))
|
1727
1522
|
|
1728
|
-
|
1729
|
-
|
1730
|
-
|
1523
|
+
def finalize_pending(self) -> None:
|
1524
|
+
for key, flag in self._pending_flags:
|
1525
|
+
self.set(key, flag, recursive=False)
|
1526
|
+
self._pending_flags.clear()
|
1731
1527
|
|
1732
|
-
|
1733
|
-
|
1528
|
+
def unset_all(self, key: TomlKey) -> None:
|
1529
|
+
cont = self._flags
|
1530
|
+
for k in key[:-1]:
|
1531
|
+
if k not in cont:
|
1532
|
+
return
|
1533
|
+
cont = cont[k]['nested']
|
1534
|
+
cont.pop(key[-1], None)
|
1734
1535
|
|
1735
|
-
#
|
1736
|
-
|
1737
|
-
|
1738
|
-
|
1739
|
-
|
1536
|
+
def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
|
1537
|
+
cont = self._flags
|
1538
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1539
|
+
for k in key_parent:
|
1540
|
+
if k not in cont:
|
1541
|
+
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
1542
|
+
cont = cont[k]['nested']
|
1543
|
+
if key_stem not in cont:
|
1544
|
+
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
1545
|
+
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
1740
1546
|
|
1741
|
-
|
1742
|
-
|
1743
|
-
|
1744
|
-
|
1745
|
-
|
1547
|
+
def is_(self, key: TomlKey, flag: int) -> bool:
|
1548
|
+
if not key:
|
1549
|
+
return False # document root has no flags
|
1550
|
+
cont = self._flags
|
1551
|
+
for k in key[:-1]:
|
1552
|
+
if k not in cont:
|
1553
|
+
return False
|
1554
|
+
inner_cont = cont[k]
|
1555
|
+
if flag in inner_cont['recursive_flags']:
|
1556
|
+
return True
|
1557
|
+
cont = inner_cont['nested']
|
1558
|
+
key_stem = key[-1]
|
1559
|
+
if key_stem in cont:
|
1560
|
+
cont = cont[key_stem]
|
1561
|
+
return flag in cont['flags'] or flag in cont['recursive_flags']
|
1562
|
+
return False
|
1746
1563
|
|
1747
|
-
if (mn := options.main_name_override) is not None:
|
1748
|
-
# Inspections like typing.get_type_hints need an entry in sys.modules.
|
1749
|
-
sys.modules[mn] = sys.modules['__main__']
|
1750
1564
|
|
1751
|
-
|
1752
|
-
|
1565
|
+
class TomlNestedDict:
|
1566
|
+
def __init__(self) -> None:
|
1567
|
+
# The parsed content of the TOML document
|
1568
|
+
self.dict: ta.Dict[str, ta.Any] = {}
|
1753
1569
|
|
1754
|
-
|
1755
|
-
|
1756
|
-
|
1757
|
-
|
1758
|
-
|
1759
|
-
|
1760
|
-
|
1761
|
-
|
1762
|
-
|
1570
|
+
def get_or_create_nest(
|
1571
|
+
self,
|
1572
|
+
key: TomlKey,
|
1573
|
+
*,
|
1574
|
+
access_lists: bool = True,
|
1575
|
+
) -> dict:
|
1576
|
+
cont: ta.Any = self.dict
|
1577
|
+
for k in key:
|
1578
|
+
if k not in cont:
|
1579
|
+
cont[k] = {}
|
1580
|
+
cont = cont[k]
|
1581
|
+
if access_lists and isinstance(cont, list):
|
1582
|
+
cont = cont[-1]
|
1583
|
+
if not isinstance(cont, dict):
|
1584
|
+
raise KeyError('There is no nest behind this key')
|
1585
|
+
return cont
|
1763
1586
|
|
1587
|
+
def append_nest_to_list(self, key: TomlKey) -> None:
|
1588
|
+
cont = self.get_or_create_nest(key[:-1])
|
1589
|
+
last_key = key[-1]
|
1590
|
+
if last_key in cont:
|
1591
|
+
list_ = cont[last_key]
|
1592
|
+
if not isinstance(list_, list):
|
1593
|
+
raise KeyError('An object other than list found behind this key')
|
1594
|
+
list_.append({})
|
1595
|
+
else:
|
1596
|
+
cont[last_key] = [{}]
|
1764
1597
|
|
1765
|
-
##
|
1766
1598
|
|
1599
|
+
class TomlOutput(ta.NamedTuple):
|
1600
|
+
data: TomlNestedDict
|
1601
|
+
flags: TomlFlags
|
1767
1602
|
|
1768
|
-
class PyremoteBootstrapDriver:
|
1769
|
-
def __init__(
|
1770
|
-
self,
|
1771
|
-
payload_src: ta.Union[str, ta.Sequence[str]],
|
1772
|
-
options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
|
1773
|
-
) -> None:
|
1774
|
-
super().__init__()
|
1775
1603
|
|
1776
|
-
|
1777
|
-
|
1604
|
+
def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
|
1605
|
+
try:
|
1606
|
+
while src[pos] in chars:
|
1607
|
+
pos += 1
|
1608
|
+
except IndexError:
|
1609
|
+
pass
|
1610
|
+
return pos
|
1778
1611
|
|
1779
|
-
self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
|
1780
|
-
self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
|
1781
1612
|
|
1782
|
-
|
1783
|
-
|
1613
|
+
def toml_skip_until(
|
1614
|
+
src: str,
|
1615
|
+
pos: TomlPos,
|
1616
|
+
expect: str,
|
1617
|
+
*,
|
1618
|
+
error_on: ta.FrozenSet[str],
|
1619
|
+
error_on_eof: bool,
|
1620
|
+
) -> TomlPos:
|
1621
|
+
try:
|
1622
|
+
new_pos = src.index(expect, pos)
|
1623
|
+
except ValueError:
|
1624
|
+
new_pos = len(src)
|
1625
|
+
if error_on_eof:
|
1626
|
+
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
1784
1627
|
|
1785
|
-
|
1786
|
-
|
1787
|
-
|
1788
|
-
|
1789
|
-
|
1790
|
-
) -> str:
|
1791
|
-
parts: ta.List[str]
|
1792
|
-
if isinstance(payload_src, str):
|
1793
|
-
parts = [payload_src]
|
1794
|
-
else:
|
1795
|
-
parts = list(payload_src)
|
1628
|
+
if not error_on.isdisjoint(src[pos:new_pos]):
|
1629
|
+
while src[pos] not in error_on:
|
1630
|
+
pos += 1
|
1631
|
+
raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
|
1632
|
+
return new_pos
|
1796
1633
|
|
1797
|
-
if (mn := options.main_name_override) is not None:
|
1798
|
-
parts.insert(0, f'__name__ = {mn!r}')
|
1799
1634
|
|
1800
|
-
|
1801
|
-
|
1802
|
-
|
1803
|
-
|
1635
|
+
def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
|
1636
|
+
try:
|
1637
|
+
char: ta.Optional[str] = src[pos]
|
1638
|
+
except IndexError:
|
1639
|
+
char = None
|
1640
|
+
if char == '#':
|
1641
|
+
return toml_skip_until(
|
1642
|
+
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
1643
|
+
)
|
1644
|
+
return pos
|
1804
1645
|
|
1805
|
-
#
|
1806
1646
|
|
1807
|
-
|
1808
|
-
|
1809
|
-
|
1647
|
+
def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
|
1648
|
+
while True:
|
1649
|
+
pos_before_skip = pos
|
1650
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1651
|
+
pos = toml_skip_comment(src, pos)
|
1652
|
+
if pos == pos_before_skip:
|
1653
|
+
return pos
|
1810
1654
|
|
1811
|
-
@dc.dataclass(frozen=True)
|
1812
|
-
class Write:
|
1813
|
-
d: bytes
|
1814
1655
|
|
1815
|
-
|
1816
|
-
|
1656
|
+
def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
1657
|
+
pos += 1 # Skip "["
|
1658
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1659
|
+
pos, key = toml_parse_key(src, pos)
|
1817
1660
|
|
1818
|
-
|
1819
|
-
|
1820
|
-
|
1821
|
-
|
1661
|
+
if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
|
1662
|
+
raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
|
1663
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
1664
|
+
try:
|
1665
|
+
out.data.get_or_create_nest(key)
|
1666
|
+
except KeyError:
|
1667
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1822
1668
|
|
1823
|
-
|
1824
|
-
|
1825
|
-
|
1669
|
+
if not src.startswith(']', pos):
|
1670
|
+
raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
|
1671
|
+
return pos + 1, key
|
1826
1672
|
|
1827
|
-
# Read pid
|
1828
|
-
d = yield from self._read(8)
|
1829
|
-
pid = struct.unpack('<Q', d)[0]
|
1830
1673
|
|
1831
|
-
|
1832
|
-
|
1833
|
-
|
1674
|
+
def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
1675
|
+
pos += 2 # Skip "[["
|
1676
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1677
|
+
pos, key = toml_parse_key(src, pos)
|
1834
1678
|
|
1835
|
-
|
1836
|
-
|
1679
|
+
if out.flags.is_(key, TomlFlags.FROZEN):
|
1680
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1681
|
+
# Free the namespace now that it points to another empty list item...
|
1682
|
+
out.flags.unset_all(key)
|
1683
|
+
# ...but this key precisely is still prohibited from table declaration
|
1684
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
1685
|
+
try:
|
1686
|
+
out.data.append_nest_to_list(key)
|
1687
|
+
except KeyError:
|
1688
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1837
1689
|
|
1838
|
-
|
1839
|
-
|
1840
|
-
|
1690
|
+
if not src.startswith(']]', pos):
|
1691
|
+
raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
|
1692
|
+
return pos + 2, key
|
1841
1693
|
|
1842
|
-
# Read third ack (after reaping child process)
|
1843
|
-
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
|
1844
1694
|
|
1845
|
-
|
1846
|
-
|
1847
|
-
|
1848
|
-
|
1849
|
-
|
1850
|
-
|
1695
|
+
def toml_key_value_rule(
|
1696
|
+
src: str,
|
1697
|
+
pos: TomlPos,
|
1698
|
+
out: TomlOutput,
|
1699
|
+
header: TomlKey,
|
1700
|
+
parse_float: TomlParseFloat,
|
1701
|
+
) -> TomlPos:
|
1702
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1703
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1704
|
+
abs_key_parent = header + key_parent
|
1851
1705
|
|
1852
|
-
|
1853
|
-
|
1706
|
+
relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
|
1707
|
+
for cont_key in relative_path_cont_keys:
|
1708
|
+
# Check that dotted key syntax does not redefine an existing table
|
1709
|
+
if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
|
1710
|
+
raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
|
1711
|
+
# Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
|
1712
|
+
# table sections.
|
1713
|
+
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
1854
1714
|
|
1855
|
-
|
1856
|
-
|
1857
|
-
|
1858
|
-
|
1715
|
+
if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
|
1716
|
+
raise toml_suffixed_err(
|
1717
|
+
src,
|
1718
|
+
pos,
|
1719
|
+
f'Cannot mutate immutable namespace {abs_key_parent}',
|
1859
1720
|
)
|
1860
1721
|
|
1861
|
-
|
1862
|
-
|
1863
|
-
|
1864
|
-
|
1865
|
-
|
1866
|
-
|
1867
|
-
|
1722
|
+
try:
|
1723
|
+
nest = out.data.get_or_create_nest(abs_key_parent)
|
1724
|
+
except KeyError:
|
1725
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1726
|
+
if key_stem in nest:
|
1727
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
1728
|
+
# Mark inline table and array namespaces recursively immutable
|
1729
|
+
if isinstance(value, (dict, list)):
|
1730
|
+
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
1731
|
+
nest[key_stem] = value
|
1732
|
+
return pos
|
1868
1733
|
|
1869
|
-
def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
|
1870
|
-
d = yield from self._read(len(e))
|
1871
|
-
if d != e:
|
1872
|
-
raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
|
1873
1734
|
|
1874
|
-
|
1875
|
-
|
1876
|
-
|
1877
|
-
|
1735
|
+
def toml_parse_key_value_pair(
|
1736
|
+
src: str,
|
1737
|
+
pos: TomlPos,
|
1738
|
+
parse_float: TomlParseFloat,
|
1739
|
+
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
1740
|
+
pos, key = toml_parse_key(src, pos)
|
1741
|
+
try:
|
1742
|
+
char: ta.Optional[str] = src[pos]
|
1743
|
+
except IndexError:
|
1744
|
+
char = None
|
1745
|
+
if char != '=':
|
1746
|
+
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
1747
|
+
pos += 1
|
1748
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1749
|
+
pos, value = toml_parse_value(src, pos, parse_float)
|
1750
|
+
return pos, key, value
|
1878
1751
|
|
1879
|
-
#
|
1880
1752
|
|
1881
|
-
|
1882
|
-
|
1753
|
+
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
1754
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
1755
|
+
key: TomlKey = (key_part,)
|
1756
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1757
|
+
while True:
|
1758
|
+
try:
|
1759
|
+
char: ta.Optional[str] = src[pos]
|
1760
|
+
except IndexError:
|
1761
|
+
char = None
|
1762
|
+
if char != '.':
|
1763
|
+
return pos, key
|
1764
|
+
pos += 1
|
1765
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1766
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
1767
|
+
key += (key_part,)
|
1768
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1769
|
+
|
1770
|
+
|
1771
|
+
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1772
|
+
try:
|
1773
|
+
char: ta.Optional[str] = src[pos]
|
1774
|
+
except IndexError:
|
1775
|
+
char = None
|
1776
|
+
if char in TOML_BARE_KEY_CHARS:
|
1777
|
+
start_pos = pos
|
1778
|
+
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
1779
|
+
return pos, src[start_pos:pos]
|
1780
|
+
if char == "'":
|
1781
|
+
return toml_parse_literal_str(src, pos)
|
1782
|
+
if char == '"':
|
1783
|
+
return toml_parse_one_line_basic_str(src, pos)
|
1784
|
+
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
1785
|
+
|
1786
|
+
|
1787
|
+
def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1788
|
+
pos += 1
|
1789
|
+
return toml_parse_basic_str(src, pos, multiline=False)
|
1790
|
+
|
1791
|
+
|
1792
|
+
def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
|
1793
|
+
pos += 1
|
1794
|
+
array: list = []
|
1795
|
+
|
1796
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1797
|
+
if src.startswith(']', pos):
|
1798
|
+
return pos + 1, array
|
1799
|
+
while True:
|
1800
|
+
pos, val = toml_parse_value(src, pos, parse_float)
|
1801
|
+
array.append(val)
|
1802
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1803
|
+
|
1804
|
+
c = src[pos:pos + 1]
|
1805
|
+
if c == ']':
|
1806
|
+
return pos + 1, array
|
1807
|
+
if c != ',':
|
1808
|
+
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
1809
|
+
pos += 1
|
1810
|
+
|
1811
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1812
|
+
if src.startswith(']', pos):
|
1813
|
+
return pos + 1, array
|
1814
|
+
|
1815
|
+
|
1816
|
+
def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
|
1817
|
+
pos += 1
|
1818
|
+
nested_dict = TomlNestedDict()
|
1819
|
+
flags = TomlFlags()
|
1820
|
+
|
1821
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1822
|
+
if src.startswith('}', pos):
|
1823
|
+
return pos + 1, nested_dict.dict
|
1824
|
+
while True:
|
1825
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1826
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1827
|
+
if flags.is_(key, TomlFlags.FROZEN):
|
1828
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1829
|
+
try:
|
1830
|
+
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
1831
|
+
except KeyError:
|
1832
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1833
|
+
if key_stem in nest:
|
1834
|
+
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
1835
|
+
nest[key_stem] = value
|
1836
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1837
|
+
c = src[pos:pos + 1]
|
1838
|
+
if c == '}':
|
1839
|
+
return pos + 1, nested_dict.dict
|
1840
|
+
if c != ',':
|
1841
|
+
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
1842
|
+
if isinstance(value, (dict, list)):
|
1843
|
+
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
1844
|
+
pos += 1
|
1845
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1846
|
+
|
1847
|
+
|
1848
|
+
def toml_parse_basic_str_escape(
|
1849
|
+
src: str,
|
1850
|
+
pos: TomlPos,
|
1851
|
+
*,
|
1852
|
+
multiline: bool = False,
|
1853
|
+
) -> ta.Tuple[TomlPos, str]:
|
1854
|
+
escape_id = src[pos:pos + 2]
|
1855
|
+
pos += 2
|
1856
|
+
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
1857
|
+
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
1858
|
+
# newline.
|
1859
|
+
if escape_id != '\\\n':
|
1860
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1861
|
+
try:
|
1862
|
+
char = src[pos]
|
1863
|
+
except IndexError:
|
1864
|
+
return pos, ''
|
1865
|
+
if char != '\n':
|
1866
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
1867
|
+
pos += 1
|
1868
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1869
|
+
return pos, ''
|
1870
|
+
if escape_id == '\\u':
|
1871
|
+
return toml_parse_hex_char(src, pos, 4)
|
1872
|
+
if escape_id == '\\U':
|
1873
|
+
return toml_parse_hex_char(src, pos, 8)
|
1874
|
+
try:
|
1875
|
+
return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
|
1876
|
+
except KeyError:
|
1877
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
1878
|
+
|
1883
1879
|
|
1884
|
-
|
1885
|
-
|
1886
|
-
try:
|
1887
|
-
if gi is not None:
|
1888
|
-
go = gen.send(gi)
|
1889
|
-
else:
|
1890
|
-
go = next(gen)
|
1891
|
-
except StopIteration as e:
|
1892
|
-
return e.value
|
1880
|
+
def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1881
|
+
return toml_parse_basic_str_escape(src, pos, multiline=True)
|
1893
1882
|
|
1894
|
-
if isinstance(go, self.Read):
|
1895
|
-
if len(gi := input.read(go.sz)) != go.sz:
|
1896
|
-
raise EOFError
|
1897
|
-
elif isinstance(go, self.Write):
|
1898
|
-
gi = None
|
1899
|
-
output.write(go.d)
|
1900
|
-
output.flush()
|
1901
|
-
else:
|
1902
|
-
raise TypeError(go)
|
1903
1883
|
|
1904
|
-
|
1905
|
-
|
1906
|
-
|
1907
|
-
|
1908
|
-
|
1909
|
-
|
1884
|
+
def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
|
1885
|
+
hex_str = src[pos:pos + hex_len]
|
1886
|
+
if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
|
1887
|
+
raise toml_suffixed_err(src, pos, 'Invalid hex value')
|
1888
|
+
pos += hex_len
|
1889
|
+
hex_int = int(hex_str, 16)
|
1890
|
+
if not toml_is_unicode_scalar_value(hex_int):
|
1891
|
+
raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
|
1892
|
+
return pos, chr(hex_int)
|
1910
1893
|
|
1911
|
-
gi: ta.Optional[bytes] = None
|
1912
|
-
while True:
|
1913
|
-
try:
|
1914
|
-
if gi is not None:
|
1915
|
-
go = gen.send(gi)
|
1916
|
-
else:
|
1917
|
-
go = next(gen)
|
1918
|
-
except StopIteration as e:
|
1919
|
-
return e.value
|
1920
1894
|
|
1921
|
-
|
1922
|
-
|
1923
|
-
|
1924
|
-
|
1925
|
-
|
1926
|
-
|
1927
|
-
|
1928
|
-
else:
|
1929
|
-
raise TypeError(go)
|
1895
|
+
def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1896
|
+
pos += 1 # Skip starting apostrophe
|
1897
|
+
start_pos = pos
|
1898
|
+
pos = toml_skip_until(
|
1899
|
+
src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
|
1900
|
+
)
|
1901
|
+
return pos + 1, src[start_pos:pos] # Skip ending apostrophe
|
1930
1902
|
|
1931
1903
|
|
1932
|
-
|
1933
|
-
|
1904
|
+
def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
|
1905
|
+
pos += 3
|
1906
|
+
if src.startswith('\n', pos):
|
1907
|
+
pos += 1
|
1934
1908
|
|
1909
|
+
if literal:
|
1910
|
+
delim = "'"
|
1911
|
+
end_pos = toml_skip_until(
|
1912
|
+
src,
|
1913
|
+
pos,
|
1914
|
+
"'''",
|
1915
|
+
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
1916
|
+
error_on_eof=True,
|
1917
|
+
)
|
1918
|
+
result = src[pos:end_pos]
|
1919
|
+
pos = end_pos + 3
|
1920
|
+
else:
|
1921
|
+
delim = '"'
|
1922
|
+
pos, result = toml_parse_basic_str(src, pos, multiline=True)
|
1935
1923
|
|
1936
|
-
|
1937
|
-
|
1938
|
-
|
1924
|
+
# Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
|
1925
|
+
if not src.startswith(delim, pos):
|
1926
|
+
return pos, result
|
1927
|
+
pos += 1
|
1928
|
+
if not src.startswith(delim, pos):
|
1929
|
+
return pos, result + delim
|
1930
|
+
pos += 1
|
1931
|
+
return pos, result + (delim * 2)
|
1939
1932
|
|
1940
|
-
self.reader = reader
|
1941
|
-
self.closed: asyncio.Future = asyncio.Future()
|
1942
1933
|
|
1943
|
-
|
1944
|
-
|
1945
|
-
|
1934
|
+
def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
|
1935
|
+
if multiline:
|
1936
|
+
error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1937
|
+
parse_escapes = toml_parse_basic_str_escape_multiline
|
1938
|
+
else:
|
1939
|
+
error_on = TOML_ILLEGAL_BASIC_STR_CHARS
|
1940
|
+
parse_escapes = toml_parse_basic_str_escape
|
1941
|
+
result = ''
|
1942
|
+
start_pos = pos
|
1943
|
+
while True:
|
1944
|
+
try:
|
1945
|
+
char = src[pos]
|
1946
|
+
except IndexError:
|
1947
|
+
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
1948
|
+
if char == '"':
|
1949
|
+
if not multiline:
|
1950
|
+
return pos + 1, result + src[start_pos:pos]
|
1951
|
+
if src.startswith('"""', pos):
|
1952
|
+
return pos + 3, result + src[start_pos:pos]
|
1953
|
+
pos += 1
|
1954
|
+
continue
|
1955
|
+
if char == '\\':
|
1956
|
+
result += src[start_pos:pos]
|
1957
|
+
pos, parsed_escape = parse_escapes(src, pos)
|
1958
|
+
result += parsed_escape
|
1959
|
+
start_pos = pos
|
1960
|
+
continue
|
1961
|
+
if char in error_on:
|
1962
|
+
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
1963
|
+
pos += 1
|
1946
1964
|
|
1947
|
-
# @ta.override
|
1948
|
-
def close(self) -> None:
|
1949
|
-
self.reader.feed_eof()
|
1950
|
-
if not self.closed.done():
|
1951
|
-
self.closed.set_result(True)
|
1952
1965
|
|
1953
|
-
|
1954
|
-
|
1955
|
-
|
1966
|
+
def toml_parse_value( # noqa: C901
|
1967
|
+
src: str,
|
1968
|
+
pos: TomlPos,
|
1969
|
+
parse_float: TomlParseFloat,
|
1970
|
+
) -> ta.Tuple[TomlPos, ta.Any]:
|
1971
|
+
try:
|
1972
|
+
char: ta.Optional[str] = src[pos]
|
1973
|
+
except IndexError:
|
1974
|
+
char = None
|
1956
1975
|
|
1976
|
+
# IMPORTANT: order conditions based on speed of checking and likelihood
|
1957
1977
|
|
1958
|
-
|
1959
|
-
|
1960
|
-
|
1961
|
-
|
1962
|
-
|
1978
|
+
# Basic strings
|
1979
|
+
if char == '"':
|
1980
|
+
if src.startswith('"""', pos):
|
1981
|
+
return toml_parse_multiline_str(src, pos, literal=False)
|
1982
|
+
return toml_parse_one_line_basic_str(src, pos)
|
1963
1983
|
|
1964
|
-
|
1965
|
-
|
1966
|
-
|
1967
|
-
|
1984
|
+
# Literal strings
|
1985
|
+
if char == "'":
|
1986
|
+
if src.startswith("'''", pos):
|
1987
|
+
return toml_parse_multiline_str(src, pos, literal=True)
|
1988
|
+
return toml_parse_literal_str(src, pos)
|
1968
1989
|
|
1969
|
-
|
1990
|
+
# Booleans
|
1991
|
+
if char == 't':
|
1992
|
+
if src.startswith('true', pos):
|
1993
|
+
return pos + 4, True
|
1994
|
+
if char == 'f':
|
1995
|
+
if src.startswith('false', pos):
|
1996
|
+
return pos + 5, False
|
1970
1997
|
|
1998
|
+
# Arrays
|
1999
|
+
if char == '[':
|
2000
|
+
return toml_parse_array(src, pos, parse_float)
|
1971
2001
|
|
1972
|
-
|
1973
|
-
|
2002
|
+
# Inline tables
|
2003
|
+
if char == '{':
|
2004
|
+
return toml_parse_inline_table(src, pos, parse_float)
|
1974
2005
|
|
2006
|
+
# Dates and times
|
2007
|
+
datetime_match = TOML_RE_DATETIME.match(src, pos)
|
2008
|
+
if datetime_match:
|
2009
|
+
try:
|
2010
|
+
datetime_obj = toml_match_to_datetime(datetime_match)
|
2011
|
+
except ValueError as e:
|
2012
|
+
raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
|
2013
|
+
return datetime_match.end(), datetime_obj
|
2014
|
+
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
2015
|
+
if localtime_match:
|
2016
|
+
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
1975
2017
|
|
1976
|
-
|
2018
|
+
# Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
|
2019
|
+
# located after handling of dates and times.
|
2020
|
+
number_match = TOML_RE_NUMBER.match(src, pos)
|
2021
|
+
if number_match:
|
2022
|
+
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
1977
2023
|
|
2024
|
+
# Special floats
|
2025
|
+
first_three = src[pos:pos + 3]
|
2026
|
+
if first_three in {'inf', 'nan'}:
|
2027
|
+
return pos + 3, parse_float(first_three)
|
2028
|
+
first_four = src[pos:pos + 4]
|
2029
|
+
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
2030
|
+
return pos + 4, parse_float(first_four)
|
1978
2031
|
|
1979
|
-
|
1980
|
-
f: ta.IO,
|
1981
|
-
loop: ta.Any = None,
|
1982
|
-
*,
|
1983
|
-
limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
|
1984
|
-
) -> asyncio.StreamReader:
|
1985
|
-
if loop is None:
|
1986
|
-
loop = asyncio.get_running_loop()
|
2032
|
+
raise toml_suffixed_err(src, pos, 'Invalid value')
|
1987
2033
|
|
1988
|
-
reader = asyncio.StreamReader(limit=limit, loop=loop)
|
1989
|
-
await loop.connect_read_pipe(
|
1990
|
-
lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
|
1991
|
-
f,
|
1992
|
-
)
|
1993
2034
|
|
1994
|
-
|
2035
|
+
def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
|
2036
|
+
"""Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
|
1995
2037
|
|
2038
|
+
def coord_repr(src: str, pos: TomlPos) -> str:
|
2039
|
+
if pos >= len(src):
|
2040
|
+
return 'end of document'
|
2041
|
+
line = src.count('\n', 0, pos) + 1
|
2042
|
+
if line == 1:
|
2043
|
+
column = pos + 1
|
2044
|
+
else:
|
2045
|
+
column = pos - src.rindex('\n', 0, pos)
|
2046
|
+
return f'line {line}, column {column}'
|
1996
2047
|
|
1997
|
-
|
1998
|
-
f: ta.IO,
|
1999
|
-
loop: ta.Any = None,
|
2000
|
-
) -> asyncio.StreamWriter:
|
2001
|
-
if loop is None:
|
2002
|
-
loop = asyncio.get_running_loop()
|
2048
|
+
return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
|
2003
2049
|
|
2004
|
-
writer_transport, writer_protocol = await loop.connect_write_pipe(
|
2005
|
-
lambda: asyncio.streams.FlowControlMixin(loop=loop),
|
2006
|
-
f,
|
2007
|
-
)
|
2008
2050
|
|
2009
|
-
|
2010
|
-
|
2011
|
-
writer_protocol,
|
2012
|
-
None,
|
2013
|
-
loop,
|
2014
|
-
)
|
2051
|
+
def toml_is_unicode_scalar_value(codepoint: int) -> bool:
|
2052
|
+
return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
|
2015
2053
|
|
2016
2054
|
|
2017
|
-
|
2018
|
-
|
2055
|
+
def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
|
2056
|
+
"""A decorator to make `parse_float` safe.
|
2057
|
+
|
2058
|
+
`parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
|
2059
|
+
thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
|
2060
|
+
"""
|
2061
|
+
# The default `float` callable never returns illegal types. Optimize it.
|
2062
|
+
if parse_float is float:
|
2063
|
+
return float
|
2019
2064
|
|
2065
|
+
def safe_parse_float(float_str: str) -> ta.Any:
|
2066
|
+
float_value = parse_float(float_str)
|
2067
|
+
if isinstance(float_value, (dict, list)):
|
2068
|
+
raise ValueError('parse_float must not return dicts or lists') # noqa
|
2069
|
+
return float_value
|
2020
2070
|
|
2021
|
-
|
2022
|
-
fut: AwaitableT,
|
2023
|
-
timeout: ta.Optional[float] = None,
|
2024
|
-
) -> AwaitableT:
|
2025
|
-
if timeout is not None:
|
2026
|
-
fut = asyncio.wait_for(fut, timeout) # type: ignore
|
2027
|
-
return fut
|
2071
|
+
return safe_parse_float
|
2028
2072
|
|
2029
2073
|
|
2030
2074
|
########################################
|
@@ -6986,30 +7030,6 @@ def build_config_named_children(
|
|
6986
7030
|
return lst
|
6987
7031
|
|
6988
7032
|
|
6989
|
-
##
|
6990
|
-
|
6991
|
-
|
6992
|
-
def render_ini_config(
|
6993
|
-
settings_by_section: IniConfigSectionSettingsMap,
|
6994
|
-
) -> str:
|
6995
|
-
out = io.StringIO()
|
6996
|
-
|
6997
|
-
for i, (section, settings) in enumerate(settings_by_section.items()):
|
6998
|
-
if i:
|
6999
|
-
out.write('\n')
|
7000
|
-
|
7001
|
-
out.write(f'[{section}]\n')
|
7002
|
-
|
7003
|
-
for k, v in settings.items():
|
7004
|
-
if isinstance(v, str):
|
7005
|
-
out.write(f'{k}={v}\n')
|
7006
|
-
else:
|
7007
|
-
for vv in v:
|
7008
|
-
out.write(f'{k}={vv}\n')
|
7009
|
-
|
7010
|
-
return out.getvalue()
|
7011
|
-
|
7012
|
-
|
7013
7033
|
########################################
|
7014
7034
|
# ../commands/marshal.py
|
7015
7035
|
|
@@ -7063,6 +7083,104 @@ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
|
|
7063
7083
|
CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
|
7064
7084
|
|
7065
7085
|
|
7086
|
+
########################################
|
7087
|
+
# ../deploy/conf/specs.py
|
7088
|
+
|
7089
|
+
|
7090
|
+
##
|
7091
|
+
|
7092
|
+
|
7093
|
+
class DeployAppConfContent(abc.ABC): # noqa
|
7094
|
+
pass
|
7095
|
+
|
7096
|
+
|
7097
|
+
#
|
7098
|
+
|
7099
|
+
|
7100
|
+
@register_single_field_type_obj_marshaler('body')
|
7101
|
+
@dc.dataclass(frozen=True)
|
7102
|
+
class RawDeployAppConfContent(DeployAppConfContent):
|
7103
|
+
body: str
|
7104
|
+
|
7105
|
+
|
7106
|
+
#
|
7107
|
+
|
7108
|
+
|
7109
|
+
@register_single_field_type_obj_marshaler('obj')
|
7110
|
+
@dc.dataclass(frozen=True)
|
7111
|
+
class JsonDeployAppConfContent(DeployAppConfContent):
|
7112
|
+
obj: ta.Any
|
7113
|
+
|
7114
|
+
|
7115
|
+
#
|
7116
|
+
|
7117
|
+
|
7118
|
+
@register_single_field_type_obj_marshaler('sections')
|
7119
|
+
@dc.dataclass(frozen=True)
|
7120
|
+
class IniDeployAppConfContent(DeployAppConfContent):
|
7121
|
+
sections: IniSectionSettingsMap
|
7122
|
+
|
7123
|
+
|
7124
|
+
#
|
7125
|
+
|
7126
|
+
|
7127
|
+
@register_single_field_type_obj_marshaler('items')
|
7128
|
+
@dc.dataclass(frozen=True)
|
7129
|
+
class NginxDeployAppConfContent(DeployAppConfContent):
|
7130
|
+
items: ta.Any
|
7131
|
+
|
7132
|
+
|
7133
|
+
##
|
7134
|
+
|
7135
|
+
|
7136
|
+
@dc.dataclass(frozen=True)
|
7137
|
+
class DeployAppConfFile:
|
7138
|
+
path: str
|
7139
|
+
content: DeployAppConfContent
|
7140
|
+
|
7141
|
+
def __post_init__(self) -> None:
|
7142
|
+
check_valid_deploy_spec_path(self.path)
|
7143
|
+
|
7144
|
+
|
7145
|
+
##
|
7146
|
+
|
7147
|
+
|
7148
|
+
@dc.dataclass(frozen=True)
|
7149
|
+
class DeployAppConfLink: # noqa
|
7150
|
+
"""
|
7151
|
+
May be either:
|
7152
|
+
- @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
|
7153
|
+
- @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
|
7154
|
+
- @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
|
7155
|
+
"""
|
7156
|
+
|
7157
|
+
src: str
|
7158
|
+
|
7159
|
+
kind: ta.Literal['current_only', 'all_active'] = 'current_only'
|
7160
|
+
|
7161
|
+
def __post_init__(self) -> None:
|
7162
|
+
check_valid_deploy_spec_path(self.src)
|
7163
|
+
if '/' in self.src:
|
7164
|
+
check.equal(self.src.count('/'), 1)
|
7165
|
+
|
7166
|
+
|
7167
|
+
##
|
7168
|
+
|
7169
|
+
|
7170
|
+
@dc.dataclass(frozen=True)
|
7171
|
+
class DeployAppConfSpec:
|
7172
|
+
files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
|
7173
|
+
|
7174
|
+
links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
|
7175
|
+
|
7176
|
+
def __post_init__(self) -> None:
|
7177
|
+
if self.files:
|
7178
|
+
seen: ta.Set[str] = set()
|
7179
|
+
for f in self.files:
|
7180
|
+
check.not_in(f.path, seen)
|
7181
|
+
seen.add(f.path)
|
7182
|
+
|
7183
|
+
|
7066
7184
|
########################################
|
7067
7185
|
# ../deploy/tags.py
|
7068
7186
|
|
@@ -7978,112 +8096,14 @@ class LocalCommandExecutor(CommandExecutor):
|
|
7978
8096
|
self,
|
7979
8097
|
*,
|
7980
8098
|
command_executors: CommandExecutorMap,
|
7981
|
-
) -> None:
|
7982
|
-
super().__init__()
|
7983
|
-
|
7984
|
-
self._command_executors = command_executors
|
7985
|
-
|
7986
|
-
async def execute(self, cmd: Command) -> Command.Output:
|
7987
|
-
ce: CommandExecutor = self._command_executors[type(cmd)]
|
7988
|
-
return await ce.execute(cmd)
|
7989
|
-
|
7990
|
-
|
7991
|
-
########################################
|
7992
|
-
# ../deploy/conf/specs.py
|
7993
|
-
|
7994
|
-
|
7995
|
-
##
|
7996
|
-
|
7997
|
-
|
7998
|
-
class DeployAppConfContent(abc.ABC): # noqa
|
7999
|
-
pass
|
8000
|
-
|
8001
|
-
|
8002
|
-
#
|
8003
|
-
|
8004
|
-
|
8005
|
-
@register_single_field_type_obj_marshaler('body')
|
8006
|
-
@dc.dataclass(frozen=True)
|
8007
|
-
class RawDeployAppConfContent(DeployAppConfContent):
|
8008
|
-
body: str
|
8009
|
-
|
8010
|
-
|
8011
|
-
#
|
8012
|
-
|
8013
|
-
|
8014
|
-
@register_single_field_type_obj_marshaler('obj')
|
8015
|
-
@dc.dataclass(frozen=True)
|
8016
|
-
class JsonDeployAppConfContent(DeployAppConfContent):
|
8017
|
-
obj: ta.Any
|
8018
|
-
|
8019
|
-
|
8020
|
-
#
|
8021
|
-
|
8022
|
-
|
8023
|
-
@register_single_field_type_obj_marshaler('sections')
|
8024
|
-
@dc.dataclass(frozen=True)
|
8025
|
-
class IniDeployAppConfContent(DeployAppConfContent):
|
8026
|
-
sections: IniConfigSectionSettingsMap
|
8027
|
-
|
8028
|
-
|
8029
|
-
#
|
8030
|
-
|
8031
|
-
|
8032
|
-
@register_single_field_type_obj_marshaler('items')
|
8033
|
-
@dc.dataclass(frozen=True)
|
8034
|
-
class NginxDeployAppConfContent(DeployAppConfContent):
|
8035
|
-
items: ta.Any
|
8036
|
-
|
8037
|
-
|
8038
|
-
##
|
8039
|
-
|
8040
|
-
|
8041
|
-
@dc.dataclass(frozen=True)
|
8042
|
-
class DeployAppConfFile:
|
8043
|
-
path: str
|
8044
|
-
content: DeployAppConfContent
|
8045
|
-
|
8046
|
-
def __post_init__(self) -> None:
|
8047
|
-
check_valid_deploy_spec_path(self.path)
|
8048
|
-
|
8049
|
-
|
8050
|
-
##
|
8051
|
-
|
8052
|
-
|
8053
|
-
@dc.dataclass(frozen=True)
|
8054
|
-
class DeployAppConfLink: # noqa
|
8055
|
-
"""
|
8056
|
-
May be either:
|
8057
|
-
- @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
|
8058
|
-
- @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
|
8059
|
-
- @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
|
8060
|
-
"""
|
8061
|
-
|
8062
|
-
src: str
|
8063
|
-
|
8064
|
-
kind: ta.Literal['current_only', 'all_active'] = 'current_only'
|
8065
|
-
|
8066
|
-
def __post_init__(self) -> None:
|
8067
|
-
check_valid_deploy_spec_path(self.src)
|
8068
|
-
if '/' in self.src:
|
8069
|
-
check.equal(self.src.count('/'), 1)
|
8070
|
-
|
8071
|
-
|
8072
|
-
##
|
8073
|
-
|
8074
|
-
|
8075
|
-
@dc.dataclass(frozen=True)
|
8076
|
-
class DeployAppConfSpec:
|
8077
|
-
files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
|
8078
|
-
|
8079
|
-
links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
|
8080
|
-
|
8081
|
-
def __post_init__(self) -> None:
|
8082
|
-
if self.files:
|
8083
|
-
seen: ta.Set[str] = set()
|
8084
|
-
for f in self.files:
|
8085
|
-
check.not_in(f.path, seen)
|
8086
|
-
seen.add(f.path)
|
8099
|
+
) -> None:
|
8100
|
+
super().__init__()
|
8101
|
+
|
8102
|
+
self._command_executors = command_executors
|
8103
|
+
|
8104
|
+
async def execute(self, cmd: Command) -> Command.Output:
|
8105
|
+
ce: CommandExecutor = self._command_executors[type(cmd)]
|
8106
|
+
return await ce.execute(cmd)
|
8087
8107
|
|
8088
8108
|
|
8089
8109
|
########################################
|
@@ -8288,6 +8308,127 @@ class DeployPath(DeployPathRenderable):
|
|
8288
8308
|
return cls(tuple(DeployPathPart.parse(p) for p in ps))
|
8289
8309
|
|
8290
8310
|
|
8311
|
+
########################################
|
8312
|
+
# ../deploy/specs.py
|
8313
|
+
|
8314
|
+
|
8315
|
+
##
|
8316
|
+
|
8317
|
+
|
8318
|
+
class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
|
8319
|
+
@cached_nullary
|
8320
|
+
def _key_str(self) -> str:
|
8321
|
+
return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
|
8322
|
+
|
8323
|
+
@abc.abstractmethod
|
8324
|
+
def key(self) -> KeyDeployTagT:
|
8325
|
+
raise NotImplementedError
|
8326
|
+
|
8327
|
+
|
8328
|
+
##
|
8329
|
+
|
8330
|
+
|
8331
|
+
@dc.dataclass(frozen=True)
|
8332
|
+
class DeployGitRepo:
|
8333
|
+
host: ta.Optional[str] = None
|
8334
|
+
username: ta.Optional[str] = None
|
8335
|
+
path: ta.Optional[str] = None
|
8336
|
+
|
8337
|
+
def __post_init__(self) -> None:
|
8338
|
+
check.not_in('..', check.non_empty_str(self.host))
|
8339
|
+
check.not_in('.', check.non_empty_str(self.path))
|
8340
|
+
|
8341
|
+
|
8342
|
+
@dc.dataclass(frozen=True)
|
8343
|
+
class DeployGitSpec:
|
8344
|
+
repo: DeployGitRepo
|
8345
|
+
rev: DeployRev
|
8346
|
+
|
8347
|
+
subtrees: ta.Optional[ta.Sequence[str]] = None
|
8348
|
+
|
8349
|
+
def __post_init__(self) -> None:
|
8350
|
+
check.non_empty_str(self.rev)
|
8351
|
+
if self.subtrees is not None:
|
8352
|
+
for st in self.subtrees:
|
8353
|
+
check.non_empty_str(st)
|
8354
|
+
|
8355
|
+
|
8356
|
+
##
|
8357
|
+
|
8358
|
+
|
8359
|
+
@dc.dataclass(frozen=True)
|
8360
|
+
class DeployVenvSpec:
|
8361
|
+
interp: ta.Optional[str] = None
|
8362
|
+
|
8363
|
+
requirements_files: ta.Optional[ta.Sequence[str]] = None
|
8364
|
+
extra_dependencies: ta.Optional[ta.Sequence[str]] = None
|
8365
|
+
|
8366
|
+
use_uv: bool = False
|
8367
|
+
|
8368
|
+
|
8369
|
+
##
|
8370
|
+
|
8371
|
+
|
8372
|
+
@dc.dataclass(frozen=True)
|
8373
|
+
class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
|
8374
|
+
app: DeployApp
|
8375
|
+
|
8376
|
+
git: DeployGitSpec
|
8377
|
+
|
8378
|
+
venv: ta.Optional[DeployVenvSpec] = None
|
8379
|
+
|
8380
|
+
conf: ta.Optional[DeployAppConfSpec] = None
|
8381
|
+
|
8382
|
+
# @ta.override
|
8383
|
+
def key(self) -> DeployAppKey:
|
8384
|
+
return DeployAppKey(self._key_str())
|
8385
|
+
|
8386
|
+
|
8387
|
+
@dc.dataclass(frozen=True)
|
8388
|
+
class DeployAppLinksSpec:
|
8389
|
+
apps: ta.Sequence[DeployApp] = ()
|
8390
|
+
|
8391
|
+
removed_apps: ta.Sequence[DeployApp] = ()
|
8392
|
+
|
8393
|
+
exclude_unspecified: bool = False
|
8394
|
+
|
8395
|
+
|
8396
|
+
##
|
8397
|
+
|
8398
|
+
|
8399
|
+
@dc.dataclass(frozen=True)
|
8400
|
+
class DeploySystemdSpec:
|
8401
|
+
# ~/.config/systemd/user/
|
8402
|
+
unit_dir: ta.Optional[str] = None
|
8403
|
+
|
8404
|
+
|
8405
|
+
##
|
8406
|
+
|
8407
|
+
|
8408
|
+
@dc.dataclass(frozen=True)
|
8409
|
+
class DeploySpec(DeploySpecKeyed[DeployKey]):
|
8410
|
+
home: DeployHome
|
8411
|
+
|
8412
|
+
apps: ta.Sequence[DeployAppSpec] = ()
|
8413
|
+
|
8414
|
+
app_links: DeployAppLinksSpec = DeployAppLinksSpec()
|
8415
|
+
|
8416
|
+
systemd: ta.Optional[DeploySystemdSpec] = None
|
8417
|
+
|
8418
|
+
def __post_init__(self) -> None:
|
8419
|
+
check.non_empty_str(self.home)
|
8420
|
+
|
8421
|
+
seen: ta.Set[DeployApp] = set()
|
8422
|
+
for a in self.apps:
|
8423
|
+
if a.app in seen:
|
8424
|
+
raise KeyError(a.app)
|
8425
|
+
seen.add(a.app)
|
8426
|
+
|
8427
|
+
# @ta.override
|
8428
|
+
def key(self) -> DeployKey:
|
8429
|
+
return DeployKey(self._key_str())
|
8430
|
+
|
8431
|
+
|
8291
8432
|
########################################
|
8292
8433
|
# ../remote/execution.py
|
8293
8434
|
"""
|
@@ -9320,7 +9461,7 @@ class DeployConfManager:
|
|
9320
9461
|
|
9321
9462
|
elif isinstance(ac, IniDeployAppConfContent):
|
9322
9463
|
ini_sections = pcc(ac.sections)
|
9323
|
-
return strip_with_newline(
|
9464
|
+
return strip_with_newline(render_ini_sections(ini_sections))
|
9324
9465
|
|
9325
9466
|
elif isinstance(ac, NginxDeployAppConfContent):
|
9326
9467
|
nginx_items = NginxConfigItems.of(pcc(ac.items))
|
@@ -9353,9 +9494,15 @@ class DeployConfManager:
|
|
9353
9494
|
self,
|
9354
9495
|
spec: DeployAppConfSpec,
|
9355
9496
|
app_conf_dir: str,
|
9497
|
+
*,
|
9498
|
+
string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
9356
9499
|
) -> None:
|
9357
|
-
|
9358
|
-
|
9500
|
+
process_str: ta.Any
|
9501
|
+
if string_ns is not None:
|
9502
|
+
def process_str(s: str) -> str:
|
9503
|
+
return s.format(**string_ns)
|
9504
|
+
else:
|
9505
|
+
process_str = None
|
9359
9506
|
|
9360
9507
|
for acf in spec.files or []:
|
9361
9508
|
await self._write_app_conf_file(
|
@@ -9528,125 +9675,6 @@ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
|
|
9528
9675
|
return self._owned_deploy_paths
|
9529
9676
|
|
9530
9677
|
|
9531
|
-
########################################
|
9532
|
-
# ../deploy/specs.py
|
9533
|
-
|
9534
|
-
|
9535
|
-
##
|
9536
|
-
|
9537
|
-
|
9538
|
-
class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
|
9539
|
-
@cached_nullary
|
9540
|
-
def _key_str(self) -> str:
|
9541
|
-
return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
|
9542
|
-
|
9543
|
-
@abc.abstractmethod
|
9544
|
-
def key(self) -> KeyDeployTagT:
|
9545
|
-
raise NotImplementedError
|
9546
|
-
|
9547
|
-
|
9548
|
-
##
|
9549
|
-
|
9550
|
-
|
9551
|
-
@dc.dataclass(frozen=True)
|
9552
|
-
class DeployGitRepo:
|
9553
|
-
host: ta.Optional[str] = None
|
9554
|
-
username: ta.Optional[str] = None
|
9555
|
-
path: ta.Optional[str] = None
|
9556
|
-
|
9557
|
-
def __post_init__(self) -> None:
|
9558
|
-
check.not_in('..', check.non_empty_str(self.host))
|
9559
|
-
check.not_in('.', check.non_empty_str(self.path))
|
9560
|
-
|
9561
|
-
|
9562
|
-
@dc.dataclass(frozen=True)
|
9563
|
-
class DeployGitSpec:
|
9564
|
-
repo: DeployGitRepo
|
9565
|
-
rev: DeployRev
|
9566
|
-
|
9567
|
-
subtrees: ta.Optional[ta.Sequence[str]] = None
|
9568
|
-
|
9569
|
-
def __post_init__(self) -> None:
|
9570
|
-
check.non_empty_str(self.rev)
|
9571
|
-
if self.subtrees is not None:
|
9572
|
-
for st in self.subtrees:
|
9573
|
-
check.non_empty_str(st)
|
9574
|
-
|
9575
|
-
|
9576
|
-
##
|
9577
|
-
|
9578
|
-
|
9579
|
-
@dc.dataclass(frozen=True)
|
9580
|
-
class DeployVenvSpec:
|
9581
|
-
interp: ta.Optional[str] = None
|
9582
|
-
|
9583
|
-
requirements_files: ta.Optional[ta.Sequence[str]] = None
|
9584
|
-
extra_dependencies: ta.Optional[ta.Sequence[str]] = None
|
9585
|
-
|
9586
|
-
use_uv: bool = False
|
9587
|
-
|
9588
|
-
|
9589
|
-
##
|
9590
|
-
|
9591
|
-
|
9592
|
-
@dc.dataclass(frozen=True)
|
9593
|
-
class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
|
9594
|
-
app: DeployApp
|
9595
|
-
|
9596
|
-
git: DeployGitSpec
|
9597
|
-
|
9598
|
-
venv: ta.Optional[DeployVenvSpec] = None
|
9599
|
-
|
9600
|
-
conf: ta.Optional[DeployAppConfSpec] = None
|
9601
|
-
|
9602
|
-
# @ta.override
|
9603
|
-
def key(self) -> DeployAppKey:
|
9604
|
-
return DeployAppKey(self._key_str())
|
9605
|
-
|
9606
|
-
|
9607
|
-
@dc.dataclass(frozen=True)
|
9608
|
-
class DeployAppLinksSpec:
|
9609
|
-
apps: ta.Sequence[DeployApp] = ()
|
9610
|
-
|
9611
|
-
exclude_unspecified: bool = False
|
9612
|
-
|
9613
|
-
|
9614
|
-
##
|
9615
|
-
|
9616
|
-
|
9617
|
-
@dc.dataclass(frozen=True)
|
9618
|
-
class DeploySystemdSpec:
|
9619
|
-
# ~/.config/systemd/user/
|
9620
|
-
unit_dir: ta.Optional[str] = None
|
9621
|
-
|
9622
|
-
|
9623
|
-
##
|
9624
|
-
|
9625
|
-
|
9626
|
-
@dc.dataclass(frozen=True)
|
9627
|
-
class DeploySpec(DeploySpecKeyed[DeployKey]):
|
9628
|
-
home: DeployHome
|
9629
|
-
|
9630
|
-
apps: ta.Sequence[DeployAppSpec] = ()
|
9631
|
-
|
9632
|
-
app_links: DeployAppLinksSpec = DeployAppLinksSpec()
|
9633
|
-
|
9634
|
-
systemd: ta.Optional[DeploySystemdSpec] = None
|
9635
|
-
|
9636
|
-
def __post_init__(self) -> None:
|
9637
|
-
check.non_empty_str(self.home)
|
9638
|
-
|
9639
|
-
seen: ta.Set[DeployApp] = set()
|
9640
|
-
for a in self.apps:
|
9641
|
-
if a.app in seen:
|
9642
|
-
raise KeyError(a.app)
|
9643
|
-
seen.add(a.app)
|
9644
|
-
|
9645
|
-
# @ta.override
|
9646
|
-
def key(self) -> DeployKey:
|
9647
|
-
return DeployKey(self._key_str())
|
9648
|
-
|
9649
|
-
|
9650
9678
|
########################################
|
9651
9679
|
# ../remote/_main.py
|
9652
9680
|
|
@@ -10967,7 +10995,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
|
|
10967
10995
|
|
10968
10996
|
async def fetch(self, rev: DeployRev) -> None:
|
10969
10997
|
await self.init()
|
10970
|
-
|
10998
|
+
|
10999
|
+
# This fetch shouldn't be depth=1 - git doesn't reuse local data with shallow fetches.
|
11000
|
+
await self._call('git', 'fetch', 'origin', rev)
|
10971
11001
|
|
10972
11002
|
#
|
10973
11003
|
|
@@ -11125,7 +11155,7 @@ class DeploySystemdManager:
|
|
11125
11155
|
|
11126
11156
|
#
|
11127
11157
|
|
11128
|
-
if sys.platform == 'linux':
|
11158
|
+
if sys.platform == 'linux' and shutil.which('systemctl') is not None:
|
11129
11159
|
async def reload() -> None:
|
11130
11160
|
await asyncio_subprocesses.check_call('systemctl', '--user', 'daemon-reload')
|
11131
11161
|
|
@@ -11614,6 +11644,8 @@ class DeployAppManager(DeployPathOwner):
|
|
11614
11644
|
spec: DeployAppSpec,
|
11615
11645
|
home: DeployHome,
|
11616
11646
|
tags: DeployTagMap,
|
11647
|
+
*,
|
11648
|
+
conf_string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
11617
11649
|
) -> PreparedApp:
|
11618
11650
|
spec_json = json_dumps_pretty(self._msh.marshal_obj(spec))
|
11619
11651
|
|
@@ -11670,9 +11702,17 @@ class DeployAppManager(DeployPathOwner):
|
|
11670
11702
|
if spec.conf is not None:
|
11671
11703
|
conf_dir = os.path.join(app_dir, 'conf')
|
11672
11704
|
rkw.update(conf_dir=conf_dir)
|
11705
|
+
|
11706
|
+
conf_ns: ta.Dict[str, ta.Any] = dict(
|
11707
|
+
**(conf_string_ns or {}),
|
11708
|
+
app=spec.app.s,
|
11709
|
+
app_dir=app_dir.rstrip('/'),
|
11710
|
+
)
|
11711
|
+
|
11673
11712
|
await self._conf.write_app_conf(
|
11674
11713
|
spec.conf,
|
11675
11714
|
conf_dir,
|
11715
|
+
string_ns=conf_ns,
|
11676
11716
|
)
|
11677
11717
|
|
11678
11718
|
#
|
@@ -11735,7 +11775,7 @@ class DeployAppManager(DeployPathOwner):
|
|
11735
11775
|
##
|
11736
11776
|
|
11737
11777
|
|
11738
|
-
DEPLOY_TAG_DATETIME_FMT = '%Y
|
11778
|
+
DEPLOY_TAG_DATETIME_FMT = '%Y-%m-%d-T-%H-%M-%S-%f-Z'
|
11739
11779
|
|
11740
11780
|
|
11741
11781
|
DeployManagerUtcClock = ta.NewType('DeployManagerUtcClock', Func0[datetime.datetime])
|
@@ -11888,8 +11928,9 @@ class DeployDriver:
|
|
11888
11928
|
|
11889
11929
|
das: ta.Set[DeployApp] = {a.app for a in self._spec.apps}
|
11890
11930
|
las: ta.Set[DeployApp] = set(self._spec.app_links.apps)
|
11891
|
-
|
11892
|
-
|
11931
|
+
ras: ta.Set[DeployApp] = set(self._spec.app_links.removed_apps)
|
11932
|
+
check.empty(das & (las | ras))
|
11933
|
+
check.empty(las & ras)
|
11893
11934
|
|
11894
11935
|
#
|
11895
11936
|
|
@@ -11935,10 +11976,10 @@ class DeployDriver:
|
|
11935
11976
|
cad = abs_real_path(os.path.join(current_link, 'apps'))
|
11936
11977
|
if os.path.exists(cad):
|
11937
11978
|
for d in os.listdir(cad):
|
11938
|
-
if (da := DeployApp(d)) not in das:
|
11979
|
+
if (da := DeployApp(d)) not in das and da not in ras:
|
11939
11980
|
las.add(da)
|
11940
11981
|
|
11941
|
-
for la in
|
11982
|
+
for la in las:
|
11942
11983
|
await self._drive_app_link(
|
11943
11984
|
la,
|
11944
11985
|
current_link,
|
@@ -11968,10 +12009,16 @@ class DeployDriver:
|
|
11968
12009
|
#
|
11969
12010
|
|
11970
12011
|
async def _drive_app_deploy(self, app: DeployAppSpec) -> None:
|
12012
|
+
current_deploy_link = os.path.join(self._home, self._deploys.CURRENT_DEPLOY_LINK.render())
|
12013
|
+
|
11971
12014
|
pa = await self._apps.prepare_app(
|
11972
12015
|
app,
|
11973
12016
|
self._home,
|
11974
12017
|
self.tags,
|
12018
|
+
conf_string_ns=dict(
|
12019
|
+
deploy_home=self._home,
|
12020
|
+
current_deploy_link=current_deploy_link,
|
12021
|
+
),
|
11975
12022
|
)
|
11976
12023
|
|
11977
12024
|
#
|