ominfra 0.0.0.dev191__py3-none-any.whl → 0.0.0.dev193__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/configs.py +1 -28
- ominfra/manage/deploy/apps.py +10 -0
- ominfra/manage/deploy/conf/manager.py +10 -4
- ominfra/manage/deploy/conf/specs.py +2 -2
- ominfra/manage/deploy/deploy.py +12 -5
- ominfra/manage/deploy/git.py +3 -1
- ominfra/manage/deploy/specs.py +2 -0
- ominfra/manage/deploy/systemd.py +2 -1
- ominfra/scripts/journald2aws.py +2 -27
- ominfra/scripts/manage.py +1495 -1448
- ominfra/scripts/supervisor.py +1056 -1072
- ominfra/supervisor/configs.py +6 -0
- ominfra/supervisor/http.py +1 -1
- ominfra/supervisor/inject.py +11 -8
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/METADATA +4 -4
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/RECORD +20 -20
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev191.dist-info → ominfra-0.0.0.dev193.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -16,6 +16,7 @@ import asyncio.subprocess
|
|
16
16
|
import base64
|
17
17
|
import collections
|
18
18
|
import collections.abc
|
19
|
+
import configparser
|
19
20
|
import contextlib
|
20
21
|
import contextvars
|
21
22
|
import ctypes as ct
|
@@ -73,17 +74,20 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
|
|
73
74
|
VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
|
74
75
|
VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
|
75
76
|
|
76
|
-
# ../../omdev/toml/parser.py
|
77
|
-
TomlParseFloat = ta.Callable[[str], ta.Any]
|
78
|
-
TomlKey = ta.Tuple[str, ...]
|
79
|
-
TomlPos = int # ta.TypeAlias
|
80
|
-
|
81
77
|
# deploy/paths/types.py
|
82
78
|
DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
|
83
79
|
|
84
80
|
# ../../omlish/asyncs/asyncio/timeouts.py
|
85
81
|
AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
|
86
82
|
|
83
|
+
# ../../omlish/formats/ini/sections.py
|
84
|
+
IniSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
|
85
|
+
|
86
|
+
# ../../omlish/formats/toml/parser.py
|
87
|
+
TomlParseFloat = ta.Callable[[str], ta.Any]
|
88
|
+
TomlKey = ta.Tuple[str, ...]
|
89
|
+
TomlPos = int # ta.TypeAlias
|
90
|
+
|
87
91
|
# ../../omlish/lite/cached.py
|
88
92
|
T = ta.TypeVar('T')
|
89
93
|
CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
@@ -129,7 +133,6 @@ AtomicPathSwapState = ta.Literal['open', 'committed', 'aborted'] # ta.TypeAlias
|
|
129
133
|
|
130
134
|
# ../configs.py
|
131
135
|
ConfigMapping = ta.Mapping[str, ta.Any]
|
132
|
-
IniConfigSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
|
133
136
|
|
134
137
|
# ../../omlish/subprocesses.py
|
135
138
|
SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
|
@@ -549,1482 +552,1523 @@ def canonicalize_version(
|
|
549
552
|
|
550
553
|
|
551
554
|
########################################
|
552
|
-
#
|
553
|
-
# SPDX-License-Identifier: MIT
|
554
|
-
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
555
|
-
# Licensed to PSF under a Contributor Agreement.
|
556
|
-
#
|
557
|
-
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
558
|
-
# --------------------------------------------
|
559
|
-
#
|
560
|
-
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
561
|
-
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
562
|
-
# documentation.
|
563
|
-
#
|
564
|
-
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
565
|
-
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
566
|
-
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
567
|
-
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
568
|
-
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
569
|
-
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
570
|
-
#
|
571
|
-
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
572
|
-
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
573
|
-
# any such work a brief summary of the changes made to Python.
|
574
|
-
#
|
575
|
-
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
576
|
-
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
577
|
-
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
578
|
-
# RIGHTS.
|
579
|
-
#
|
580
|
-
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
581
|
-
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
582
|
-
# ADVISED OF THE POSSIBILITY THEREOF.
|
583
|
-
#
|
584
|
-
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
585
|
-
#
|
586
|
-
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
587
|
-
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
588
|
-
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
589
|
-
#
|
590
|
-
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
591
|
-
# License Agreement.
|
592
|
-
#
|
593
|
-
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
555
|
+
# ../config.py
|
594
556
|
|
595
557
|
|
596
|
-
|
558
|
+
@dc.dataclass(frozen=True)
|
559
|
+
class MainConfig:
|
560
|
+
log_level: ta.Optional[str] = 'INFO'
|
597
561
|
|
562
|
+
debug: bool = False
|
598
563
|
|
599
|
-
_TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
|
600
564
|
|
601
|
-
|
602
|
-
|
603
|
-
0
|
604
|
-
(?:
|
605
|
-
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
606
|
-
|
|
607
|
-
b[01](?:_?[01])* # bin
|
608
|
-
|
|
609
|
-
o[0-7](?:_?[0-7])* # oct
|
610
|
-
)
|
611
|
-
|
|
612
|
-
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
613
|
-
(?P<floatpart>
|
614
|
-
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
615
|
-
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
616
|
-
)
|
617
|
-
""",
|
618
|
-
flags=re.VERBOSE,
|
619
|
-
)
|
620
|
-
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
621
|
-
TOML_RE_DATETIME = re.compile(
|
622
|
-
rf"""
|
623
|
-
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
624
|
-
(?:
|
625
|
-
[Tt ]
|
626
|
-
{_TOML_TIME_RE_STR}
|
627
|
-
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
628
|
-
)?
|
629
|
-
""",
|
630
|
-
flags=re.VERBOSE,
|
631
|
-
)
|
565
|
+
########################################
|
566
|
+
# ../deploy/config.py
|
632
567
|
|
633
568
|
|
634
|
-
|
635
|
-
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
569
|
+
##
|
636
570
|
|
637
|
-
Raises ValueError if the match does not correspond to a valid date or datetime.
|
638
|
-
"""
|
639
|
-
(
|
640
|
-
year_str,
|
641
|
-
month_str,
|
642
|
-
day_str,
|
643
|
-
hour_str,
|
644
|
-
minute_str,
|
645
|
-
sec_str,
|
646
|
-
micros_str,
|
647
|
-
zulu_time,
|
648
|
-
offset_sign_str,
|
649
|
-
offset_hour_str,
|
650
|
-
offset_minute_str,
|
651
|
-
) = match.groups()
|
652
|
-
year, month, day = int(year_str), int(month_str), int(day_str)
|
653
|
-
if hour_str is None:
|
654
|
-
return datetime.date(year, month, day)
|
655
|
-
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
656
|
-
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
657
|
-
if offset_sign_str:
|
658
|
-
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
659
|
-
offset_hour_str, offset_minute_str, offset_sign_str,
|
660
|
-
)
|
661
|
-
elif zulu_time:
|
662
|
-
tz = datetime.UTC
|
663
|
-
else: # local date-time
|
664
|
-
tz = None
|
665
|
-
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
666
571
|
|
572
|
+
@dc.dataclass(frozen=True)
|
573
|
+
class DeployConfig:
|
574
|
+
pass
|
667
575
|
|
668
|
-
@functools.lru_cache() # noqa
|
669
|
-
def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
|
670
|
-
sign = 1 if sign_str == '+' else -1
|
671
|
-
return datetime.timezone(
|
672
|
-
datetime.timedelta(
|
673
|
-
hours=sign * int(hour_str),
|
674
|
-
minutes=sign * int(minute_str),
|
675
|
-
),
|
676
|
-
)
|
677
576
|
|
577
|
+
########################################
|
578
|
+
# ../deploy/paths/types.py
|
678
579
|
|
679
|
-
def toml_match_to_localtime(match: re.Match) -> datetime.time:
|
680
|
-
hour_str, minute_str, sec_str, micros_str = match.groups()
|
681
|
-
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
682
|
-
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
683
580
|
|
581
|
+
##
|
684
582
|
|
685
|
-
def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
|
686
|
-
if match.group('floatpart'):
|
687
|
-
return parse_float(match.group())
|
688
|
-
return int(match.group(), 0)
|
689
583
|
|
584
|
+
########################################
|
585
|
+
# ../deploy/types.py
|
690
586
|
|
691
|
-
TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
|
692
587
|
|
693
|
-
|
694
|
-
# functions.
|
695
|
-
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
696
|
-
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
588
|
+
##
|
697
589
|
|
698
|
-
TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
699
|
-
TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
700
590
|
|
701
|
-
|
591
|
+
DeployHome = ta.NewType('DeployHome', str)
|
702
592
|
|
703
|
-
|
704
|
-
TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
|
705
|
-
TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
|
706
|
-
TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
|
707
|
-
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
593
|
+
DeployRev = ta.NewType('DeployRev', str)
|
708
594
|
|
709
|
-
TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
|
710
|
-
{
|
711
|
-
'\\b': '\u0008', # backspace
|
712
|
-
'\\t': '\u0009', # tab
|
713
|
-
'\\n': '\u000A', # linefeed
|
714
|
-
'\\f': '\u000C', # form feed
|
715
|
-
'\\r': '\u000D', # carriage return
|
716
|
-
'\\"': '\u0022', # quote
|
717
|
-
'\\\\': '\u005C', # backslash
|
718
|
-
},
|
719
|
-
)
|
720
595
|
|
596
|
+
########################################
|
597
|
+
# ../../pyremote.py
|
598
|
+
"""
|
599
|
+
Basically this: https://mitogen.networkgenomics.com/howitworks.html
|
721
600
|
|
722
|
-
|
723
|
-
|
601
|
+
TODO:
|
602
|
+
- log: ta.Optional[logging.Logger] = None + log.debug's
|
603
|
+
"""
|
724
604
|
|
725
605
|
|
726
|
-
|
727
|
-
"""Parse TOML from a binary file object."""
|
728
|
-
b = fp.read()
|
729
|
-
try:
|
730
|
-
s = b.decode()
|
731
|
-
except AttributeError:
|
732
|
-
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
733
|
-
return toml_loads(s, parse_float=parse_float)
|
606
|
+
##
|
734
607
|
|
735
608
|
|
736
|
-
|
737
|
-
|
609
|
+
@dc.dataclass(frozen=True)
|
610
|
+
class PyremoteBootstrapOptions:
|
611
|
+
debug: bool = False
|
738
612
|
|
739
|
-
|
740
|
-
|
741
|
-
src = s.replace('\r\n', '\n')
|
742
|
-
except (AttributeError, TypeError):
|
743
|
-
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
744
|
-
pos = 0
|
745
|
-
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
746
|
-
header: TomlKey = ()
|
747
|
-
parse_float = toml_make_safe_parse_float(parse_float)
|
613
|
+
DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
|
614
|
+
main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
|
748
615
|
|
749
|
-
# Parse one statement at a time (typically means one line in TOML source)
|
750
|
-
while True:
|
751
|
-
# 1. Skip line leading whitespace
|
752
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
753
616
|
|
754
|
-
|
755
|
-
# - end of file
|
756
|
-
# - end of line
|
757
|
-
# - comment
|
758
|
-
# - key/value pair
|
759
|
-
# - append dict to list (and move to its namespace)
|
760
|
-
# - create dict (and move to its namespace)
|
761
|
-
# Skip trailing whitespace when applicable.
|
762
|
-
try:
|
763
|
-
char = src[pos]
|
764
|
-
except IndexError:
|
765
|
-
break
|
766
|
-
if char == '\n':
|
767
|
-
pos += 1
|
768
|
-
continue
|
769
|
-
if char in TOML_KEY_INITIAL_CHARS:
|
770
|
-
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
771
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
772
|
-
elif char == '[':
|
773
|
-
try:
|
774
|
-
second_char: ta.Optional[str] = src[pos + 1]
|
775
|
-
except IndexError:
|
776
|
-
second_char = None
|
777
|
-
out.flags.finalize_pending()
|
778
|
-
if second_char == '[':
|
779
|
-
pos, header = toml_create_list_rule(src, pos, out)
|
780
|
-
else:
|
781
|
-
pos, header = toml_create_dict_rule(src, pos, out)
|
782
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
783
|
-
elif char != '#':
|
784
|
-
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
617
|
+
##
|
785
618
|
|
786
|
-
# 3. Skip comment
|
787
|
-
pos = toml_skip_comment(src, pos)
|
788
619
|
|
789
|
-
|
790
|
-
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
620
|
+
@dc.dataclass(frozen=True)
|
621
|
+
class PyremoteEnvInfo:
|
622
|
+
sys_base_prefix: str
|
623
|
+
sys_byteorder: str
|
624
|
+
sys_defaultencoding: str
|
625
|
+
sys_exec_prefix: str
|
626
|
+
sys_executable: str
|
627
|
+
sys_implementation_name: str
|
628
|
+
sys_path: ta.List[str]
|
629
|
+
sys_platform: str
|
630
|
+
sys_prefix: str
|
631
|
+
sys_version: str
|
632
|
+
sys_version_info: ta.List[ta.Union[int, str]]
|
799
633
|
|
800
|
-
|
634
|
+
platform_architecture: ta.List[str]
|
635
|
+
platform_machine: str
|
636
|
+
platform_platform: str
|
637
|
+
platform_processor: str
|
638
|
+
platform_system: str
|
639
|
+
platform_release: str
|
640
|
+
platform_version: str
|
801
641
|
|
642
|
+
site_userbase: str
|
802
643
|
|
803
|
-
|
804
|
-
|
644
|
+
os_cwd: str
|
645
|
+
os_gid: int
|
646
|
+
os_loadavg: ta.List[float]
|
647
|
+
os_login: ta.Optional[str]
|
648
|
+
os_pgrp: int
|
649
|
+
os_pid: int
|
650
|
+
os_ppid: int
|
651
|
+
os_uid: int
|
805
652
|
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
|
653
|
+
pw_name: str
|
654
|
+
pw_uid: int
|
655
|
+
pw_gid: int
|
656
|
+
pw_gecos: str
|
657
|
+
pw_dir: str
|
658
|
+
pw_shell: str
|
810
659
|
|
811
|
-
|
812
|
-
self._flags: ta.Dict[str, dict] = {}
|
813
|
-
self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
|
660
|
+
env_path: ta.Optional[str]
|
814
661
|
|
815
|
-
def add_pending(self, key: TomlKey, flag: int) -> None:
|
816
|
-
self._pending_flags.add((key, flag))
|
817
662
|
|
818
|
-
|
819
|
-
|
820
|
-
self.set(key, flag, recursive=False)
|
821
|
-
self._pending_flags.clear()
|
663
|
+
def _get_pyremote_env_info() -> PyremoteEnvInfo:
|
664
|
+
os_uid = os.getuid()
|
822
665
|
|
823
|
-
|
824
|
-
cont = self._flags
|
825
|
-
for k in key[:-1]:
|
826
|
-
if k not in cont:
|
827
|
-
return
|
828
|
-
cont = cont[k]['nested']
|
829
|
-
cont.pop(key[-1], None)
|
666
|
+
pw = pwd.getpwuid(os_uid)
|
830
667
|
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
|
835
|
-
|
836
|
-
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
837
|
-
cont = cont[k]['nested']
|
838
|
-
if key_stem not in cont:
|
839
|
-
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
840
|
-
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
668
|
+
os_login: ta.Optional[str]
|
669
|
+
try:
|
670
|
+
os_login = os.getlogin()
|
671
|
+
except OSError:
|
672
|
+
os_login = None
|
841
673
|
|
842
|
-
|
843
|
-
|
844
|
-
|
845
|
-
|
846
|
-
|
847
|
-
|
848
|
-
|
849
|
-
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
if key_stem in cont:
|
855
|
-
cont = cont[key_stem]
|
856
|
-
return flag in cont['flags'] or flag in cont['recursive_flags']
|
857
|
-
return False
|
674
|
+
return PyremoteEnvInfo(
|
675
|
+
sys_base_prefix=sys.base_prefix,
|
676
|
+
sys_byteorder=sys.byteorder,
|
677
|
+
sys_defaultencoding=sys.getdefaultencoding(),
|
678
|
+
sys_exec_prefix=sys.exec_prefix,
|
679
|
+
sys_executable=sys.executable,
|
680
|
+
sys_implementation_name=sys.implementation.name,
|
681
|
+
sys_path=sys.path,
|
682
|
+
sys_platform=sys.platform,
|
683
|
+
sys_prefix=sys.prefix,
|
684
|
+
sys_version=sys.version,
|
685
|
+
sys_version_info=list(sys.version_info),
|
858
686
|
|
687
|
+
platform_architecture=list(platform.architecture()),
|
688
|
+
platform_machine=platform.machine(),
|
689
|
+
platform_platform=platform.platform(),
|
690
|
+
platform_processor=platform.processor(),
|
691
|
+
platform_system=platform.system(),
|
692
|
+
platform_release=platform.release(),
|
693
|
+
platform_version=platform.version(),
|
859
694
|
|
860
|
-
|
861
|
-
def __init__(self) -> None:
|
862
|
-
# The parsed content of the TOML document
|
863
|
-
self.dict: ta.Dict[str, ta.Any] = {}
|
695
|
+
site_userbase=site.getuserbase(),
|
864
696
|
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
if k not in cont:
|
874
|
-
cont[k] = {}
|
875
|
-
cont = cont[k]
|
876
|
-
if access_lists and isinstance(cont, list):
|
877
|
-
cont = cont[-1]
|
878
|
-
if not isinstance(cont, dict):
|
879
|
-
raise KeyError('There is no nest behind this key')
|
880
|
-
return cont
|
697
|
+
os_cwd=os.getcwd(),
|
698
|
+
os_gid=os.getgid(),
|
699
|
+
os_loadavg=list(os.getloadavg()),
|
700
|
+
os_login=os_login,
|
701
|
+
os_pgrp=os.getpgrp(),
|
702
|
+
os_pid=os.getpid(),
|
703
|
+
os_ppid=os.getppid(),
|
704
|
+
os_uid=os_uid,
|
881
705
|
|
882
|
-
|
883
|
-
|
884
|
-
|
885
|
-
|
886
|
-
|
887
|
-
|
888
|
-
raise KeyError('An object other than list found behind this key')
|
889
|
-
list_.append({})
|
890
|
-
else:
|
891
|
-
cont[last_key] = [{}]
|
706
|
+
pw_name=pw.pw_name,
|
707
|
+
pw_uid=pw.pw_uid,
|
708
|
+
pw_gid=pw.pw_gid,
|
709
|
+
pw_gecos=pw.pw_gecos,
|
710
|
+
pw_dir=pw.pw_dir,
|
711
|
+
pw_shell=pw.pw_shell,
|
892
712
|
|
713
|
+
env_path=os.environ.get('PATH'),
|
714
|
+
)
|
893
715
|
|
894
|
-
class TomlOutput(ta.NamedTuple):
|
895
|
-
data: TomlNestedDict
|
896
|
-
flags: TomlFlags
|
897
716
|
|
717
|
+
##
|
898
718
|
|
899
|
-
def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
|
900
|
-
try:
|
901
|
-
while src[pos] in chars:
|
902
|
-
pos += 1
|
903
|
-
except IndexError:
|
904
|
-
pass
|
905
|
-
return pos
|
906
719
|
|
720
|
+
_PYREMOTE_BOOTSTRAP_INPUT_FD = 100
|
721
|
+
_PYREMOTE_BOOTSTRAP_SRC_FD = 101
|
907
722
|
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
|
913
|
-
error_on: ta.FrozenSet[str],
|
914
|
-
error_on_eof: bool,
|
915
|
-
) -> TomlPos:
|
916
|
-
try:
|
917
|
-
new_pos = src.index(expect, pos)
|
918
|
-
except ValueError:
|
919
|
-
new_pos = len(src)
|
920
|
-
if error_on_eof:
|
921
|
-
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
723
|
+
_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
|
724
|
+
_PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
|
725
|
+
_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
|
726
|
+
_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
|
727
|
+
_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
|
922
728
|
|
923
|
-
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
return new_pos
|
729
|
+
_PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
|
730
|
+
_PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
|
731
|
+
_PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
|
732
|
+
_PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
|
928
733
|
|
734
|
+
_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
|
929
735
|
|
930
|
-
|
931
|
-
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
|
937
|
-
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
938
|
-
)
|
939
|
-
return pos
|
736
|
+
_PYREMOTE_BOOTSTRAP_IMPORTS = [
|
737
|
+
'base64',
|
738
|
+
'os',
|
739
|
+
'struct',
|
740
|
+
'sys',
|
741
|
+
'zlib',
|
742
|
+
]
|
940
743
|
|
941
744
|
|
942
|
-
def
|
943
|
-
|
944
|
-
|
945
|
-
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
946
|
-
pos = toml_skip_comment(src, pos)
|
947
|
-
if pos == pos_before_skip:
|
948
|
-
return pos
|
745
|
+
def _pyremote_bootstrap_main(context_name: str) -> None:
|
746
|
+
# Get pid
|
747
|
+
pid = os.getpid()
|
949
748
|
|
749
|
+
# Two copies of payload src to be sent to parent
|
750
|
+
r0, w0 = os.pipe()
|
751
|
+
r1, w1 = os.pipe()
|
950
752
|
|
951
|
-
|
952
|
-
|
953
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
954
|
-
pos, key = toml_parse_key(src, pos)
|
753
|
+
if (cp := os.fork()):
|
754
|
+
# Parent process
|
955
755
|
|
956
|
-
|
957
|
-
|
958
|
-
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
959
|
-
try:
|
960
|
-
out.data.get_or_create_nest(key)
|
961
|
-
except KeyError:
|
962
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
756
|
+
# Dup original stdin to comm_fd for use as comm channel
|
757
|
+
os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
|
963
758
|
|
964
|
-
|
965
|
-
|
966
|
-
return pos + 1, key
|
759
|
+
# Overwrite stdin (fed to python repl) with first copy of src
|
760
|
+
os.dup2(r0, 0)
|
967
761
|
|
762
|
+
# Dup second copy of src to src_fd to recover after launch
|
763
|
+
os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
|
968
764
|
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
pos, key = toml_parse_key(src, pos)
|
973
|
-
|
974
|
-
if out.flags.is_(key, TomlFlags.FROZEN):
|
975
|
-
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
976
|
-
# Free the namespace now that it points to another empty list item...
|
977
|
-
out.flags.unset_all(key)
|
978
|
-
# ...but this key precisely is still prohibited from table declaration
|
979
|
-
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
980
|
-
try:
|
981
|
-
out.data.append_nest_to_list(key)
|
982
|
-
except KeyError:
|
983
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
765
|
+
# Close remaining fd's
|
766
|
+
for f in [r0, w0, r1, w1]:
|
767
|
+
os.close(f)
|
984
768
|
|
985
|
-
|
986
|
-
|
987
|
-
|
769
|
+
# Save vars
|
770
|
+
env = os.environ
|
771
|
+
exe = sys.executable
|
772
|
+
env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
|
773
|
+
env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
|
774
|
+
env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
|
988
775
|
|
776
|
+
# Start repl reading stdin from r0
|
777
|
+
os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
|
989
778
|
|
990
|
-
|
991
|
-
|
992
|
-
pos: TomlPos,
|
993
|
-
out: TomlOutput,
|
994
|
-
header: TomlKey,
|
995
|
-
parse_float: TomlParseFloat,
|
996
|
-
) -> TomlPos:
|
997
|
-
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
998
|
-
key_parent, key_stem = key[:-1], key[-1]
|
999
|
-
abs_key_parent = header + key_parent
|
779
|
+
else:
|
780
|
+
# Child process
|
1000
781
|
|
1001
|
-
|
1002
|
-
|
1003
|
-
# Check that dotted key syntax does not redefine an existing table
|
1004
|
-
if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
|
1005
|
-
raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
|
1006
|
-
# Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
|
1007
|
-
# table sections.
|
1008
|
-
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
782
|
+
# Write first ack
|
783
|
+
os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
|
1009
784
|
|
1010
|
-
|
1011
|
-
|
1012
|
-
src,
|
1013
|
-
pos,
|
1014
|
-
f'Cannot mutate immutable namespace {abs_key_parent}',
|
1015
|
-
)
|
785
|
+
# Write pid
|
786
|
+
os.write(1, struct.pack('<Q', pid))
|
1016
787
|
|
1017
|
-
|
1018
|
-
|
1019
|
-
|
1020
|
-
|
1021
|
-
|
1022
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
1023
|
-
# Mark inline table and array namespaces recursively immutable
|
1024
|
-
if isinstance(value, (dict, list)):
|
1025
|
-
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
1026
|
-
nest[key_stem] = value
|
1027
|
-
return pos
|
788
|
+
# Read payload src from stdin
|
789
|
+
payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
|
790
|
+
if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
|
791
|
+
raise EOFError
|
792
|
+
payload_src = zlib.decompress(payload_z)
|
1028
793
|
|
794
|
+
# Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
|
795
|
+
# fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
796
|
+
for w in [w0, w1]:
|
797
|
+
fp = os.fdopen(w, 'wb', 0)
|
798
|
+
fp.write(payload_src)
|
799
|
+
fp.close()
|
1029
800
|
|
1030
|
-
|
1031
|
-
|
1032
|
-
pos: TomlPos,
|
1033
|
-
parse_float: TomlParseFloat,
|
1034
|
-
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
1035
|
-
pos, key = toml_parse_key(src, pos)
|
1036
|
-
try:
|
1037
|
-
char: ta.Optional[str] = src[pos]
|
1038
|
-
except IndexError:
|
1039
|
-
char = None
|
1040
|
-
if char != '=':
|
1041
|
-
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
1042
|
-
pos += 1
|
1043
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1044
|
-
pos, value = toml_parse_value(src, pos, parse_float)
|
1045
|
-
return pos, key, value
|
801
|
+
# Write second ack
|
802
|
+
os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
|
1046
803
|
|
804
|
+
# Exit child
|
805
|
+
sys.exit(0)
|
1047
806
|
|
1048
|
-
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
1049
|
-
pos, key_part = toml_parse_key_part(src, pos)
|
1050
|
-
key: TomlKey = (key_part,)
|
1051
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1052
|
-
while True:
|
1053
|
-
try:
|
1054
|
-
char: ta.Optional[str] = src[pos]
|
1055
|
-
except IndexError:
|
1056
|
-
char = None
|
1057
|
-
if char != '.':
|
1058
|
-
return pos, key
|
1059
|
-
pos += 1
|
1060
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1061
|
-
pos, key_part = toml_parse_key_part(src, pos)
|
1062
|
-
key += (key_part,)
|
1063
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1064
807
|
|
808
|
+
##
|
1065
809
|
|
1066
|
-
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1067
|
-
try:
|
1068
|
-
char: ta.Optional[str] = src[pos]
|
1069
|
-
except IndexError:
|
1070
|
-
char = None
|
1071
|
-
if char in TOML_BARE_KEY_CHARS:
|
1072
|
-
start_pos = pos
|
1073
|
-
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
1074
|
-
return pos, src[start_pos:pos]
|
1075
|
-
if char == "'":
|
1076
|
-
return toml_parse_literal_str(src, pos)
|
1077
|
-
if char == '"':
|
1078
|
-
return toml_parse_one_line_basic_str(src, pos)
|
1079
|
-
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
1080
810
|
|
811
|
+
def pyremote_build_bootstrap_cmd(context_name: str) -> str:
|
812
|
+
if any(c in context_name for c in '\'"'):
|
813
|
+
raise NameError(context_name)
|
1081
814
|
|
1082
|
-
|
1083
|
-
|
1084
|
-
|
815
|
+
import inspect
|
816
|
+
import textwrap
|
817
|
+
bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
|
1085
818
|
|
819
|
+
for gl in [
|
820
|
+
'_PYREMOTE_BOOTSTRAP_INPUT_FD',
|
821
|
+
'_PYREMOTE_BOOTSTRAP_SRC_FD',
|
1086
822
|
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
823
|
+
'_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
|
824
|
+
'_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
|
825
|
+
'_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
|
1090
826
|
|
1091
|
-
|
1092
|
-
|
1093
|
-
return pos + 1, array
|
1094
|
-
while True:
|
1095
|
-
pos, val = toml_parse_value(src, pos, parse_float)
|
1096
|
-
array.append(val)
|
1097
|
-
pos = toml_skip_comments_and_array_ws(src, pos)
|
827
|
+
'_PYREMOTE_BOOTSTRAP_ACK0',
|
828
|
+
'_PYREMOTE_BOOTSTRAP_ACK1',
|
1098
829
|
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1102
|
-
if c != ',':
|
1103
|
-
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
1104
|
-
pos += 1
|
830
|
+
'_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
|
831
|
+
]:
|
832
|
+
bs_src = bs_src.replace(gl, repr(globals()[gl]))
|
1105
833
|
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
834
|
+
bs_src = '\n'.join(
|
835
|
+
cl
|
836
|
+
for l in bs_src.splitlines()
|
837
|
+
if (cl := (l.split('#')[0]).rstrip())
|
838
|
+
if cl.strip()
|
839
|
+
)
|
1109
840
|
|
841
|
+
bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
|
842
|
+
bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
|
843
|
+
if b'"' in bs_z85:
|
844
|
+
raise ValueError(bs_z85)
|
1110
845
|
|
1111
|
-
|
1112
|
-
|
1113
|
-
|
1114
|
-
|
846
|
+
stmts = [
|
847
|
+
f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
|
848
|
+
f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
|
849
|
+
f'_pyremote_bootstrap_main("{context_name}")',
|
850
|
+
]
|
1115
851
|
|
1116
|
-
|
1117
|
-
|
1118
|
-
return pos + 1, nested_dict.dict
|
1119
|
-
while True:
|
1120
|
-
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1121
|
-
key_parent, key_stem = key[:-1], key[-1]
|
1122
|
-
if flags.is_(key, TomlFlags.FROZEN):
|
1123
|
-
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1124
|
-
try:
|
1125
|
-
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
1126
|
-
except KeyError:
|
1127
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1128
|
-
if key_stem in nest:
|
1129
|
-
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
1130
|
-
nest[key_stem] = value
|
1131
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1132
|
-
c = src[pos:pos + 1]
|
1133
|
-
if c == '}':
|
1134
|
-
return pos + 1, nested_dict.dict
|
1135
|
-
if c != ',':
|
1136
|
-
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
1137
|
-
if isinstance(value, (dict, list)):
|
1138
|
-
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
1139
|
-
pos += 1
|
1140
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
852
|
+
cmd = '; '.join(stmts)
|
853
|
+
return cmd
|
1141
854
|
|
1142
855
|
|
1143
|
-
|
1144
|
-
src: str,
|
1145
|
-
pos: TomlPos,
|
1146
|
-
*,
|
1147
|
-
multiline: bool = False,
|
1148
|
-
) -> ta.Tuple[TomlPos, str]:
|
1149
|
-
escape_id = src[pos:pos + 2]
|
1150
|
-
pos += 2
|
1151
|
-
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
1152
|
-
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
1153
|
-
# newline.
|
1154
|
-
if escape_id != '\\\n':
|
1155
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
1156
|
-
try:
|
1157
|
-
char = src[pos]
|
1158
|
-
except IndexError:
|
1159
|
-
return pos, ''
|
1160
|
-
if char != '\n':
|
1161
|
-
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
1162
|
-
pos += 1
|
1163
|
-
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1164
|
-
return pos, ''
|
1165
|
-
if escape_id == '\\u':
|
1166
|
-
return toml_parse_hex_char(src, pos, 4)
|
1167
|
-
if escape_id == '\\U':
|
1168
|
-
return toml_parse_hex_char(src, pos, 8)
|
1169
|
-
try:
|
1170
|
-
return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
|
1171
|
-
except KeyError:
|
1172
|
-
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
856
|
+
##
|
1173
857
|
|
1174
858
|
|
1175
|
-
|
1176
|
-
|
859
|
+
@dc.dataclass(frozen=True)
|
860
|
+
class PyremotePayloadRuntime:
|
861
|
+
input: ta.BinaryIO
|
862
|
+
output: ta.BinaryIO
|
863
|
+
context_name: str
|
864
|
+
payload_src: str
|
865
|
+
options: PyremoteBootstrapOptions
|
866
|
+
env_info: PyremoteEnvInfo
|
1177
867
|
|
1178
868
|
|
1179
|
-
def
|
1180
|
-
|
1181
|
-
if
|
1182
|
-
|
1183
|
-
|
1184
|
-
|
1185
|
-
|
1186
|
-
|
1187
|
-
|
869
|
+
def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
870
|
+
# If src file var is not present we need to do initial finalization
|
871
|
+
if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
|
872
|
+
# Read second copy of payload src
|
873
|
+
r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
|
874
|
+
payload_src = r1.read().decode('utf-8')
|
875
|
+
r1.close()
|
876
|
+
|
877
|
+
# Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
|
878
|
+
# a pipe at once.
|
879
|
+
os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
|
1188
880
|
|
881
|
+
# Read options
|
882
|
+
options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
|
883
|
+
if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
|
884
|
+
raise EOFError
|
885
|
+
options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
|
1189
886
|
|
1190
|
-
|
1191
|
-
|
1192
|
-
|
1193
|
-
|
1194
|
-
|
1195
|
-
|
1196
|
-
|
887
|
+
# If debugging, re-exec as file
|
888
|
+
if options.debug:
|
889
|
+
# Write temp source file
|
890
|
+
import tempfile
|
891
|
+
tfd, tfn = tempfile.mkstemp('-pyremote.py')
|
892
|
+
os.write(tfd, payload_src.encode('utf-8'))
|
893
|
+
os.close(tfd)
|
1197
894
|
|
895
|
+
# Set vars
|
896
|
+
os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
|
897
|
+
os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
|
1198
898
|
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
899
|
+
# Re-exec temp file
|
900
|
+
exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
|
901
|
+
context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
|
902
|
+
os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
|
1203
903
|
|
1204
|
-
if literal:
|
1205
|
-
delim = "'"
|
1206
|
-
end_pos = toml_skip_until(
|
1207
|
-
src,
|
1208
|
-
pos,
|
1209
|
-
"'''",
|
1210
|
-
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
1211
|
-
error_on_eof=True,
|
1212
|
-
)
|
1213
|
-
result = src[pos:end_pos]
|
1214
|
-
pos = end_pos + 3
|
1215
904
|
else:
|
1216
|
-
|
1217
|
-
|
905
|
+
# Load options json var
|
906
|
+
options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
|
907
|
+
options = PyremoteBootstrapOptions(**json.loads(options_json_str))
|
1218
908
|
|
1219
|
-
|
1220
|
-
|
1221
|
-
|
1222
|
-
pos += 1
|
1223
|
-
if not src.startswith(delim, pos):
|
1224
|
-
return pos, result + delim
|
1225
|
-
pos += 1
|
1226
|
-
return pos, result + (delim * 2)
|
909
|
+
# Read temp source file
|
910
|
+
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
911
|
+
payload_src = sf.read()
|
1227
912
|
|
913
|
+
# Restore vars
|
914
|
+
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
915
|
+
context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
|
1228
916
|
|
1229
|
-
|
1230
|
-
|
1231
|
-
error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1232
|
-
parse_escapes = toml_parse_basic_str_escape_multiline
|
1233
|
-
else:
|
1234
|
-
error_on = TOML_ILLEGAL_BASIC_STR_CHARS
|
1235
|
-
parse_escapes = toml_parse_basic_str_escape
|
1236
|
-
result = ''
|
1237
|
-
start_pos = pos
|
1238
|
-
while True:
|
1239
|
-
try:
|
1240
|
-
char = src[pos]
|
1241
|
-
except IndexError:
|
1242
|
-
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
1243
|
-
if char == '"':
|
1244
|
-
if not multiline:
|
1245
|
-
return pos + 1, result + src[start_pos:pos]
|
1246
|
-
if src.startswith('"""', pos):
|
1247
|
-
return pos + 3, result + src[start_pos:pos]
|
1248
|
-
pos += 1
|
1249
|
-
continue
|
1250
|
-
if char == '\\':
|
1251
|
-
result += src[start_pos:pos]
|
1252
|
-
pos, parsed_escape = parse_escapes(src, pos)
|
1253
|
-
result += parsed_escape
|
1254
|
-
start_pos = pos
|
1255
|
-
continue
|
1256
|
-
if char in error_on:
|
1257
|
-
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
1258
|
-
pos += 1
|
917
|
+
# Write third ack
|
918
|
+
os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
|
1259
919
|
|
920
|
+
# Write env info
|
921
|
+
env_info = _get_pyremote_env_info()
|
922
|
+
env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
|
923
|
+
os.write(1, struct.pack('<I', len(env_info_json)))
|
924
|
+
os.write(1, env_info_json.encode('utf-8'))
|
1260
925
|
|
1261
|
-
|
1262
|
-
|
1263
|
-
|
1264
|
-
|
1265
|
-
)
|
1266
|
-
try:
|
1267
|
-
char: ta.Optional[str] = src[pos]
|
1268
|
-
except IndexError:
|
1269
|
-
char = None
|
926
|
+
# Setup IO
|
927
|
+
input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
|
928
|
+
output = os.fdopen(os.dup(1), 'wb', 0) # noqa
|
929
|
+
os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
|
930
|
+
os.close(nfd)
|
1270
931
|
|
1271
|
-
|
932
|
+
if (mn := options.main_name_override) is not None:
|
933
|
+
# Inspections like typing.get_type_hints need an entry in sys.modules.
|
934
|
+
sys.modules[mn] = sys.modules['__main__']
|
1272
935
|
|
1273
|
-
#
|
1274
|
-
|
1275
|
-
if src.startswith('"""', pos):
|
1276
|
-
return toml_parse_multiline_str(src, pos, literal=False)
|
1277
|
-
return toml_parse_one_line_basic_str(src, pos)
|
936
|
+
# Write fourth ack
|
937
|
+
output.write(_PYREMOTE_BOOTSTRAP_ACK3)
|
1278
938
|
|
1279
|
-
#
|
1280
|
-
|
1281
|
-
|
1282
|
-
|
1283
|
-
|
939
|
+
# Return
|
940
|
+
return PyremotePayloadRuntime(
|
941
|
+
input=input,
|
942
|
+
output=output,
|
943
|
+
context_name=context_name,
|
944
|
+
payload_src=payload_src,
|
945
|
+
options=options,
|
946
|
+
env_info=env_info,
|
947
|
+
)
|
1284
948
|
|
1285
|
-
# Booleans
|
1286
|
-
if char == 't':
|
1287
|
-
if src.startswith('true', pos):
|
1288
|
-
return pos + 4, True
|
1289
|
-
if char == 'f':
|
1290
|
-
if src.startswith('false', pos):
|
1291
|
-
return pos + 5, False
|
1292
949
|
|
1293
|
-
|
1294
|
-
if char == '[':
|
1295
|
-
return toml_parse_array(src, pos, parse_float)
|
950
|
+
##
|
1296
951
|
|
1297
|
-
# Inline tables
|
1298
|
-
if char == '{':
|
1299
|
-
return toml_parse_inline_table(src, pos, parse_float)
|
1300
952
|
|
1301
|
-
|
1302
|
-
|
1303
|
-
|
1304
|
-
|
1305
|
-
|
1306
|
-
|
1307
|
-
|
1308
|
-
return datetime_match.end(), datetime_obj
|
1309
|
-
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
1310
|
-
if localtime_match:
|
1311
|
-
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
953
|
+
class PyremoteBootstrapDriver:
|
954
|
+
def __init__(
|
955
|
+
self,
|
956
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
957
|
+
options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
|
958
|
+
) -> None:
|
959
|
+
super().__init__()
|
1312
960
|
|
1313
|
-
|
1314
|
-
|
1315
|
-
number_match = TOML_RE_NUMBER.match(src, pos)
|
1316
|
-
if number_match:
|
1317
|
-
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
961
|
+
self._payload_src = payload_src
|
962
|
+
self._options = options
|
1318
963
|
|
1319
|
-
|
1320
|
-
|
1321
|
-
if first_three in {'inf', 'nan'}:
|
1322
|
-
return pos + 3, parse_float(first_three)
|
1323
|
-
first_four = src[pos:pos + 4]
|
1324
|
-
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
1325
|
-
return pos + 4, parse_float(first_four)
|
964
|
+
self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
|
965
|
+
self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
|
1326
966
|
|
1327
|
-
|
967
|
+
self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
|
968
|
+
#
|
1328
969
|
|
970
|
+
@classmethod
|
971
|
+
def _prepare_payload_src(
|
972
|
+
cls,
|
973
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
974
|
+
options: PyremoteBootstrapOptions,
|
975
|
+
) -> str:
|
976
|
+
parts: ta.List[str]
|
977
|
+
if isinstance(payload_src, str):
|
978
|
+
parts = [payload_src]
|
979
|
+
else:
|
980
|
+
parts = list(payload_src)
|
1329
981
|
|
1330
|
-
|
1331
|
-
|
982
|
+
if (mn := options.main_name_override) is not None:
|
983
|
+
parts.insert(0, f'__name__ = {mn!r}')
|
1332
984
|
|
1333
|
-
|
1334
|
-
|
1335
|
-
return 'end of document'
|
1336
|
-
line = src.count('\n', 0, pos) + 1
|
1337
|
-
if line == 1:
|
1338
|
-
column = pos + 1
|
985
|
+
if len(parts) == 1:
|
986
|
+
return parts[0]
|
1339
987
|
else:
|
1340
|
-
|
1341
|
-
return f'line {line}, column {column}'
|
988
|
+
return '\n\n'.join(parts)
|
1342
989
|
|
1343
|
-
|
990
|
+
#
|
1344
991
|
|
992
|
+
@dc.dataclass(frozen=True)
|
993
|
+
class Read:
|
994
|
+
sz: int
|
1345
995
|
|
1346
|
-
|
1347
|
-
|
996
|
+
@dc.dataclass(frozen=True)
|
997
|
+
class Write:
|
998
|
+
d: bytes
|
1348
999
|
|
1000
|
+
class ProtocolError(Exception):
|
1001
|
+
pass
|
1349
1002
|
|
1350
|
-
|
1351
|
-
|
1003
|
+
@dc.dataclass(frozen=True)
|
1004
|
+
class Result:
|
1005
|
+
pid: int
|
1006
|
+
env_info: PyremoteEnvInfo
|
1352
1007
|
|
1353
|
-
|
1354
|
-
|
1355
|
-
|
1356
|
-
# The default `float` callable never returns illegal types. Optimize it.
|
1357
|
-
if parse_float is float:
|
1358
|
-
return float
|
1008
|
+
def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
|
1009
|
+
# Read first ack (after fork)
|
1010
|
+
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
|
1359
1011
|
|
1360
|
-
|
1361
|
-
|
1362
|
-
|
1363
|
-
raise ValueError('parse_float must not return dicts or lists') # noqa
|
1364
|
-
return float_value
|
1012
|
+
# Read pid
|
1013
|
+
d = yield from self._read(8)
|
1014
|
+
pid = struct.unpack('<Q', d)[0]
|
1365
1015
|
|
1366
|
-
|
1016
|
+
# Write payload src
|
1017
|
+
yield from self._write(struct.pack('<I', len(self._payload_z)))
|
1018
|
+
yield from self._write(self._payload_z)
|
1367
1019
|
|
1020
|
+
# Read second ack (after writing src copies)
|
1021
|
+
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
|
1368
1022
|
|
1369
|
-
|
1370
|
-
|
1023
|
+
# Write options
|
1024
|
+
yield from self._write(struct.pack('<I', len(self._options_json)))
|
1025
|
+
yield from self._write(self._options_json)
|
1371
1026
|
|
1027
|
+
# Read third ack (after reaping child process)
|
1028
|
+
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
|
1372
1029
|
|
1373
|
-
|
1374
|
-
|
1375
|
-
|
1030
|
+
# Read env info
|
1031
|
+
d = yield from self._read(4)
|
1032
|
+
env_info_json_len = struct.unpack('<I', d)[0]
|
1033
|
+
d = yield from self._read(env_info_json_len)
|
1034
|
+
env_info_json = d.decode('utf-8')
|
1035
|
+
env_info = PyremoteEnvInfo(**json.loads(env_info_json))
|
1376
1036
|
|
1377
|
-
|
1037
|
+
# Read fourth ack (after finalization completed)
|
1038
|
+
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
|
1378
1039
|
|
1040
|
+
# Return
|
1041
|
+
return self.Result(
|
1042
|
+
pid=pid,
|
1043
|
+
env_info=env_info,
|
1044
|
+
)
|
1379
1045
|
|
1380
|
-
|
1381
|
-
|
1046
|
+
def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
|
1047
|
+
d = yield self.Read(sz)
|
1048
|
+
if not isinstance(d, bytes):
|
1049
|
+
raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
|
1050
|
+
if len(d) != sz:
|
1051
|
+
raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
|
1052
|
+
return d
|
1382
1053
|
|
1054
|
+
def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
|
1055
|
+
d = yield from self._read(len(e))
|
1056
|
+
if d != e:
|
1057
|
+
raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
|
1383
1058
|
|
1384
|
-
|
1059
|
+
def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
|
1060
|
+
i = yield self.Write(d)
|
1061
|
+
if i is not None:
|
1062
|
+
raise self.ProtocolError('Unexpected input after write')
|
1385
1063
|
|
1064
|
+
#
|
1386
1065
|
|
1387
|
-
|
1388
|
-
|
1389
|
-
pass
|
1066
|
+
def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
|
1067
|
+
gen = self.gen()
|
1390
1068
|
|
1069
|
+
gi: ta.Optional[bytes] = None
|
1070
|
+
while True:
|
1071
|
+
try:
|
1072
|
+
if gi is not None:
|
1073
|
+
go = gen.send(gi)
|
1074
|
+
else:
|
1075
|
+
go = next(gen)
|
1076
|
+
except StopIteration as e:
|
1077
|
+
return e.value
|
1391
1078
|
|
1392
|
-
|
1393
|
-
|
1079
|
+
if isinstance(go, self.Read):
|
1080
|
+
if len(gi := input.read(go.sz)) != go.sz:
|
1081
|
+
raise EOFError
|
1082
|
+
elif isinstance(go, self.Write):
|
1083
|
+
gi = None
|
1084
|
+
output.write(go.d)
|
1085
|
+
output.flush()
|
1086
|
+
else:
|
1087
|
+
raise TypeError(go)
|
1394
1088
|
|
1089
|
+
async def async_run(
|
1090
|
+
self,
|
1091
|
+
input: ta.Any, # asyncio.StreamWriter # noqa
|
1092
|
+
output: ta.Any, # asyncio.StreamReader
|
1093
|
+
) -> Result:
|
1094
|
+
gen = self.gen()
|
1395
1095
|
|
1396
|
-
|
1096
|
+
gi: ta.Optional[bytes] = None
|
1097
|
+
while True:
|
1098
|
+
try:
|
1099
|
+
if gi is not None:
|
1100
|
+
go = gen.send(gi)
|
1101
|
+
else:
|
1102
|
+
go = next(gen)
|
1103
|
+
except StopIteration as e:
|
1104
|
+
return e.value
|
1105
|
+
|
1106
|
+
if isinstance(go, self.Read):
|
1107
|
+
if len(gi := await input.read(go.sz)) != go.sz:
|
1108
|
+
raise EOFError
|
1109
|
+
elif isinstance(go, self.Write):
|
1110
|
+
gi = None
|
1111
|
+
output.write(go.d)
|
1112
|
+
await output.drain()
|
1113
|
+
else:
|
1114
|
+
raise TypeError(go)
|
1397
1115
|
|
1398
1116
|
|
1399
1117
|
########################################
|
1400
|
-
#
|
1118
|
+
# ../../../omlish/asyncs/asyncio/channels.py
|
1401
1119
|
|
1402
1120
|
|
1403
|
-
|
1121
|
+
class AsyncioBytesChannelTransport(asyncio.Transport):
|
1122
|
+
def __init__(self, reader: asyncio.StreamReader) -> None:
|
1123
|
+
super().__init__()
|
1404
1124
|
|
1125
|
+
self.reader = reader
|
1126
|
+
self.closed: asyncio.Future = asyncio.Future()
|
1405
1127
|
|
1406
|
-
|
1128
|
+
# @ta.override
|
1129
|
+
def write(self, data: bytes) -> None:
|
1130
|
+
self.reader.feed_data(data)
|
1407
1131
|
|
1408
|
-
|
1132
|
+
# @ta.override
|
1133
|
+
def close(self) -> None:
|
1134
|
+
self.reader.feed_eof()
|
1135
|
+
if not self.closed.done():
|
1136
|
+
self.closed.set_result(True)
|
1409
1137
|
|
1138
|
+
# @ta.override
|
1139
|
+
def is_closing(self) -> bool:
|
1140
|
+
return self.closed.done()
|
1410
1141
|
|
1411
|
-
########################################
|
1412
|
-
# ../../pyremote.py
|
1413
|
-
"""
|
1414
|
-
Basically this: https://mitogen.networkgenomics.com/howitworks.html
|
1415
1142
|
|
1416
|
-
|
1417
|
-
|
1418
|
-
|
1143
|
+
def asyncio_create_bytes_channel(
|
1144
|
+
loop: ta.Any = None,
|
1145
|
+
) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
|
1146
|
+
if loop is None:
|
1147
|
+
loop = asyncio.get_running_loop()
|
1419
1148
|
|
1149
|
+
reader = asyncio.StreamReader()
|
1150
|
+
protocol = asyncio.StreamReaderProtocol(reader)
|
1151
|
+
transport = AsyncioBytesChannelTransport(reader)
|
1152
|
+
writer = asyncio.StreamWriter(transport, protocol, reader, loop)
|
1420
1153
|
|
1421
|
-
|
1154
|
+
return reader, writer
|
1422
1155
|
|
1423
1156
|
|
1424
|
-
|
1425
|
-
|
1426
|
-
debug: bool = False
|
1157
|
+
########################################
|
1158
|
+
# ../../../omlish/asyncs/asyncio/streams.py
|
1427
1159
|
|
1428
|
-
DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
|
1429
|
-
main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
|
1430
1160
|
|
1161
|
+
ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
|
1431
1162
|
|
1432
|
-
##
|
1433
1163
|
|
1164
|
+
async def asyncio_open_stream_reader(
|
1165
|
+
f: ta.IO,
|
1166
|
+
loop: ta.Any = None,
|
1167
|
+
*,
|
1168
|
+
limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
|
1169
|
+
) -> asyncio.StreamReader:
|
1170
|
+
if loop is None:
|
1171
|
+
loop = asyncio.get_running_loop()
|
1434
1172
|
|
1435
|
-
|
1436
|
-
|
1437
|
-
|
1438
|
-
|
1439
|
-
|
1440
|
-
sys_exec_prefix: str
|
1441
|
-
sys_executable: str
|
1442
|
-
sys_implementation_name: str
|
1443
|
-
sys_path: ta.List[str]
|
1444
|
-
sys_platform: str
|
1445
|
-
sys_prefix: str
|
1446
|
-
sys_version: str
|
1447
|
-
sys_version_info: ta.List[ta.Union[int, str]]
|
1173
|
+
reader = asyncio.StreamReader(limit=limit, loop=loop)
|
1174
|
+
await loop.connect_read_pipe(
|
1175
|
+
lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
|
1176
|
+
f,
|
1177
|
+
)
|
1448
1178
|
|
1449
|
-
|
1450
|
-
platform_machine: str
|
1451
|
-
platform_platform: str
|
1452
|
-
platform_processor: str
|
1453
|
-
platform_system: str
|
1454
|
-
platform_release: str
|
1455
|
-
platform_version: str
|
1179
|
+
return reader
|
1456
1180
|
|
1457
|
-
site_userbase: str
|
1458
1181
|
|
1459
|
-
|
1460
|
-
|
1461
|
-
|
1462
|
-
|
1463
|
-
|
1464
|
-
|
1465
|
-
os_ppid: int
|
1466
|
-
os_uid: int
|
1182
|
+
async def asyncio_open_stream_writer(
|
1183
|
+
f: ta.IO,
|
1184
|
+
loop: ta.Any = None,
|
1185
|
+
) -> asyncio.StreamWriter:
|
1186
|
+
if loop is None:
|
1187
|
+
loop = asyncio.get_running_loop()
|
1467
1188
|
|
1468
|
-
|
1469
|
-
|
1470
|
-
|
1471
|
-
|
1472
|
-
pw_dir: str
|
1473
|
-
pw_shell: str
|
1189
|
+
writer_transport, writer_protocol = await loop.connect_write_pipe(
|
1190
|
+
lambda: asyncio.streams.FlowControlMixin(loop=loop),
|
1191
|
+
f,
|
1192
|
+
)
|
1474
1193
|
|
1475
|
-
|
1194
|
+
return asyncio.streams.StreamWriter(
|
1195
|
+
writer_transport,
|
1196
|
+
writer_protocol,
|
1197
|
+
None,
|
1198
|
+
loop,
|
1199
|
+
)
|
1476
1200
|
|
1477
1201
|
|
1478
|
-
|
1479
|
-
|
1202
|
+
########################################
|
1203
|
+
# ../../../omlish/asyncs/asyncio/timeouts.py
|
1480
1204
|
|
1481
|
-
pw = pwd.getpwuid(os_uid)
|
1482
1205
|
|
1483
|
-
|
1484
|
-
|
1485
|
-
|
1486
|
-
|
1487
|
-
|
1206
|
+
def asyncio_maybe_timeout(
|
1207
|
+
fut: AwaitableT,
|
1208
|
+
timeout: ta.Optional[float] = None,
|
1209
|
+
) -> AwaitableT:
|
1210
|
+
if timeout is not None:
|
1211
|
+
fut = asyncio.wait_for(fut, timeout) # type: ignore
|
1212
|
+
return fut
|
1488
1213
|
|
1489
|
-
return PyremoteEnvInfo(
|
1490
|
-
sys_base_prefix=sys.base_prefix,
|
1491
|
-
sys_byteorder=sys.byteorder,
|
1492
|
-
sys_defaultencoding=sys.getdefaultencoding(),
|
1493
|
-
sys_exec_prefix=sys.exec_prefix,
|
1494
|
-
sys_executable=sys.executable,
|
1495
|
-
sys_implementation_name=sys.implementation.name,
|
1496
|
-
sys_path=sys.path,
|
1497
|
-
sys_platform=sys.platform,
|
1498
|
-
sys_prefix=sys.prefix,
|
1499
|
-
sys_version=sys.version,
|
1500
|
-
sys_version_info=list(sys.version_info),
|
1501
1214
|
|
1502
|
-
|
1503
|
-
|
1504
|
-
platform_platform=platform.platform(),
|
1505
|
-
platform_processor=platform.processor(),
|
1506
|
-
platform_system=platform.system(),
|
1507
|
-
platform_release=platform.release(),
|
1508
|
-
platform_version=platform.version(),
|
1215
|
+
########################################
|
1216
|
+
# ../../../omlish/formats/ini/sections.py
|
1509
1217
|
|
1510
|
-
site_userbase=site.getuserbase(),
|
1511
1218
|
|
1512
|
-
|
1513
|
-
os_gid=os.getgid(),
|
1514
|
-
os_loadavg=list(os.getloadavg()),
|
1515
|
-
os_login=os_login,
|
1516
|
-
os_pgrp=os.getpgrp(),
|
1517
|
-
os_pid=os.getpid(),
|
1518
|
-
os_ppid=os.getppid(),
|
1519
|
-
os_uid=os_uid,
|
1219
|
+
##
|
1520
1220
|
|
1521
|
-
pw_name=pw.pw_name,
|
1522
|
-
pw_uid=pw.pw_uid,
|
1523
|
-
pw_gid=pw.pw_gid,
|
1524
|
-
pw_gecos=pw.pw_gecos,
|
1525
|
-
pw_dir=pw.pw_dir,
|
1526
|
-
pw_shell=pw.pw_shell,
|
1527
1221
|
|
1528
|
-
|
1529
|
-
|
1222
|
+
def extract_ini_sections(cp: configparser.ConfigParser) -> IniSectionSettingsMap:
|
1223
|
+
config_dct: ta.Dict[str, ta.Any] = {}
|
1224
|
+
for sec in cp.sections():
|
1225
|
+
cd = config_dct
|
1226
|
+
for k in sec.split('.'):
|
1227
|
+
cd = cd.setdefault(k, {})
|
1228
|
+
cd.update(cp.items(sec))
|
1229
|
+
return config_dct
|
1530
1230
|
|
1531
1231
|
|
1532
1232
|
##
|
1533
1233
|
|
1534
1234
|
|
1535
|
-
|
1536
|
-
|
1537
|
-
|
1538
|
-
|
1539
|
-
_PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
|
1540
|
-
_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
|
1541
|
-
_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
|
1542
|
-
_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
|
1543
|
-
|
1544
|
-
_PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
|
1545
|
-
_PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
|
1546
|
-
_PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
|
1547
|
-
_PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
|
1548
|
-
|
1549
|
-
_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
|
1235
|
+
def render_ini_sections(
|
1236
|
+
settings_by_section: IniSectionSettingsMap,
|
1237
|
+
) -> str:
|
1238
|
+
out = io.StringIO()
|
1550
1239
|
|
1551
|
-
|
1552
|
-
|
1553
|
-
|
1554
|
-
'struct',
|
1555
|
-
'sys',
|
1556
|
-
'zlib',
|
1557
|
-
]
|
1240
|
+
for i, (section, settings) in enumerate(settings_by_section.items()):
|
1241
|
+
if i:
|
1242
|
+
out.write('\n')
|
1558
1243
|
|
1244
|
+
out.write(f'[{section}]\n')
|
1559
1245
|
|
1560
|
-
|
1561
|
-
|
1562
|
-
|
1246
|
+
for k, v in settings.items():
|
1247
|
+
if isinstance(v, str):
|
1248
|
+
out.write(f'{k}={v}\n')
|
1249
|
+
else:
|
1250
|
+
for vv in v:
|
1251
|
+
out.write(f'{k}={vv}\n')
|
1563
1252
|
|
1564
|
-
|
1565
|
-
r0, w0 = os.pipe()
|
1566
|
-
r1, w1 = os.pipe()
|
1253
|
+
return out.getvalue()
|
1567
1254
|
|
1568
|
-
if (cp := os.fork()):
|
1569
|
-
# Parent process
|
1570
1255
|
|
1571
|
-
|
1572
|
-
|
1256
|
+
########################################
|
1257
|
+
# ../../../omlish/formats/toml/parser.py
|
1258
|
+
# SPDX-License-Identifier: MIT
|
1259
|
+
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
1260
|
+
# Licensed to PSF under a Contributor Agreement.
|
1261
|
+
#
|
1262
|
+
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
1263
|
+
# --------------------------------------------
|
1264
|
+
#
|
1265
|
+
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
1266
|
+
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
1267
|
+
# documentation.
|
1268
|
+
#
|
1269
|
+
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
1270
|
+
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
1271
|
+
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
1272
|
+
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
1273
|
+
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
1274
|
+
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
1275
|
+
#
|
1276
|
+
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
1277
|
+
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
1278
|
+
# any such work a brief summary of the changes made to Python.
|
1279
|
+
#
|
1280
|
+
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
1281
|
+
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
1282
|
+
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
1283
|
+
# RIGHTS.
|
1284
|
+
#
|
1285
|
+
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
1286
|
+
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
1287
|
+
# ADVISED OF THE POSSIBILITY THEREOF.
|
1288
|
+
#
|
1289
|
+
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
1290
|
+
#
|
1291
|
+
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
1292
|
+
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
1293
|
+
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
1294
|
+
#
|
1295
|
+
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
1296
|
+
# License Agreement.
|
1297
|
+
#
|
1298
|
+
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
1573
1299
|
|
1574
|
-
# Overwrite stdin (fed to python repl) with first copy of src
|
1575
|
-
os.dup2(r0, 0)
|
1576
1300
|
|
1577
|
-
|
1578
|
-
os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
|
1301
|
+
##
|
1579
1302
|
|
1580
|
-
# Close remaining fd's
|
1581
|
-
for f in [r0, w0, r1, w1]:
|
1582
|
-
os.close(f)
|
1583
1303
|
|
1584
|
-
|
1585
|
-
env = os.environ
|
1586
|
-
exe = sys.executable
|
1587
|
-
env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
|
1588
|
-
env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
|
1589
|
-
env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
|
1304
|
+
_TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
|
1590
1305
|
|
1591
|
-
|
1592
|
-
|
1306
|
+
TOML_RE_NUMBER = re.compile(
|
1307
|
+
r"""
|
1308
|
+
0
|
1309
|
+
(?:
|
1310
|
+
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
1311
|
+
|
|
1312
|
+
b[01](?:_?[01])* # bin
|
1313
|
+
|
|
1314
|
+
o[0-7](?:_?[0-7])* # oct
|
1315
|
+
)
|
1316
|
+
|
|
1317
|
+
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
1318
|
+
(?P<floatpart>
|
1319
|
+
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
1320
|
+
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
1321
|
+
)
|
1322
|
+
""",
|
1323
|
+
flags=re.VERBOSE,
|
1324
|
+
)
|
1325
|
+
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
1326
|
+
TOML_RE_DATETIME = re.compile(
|
1327
|
+
rf"""
|
1328
|
+
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
1329
|
+
(?:
|
1330
|
+
[Tt ]
|
1331
|
+
{_TOML_TIME_RE_STR}
|
1332
|
+
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
1333
|
+
)?
|
1334
|
+
""",
|
1335
|
+
flags=re.VERBOSE,
|
1336
|
+
)
|
1593
1337
|
|
1594
|
-
else:
|
1595
|
-
# Child process
|
1596
1338
|
|
1597
|
-
|
1598
|
-
|
1339
|
+
def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
|
1340
|
+
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
1599
1341
|
|
1600
|
-
|
1601
|
-
|
1342
|
+
Raises ValueError if the match does not correspond to a valid date or datetime.
|
1343
|
+
"""
|
1344
|
+
(
|
1345
|
+
year_str,
|
1346
|
+
month_str,
|
1347
|
+
day_str,
|
1348
|
+
hour_str,
|
1349
|
+
minute_str,
|
1350
|
+
sec_str,
|
1351
|
+
micros_str,
|
1352
|
+
zulu_time,
|
1353
|
+
offset_sign_str,
|
1354
|
+
offset_hour_str,
|
1355
|
+
offset_minute_str,
|
1356
|
+
) = match.groups()
|
1357
|
+
year, month, day = int(year_str), int(month_str), int(day_str)
|
1358
|
+
if hour_str is None:
|
1359
|
+
return datetime.date(year, month, day)
|
1360
|
+
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
1361
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
1362
|
+
if offset_sign_str:
|
1363
|
+
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
1364
|
+
offset_hour_str, offset_minute_str, offset_sign_str,
|
1365
|
+
)
|
1366
|
+
elif zulu_time:
|
1367
|
+
tz = datetime.UTC
|
1368
|
+
else: # local date-time
|
1369
|
+
tz = None
|
1370
|
+
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
1602
1371
|
|
1603
|
-
# Read payload src from stdin
|
1604
|
-
payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
|
1605
|
-
if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
|
1606
|
-
raise EOFError
|
1607
|
-
payload_src = zlib.decompress(payload_z)
|
1608
1372
|
|
1609
|
-
|
1610
|
-
|
1611
|
-
|
1612
|
-
|
1613
|
-
|
1614
|
-
|
1373
|
+
@functools.lru_cache() # noqa
|
1374
|
+
def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
|
1375
|
+
sign = 1 if sign_str == '+' else -1
|
1376
|
+
return datetime.timezone(
|
1377
|
+
datetime.timedelta(
|
1378
|
+
hours=sign * int(hour_str),
|
1379
|
+
minutes=sign * int(minute_str),
|
1380
|
+
),
|
1381
|
+
)
|
1615
1382
|
|
1616
|
-
# Write second ack
|
1617
|
-
os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
|
1618
1383
|
|
1619
|
-
|
1620
|
-
|
1384
|
+
def toml_match_to_localtime(match: re.Match) -> datetime.time:
|
1385
|
+
hour_str, minute_str, sec_str, micros_str = match.groups()
|
1386
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
1387
|
+
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
1621
1388
|
|
1622
1389
|
|
1623
|
-
|
1390
|
+
def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
|
1391
|
+
if match.group('floatpart'):
|
1392
|
+
return parse_float(match.group())
|
1393
|
+
return int(match.group(), 0)
|
1624
1394
|
|
1625
1395
|
|
1626
|
-
|
1627
|
-
if any(c in context_name for c in '\'"'):
|
1628
|
-
raise NameError(context_name)
|
1396
|
+
TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
|
1629
1397
|
|
1630
|
-
|
1631
|
-
|
1632
|
-
|
1398
|
+
# Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
|
1399
|
+
# functions.
|
1400
|
+
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
1401
|
+
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
1633
1402
|
|
1634
|
-
|
1635
|
-
|
1636
|
-
'_PYREMOTE_BOOTSTRAP_SRC_FD',
|
1403
|
+
TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
1404
|
+
TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1637
1405
|
|
1638
|
-
|
1639
|
-
'_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
|
1640
|
-
'_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
|
1406
|
+
TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
1641
1407
|
|
1642
|
-
|
1643
|
-
|
1408
|
+
TOML_WS = frozenset(' \t')
|
1409
|
+
TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
|
1410
|
+
TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
|
1411
|
+
TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
|
1412
|
+
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
1644
1413
|
|
1645
|
-
|
1646
|
-
|
1647
|
-
|
1414
|
+
TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
|
1415
|
+
{
|
1416
|
+
'\\b': '\u0008', # backspace
|
1417
|
+
'\\t': '\u0009', # tab
|
1418
|
+
'\\n': '\u000A', # linefeed
|
1419
|
+
'\\f': '\u000C', # form feed
|
1420
|
+
'\\r': '\u000D', # carriage return
|
1421
|
+
'\\"': '\u0022', # quote
|
1422
|
+
'\\\\': '\u005C', # backslash
|
1423
|
+
},
|
1424
|
+
)
|
1648
1425
|
|
1649
|
-
bs_src = '\n'.join(
|
1650
|
-
cl
|
1651
|
-
for l in bs_src.splitlines()
|
1652
|
-
if (cl := (l.split('#')[0]).rstrip())
|
1653
|
-
if cl.strip()
|
1654
|
-
)
|
1655
1426
|
|
1656
|
-
|
1657
|
-
|
1658
|
-
if b'"' in bs_z85:
|
1659
|
-
raise ValueError(bs_z85)
|
1427
|
+
class TomlDecodeError(ValueError):
|
1428
|
+
"""An error raised if a document is not valid TOML."""
|
1660
1429
|
|
1661
|
-
stmts = [
|
1662
|
-
f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
|
1663
|
-
f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
|
1664
|
-
f'_pyremote_bootstrap_main("{context_name}")',
|
1665
|
-
]
|
1666
1430
|
|
1667
|
-
|
1668
|
-
|
1431
|
+
def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
|
1432
|
+
"""Parse TOML from a binary file object."""
|
1433
|
+
b = fp.read()
|
1434
|
+
try:
|
1435
|
+
s = b.decode()
|
1436
|
+
except AttributeError:
|
1437
|
+
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
1438
|
+
return toml_loads(s, parse_float=parse_float)
|
1669
1439
|
|
1670
1440
|
|
1671
|
-
|
1441
|
+
def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
|
1442
|
+
"""Parse TOML from a string."""
|
1672
1443
|
|
1444
|
+
# The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
|
1445
|
+
try:
|
1446
|
+
src = s.replace('\r\n', '\n')
|
1447
|
+
except (AttributeError, TypeError):
|
1448
|
+
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
1449
|
+
pos = 0
|
1450
|
+
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
1451
|
+
header: TomlKey = ()
|
1452
|
+
parse_float = toml_make_safe_parse_float(parse_float)
|
1673
1453
|
|
1674
|
-
|
1675
|
-
|
1676
|
-
|
1677
|
-
|
1678
|
-
context_name: str
|
1679
|
-
payload_src: str
|
1680
|
-
options: PyremoteBootstrapOptions
|
1681
|
-
env_info: PyremoteEnvInfo
|
1454
|
+
# Parse one statement at a time (typically means one line in TOML source)
|
1455
|
+
while True:
|
1456
|
+
# 1. Skip line leading whitespace
|
1457
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1682
1458
|
|
1459
|
+
# 2. Parse rules. Expect one of the following:
|
1460
|
+
# - end of file
|
1461
|
+
# - end of line
|
1462
|
+
# - comment
|
1463
|
+
# - key/value pair
|
1464
|
+
# - append dict to list (and move to its namespace)
|
1465
|
+
# - create dict (and move to its namespace)
|
1466
|
+
# Skip trailing whitespace when applicable.
|
1467
|
+
try:
|
1468
|
+
char = src[pos]
|
1469
|
+
except IndexError:
|
1470
|
+
break
|
1471
|
+
if char == '\n':
|
1472
|
+
pos += 1
|
1473
|
+
continue
|
1474
|
+
if char in TOML_KEY_INITIAL_CHARS:
|
1475
|
+
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
1476
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1477
|
+
elif char == '[':
|
1478
|
+
try:
|
1479
|
+
second_char: ta.Optional[str] = src[pos + 1]
|
1480
|
+
except IndexError:
|
1481
|
+
second_char = None
|
1482
|
+
out.flags.finalize_pending()
|
1483
|
+
if second_char == '[':
|
1484
|
+
pos, header = toml_create_list_rule(src, pos, out)
|
1485
|
+
else:
|
1486
|
+
pos, header = toml_create_dict_rule(src, pos, out)
|
1487
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1488
|
+
elif char != '#':
|
1489
|
+
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
1683
1490
|
|
1684
|
-
|
1685
|
-
|
1686
|
-
if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
|
1687
|
-
# Read second copy of payload src
|
1688
|
-
r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
|
1689
|
-
payload_src = r1.read().decode('utf-8')
|
1690
|
-
r1.close()
|
1491
|
+
# 3. Skip comment
|
1492
|
+
pos = toml_skip_comment(src, pos)
|
1691
1493
|
|
1692
|
-
#
|
1693
|
-
|
1694
|
-
|
1494
|
+
# 4. Expect end of line or end of file
|
1495
|
+
try:
|
1496
|
+
char = src[pos]
|
1497
|
+
except IndexError:
|
1498
|
+
break
|
1499
|
+
if char != '\n':
|
1500
|
+
raise toml_suffixed_err(
|
1501
|
+
src, pos, 'Expected newline or end of document after a statement',
|
1502
|
+
)
|
1503
|
+
pos += 1
|
1695
1504
|
|
1696
|
-
|
1697
|
-
options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
|
1698
|
-
if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
|
1699
|
-
raise EOFError
|
1700
|
-
options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
|
1505
|
+
return out.data.dict
|
1701
1506
|
|
1702
|
-
# If debugging, re-exec as file
|
1703
|
-
if options.debug:
|
1704
|
-
# Write temp source file
|
1705
|
-
import tempfile
|
1706
|
-
tfd, tfn = tempfile.mkstemp('-pyremote.py')
|
1707
|
-
os.write(tfd, payload_src.encode('utf-8'))
|
1708
|
-
os.close(tfd)
|
1709
1507
|
|
1710
|
-
|
1711
|
-
|
1712
|
-
os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
|
1508
|
+
class TomlFlags:
|
1509
|
+
"""Flags that map to parsed keys/namespaces."""
|
1713
1510
|
|
1714
|
-
|
1715
|
-
|
1716
|
-
|
1717
|
-
|
1511
|
+
# Marks an immutable namespace (inline array or inline table).
|
1512
|
+
FROZEN = 0
|
1513
|
+
# Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
|
1514
|
+
EXPLICIT_NEST = 1
|
1718
1515
|
|
1719
|
-
|
1720
|
-
|
1721
|
-
|
1722
|
-
options = PyremoteBootstrapOptions(**json.loads(options_json_str))
|
1516
|
+
def __init__(self) -> None:
|
1517
|
+
self._flags: ta.Dict[str, dict] = {}
|
1518
|
+
self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
|
1723
1519
|
|
1724
|
-
|
1725
|
-
|
1726
|
-
payload_src = sf.read()
|
1520
|
+
def add_pending(self, key: TomlKey, flag: int) -> None:
|
1521
|
+
self._pending_flags.add((key, flag))
|
1727
1522
|
|
1728
|
-
|
1729
|
-
|
1730
|
-
|
1523
|
+
def finalize_pending(self) -> None:
|
1524
|
+
for key, flag in self._pending_flags:
|
1525
|
+
self.set(key, flag, recursive=False)
|
1526
|
+
self._pending_flags.clear()
|
1731
1527
|
|
1732
|
-
|
1733
|
-
|
1528
|
+
def unset_all(self, key: TomlKey) -> None:
|
1529
|
+
cont = self._flags
|
1530
|
+
for k in key[:-1]:
|
1531
|
+
if k not in cont:
|
1532
|
+
return
|
1533
|
+
cont = cont[k]['nested']
|
1534
|
+
cont.pop(key[-1], None)
|
1734
1535
|
|
1735
|
-
#
|
1736
|
-
|
1737
|
-
|
1738
|
-
|
1739
|
-
|
1536
|
+
def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
|
1537
|
+
cont = self._flags
|
1538
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1539
|
+
for k in key_parent:
|
1540
|
+
if k not in cont:
|
1541
|
+
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
1542
|
+
cont = cont[k]['nested']
|
1543
|
+
if key_stem not in cont:
|
1544
|
+
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
1545
|
+
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
1740
1546
|
|
1741
|
-
|
1742
|
-
|
1743
|
-
|
1744
|
-
|
1745
|
-
|
1547
|
+
def is_(self, key: TomlKey, flag: int) -> bool:
|
1548
|
+
if not key:
|
1549
|
+
return False # document root has no flags
|
1550
|
+
cont = self._flags
|
1551
|
+
for k in key[:-1]:
|
1552
|
+
if k not in cont:
|
1553
|
+
return False
|
1554
|
+
inner_cont = cont[k]
|
1555
|
+
if flag in inner_cont['recursive_flags']:
|
1556
|
+
return True
|
1557
|
+
cont = inner_cont['nested']
|
1558
|
+
key_stem = key[-1]
|
1559
|
+
if key_stem in cont:
|
1560
|
+
cont = cont[key_stem]
|
1561
|
+
return flag in cont['flags'] or flag in cont['recursive_flags']
|
1562
|
+
return False
|
1746
1563
|
|
1747
|
-
if (mn := options.main_name_override) is not None:
|
1748
|
-
# Inspections like typing.get_type_hints need an entry in sys.modules.
|
1749
|
-
sys.modules[mn] = sys.modules['__main__']
|
1750
1564
|
|
1751
|
-
|
1752
|
-
|
1565
|
+
class TomlNestedDict:
|
1566
|
+
def __init__(self) -> None:
|
1567
|
+
# The parsed content of the TOML document
|
1568
|
+
self.dict: ta.Dict[str, ta.Any] = {}
|
1753
1569
|
|
1754
|
-
|
1755
|
-
|
1756
|
-
|
1757
|
-
|
1758
|
-
|
1759
|
-
|
1760
|
-
|
1761
|
-
|
1762
|
-
|
1570
|
+
def get_or_create_nest(
|
1571
|
+
self,
|
1572
|
+
key: TomlKey,
|
1573
|
+
*,
|
1574
|
+
access_lists: bool = True,
|
1575
|
+
) -> dict:
|
1576
|
+
cont: ta.Any = self.dict
|
1577
|
+
for k in key:
|
1578
|
+
if k not in cont:
|
1579
|
+
cont[k] = {}
|
1580
|
+
cont = cont[k]
|
1581
|
+
if access_lists and isinstance(cont, list):
|
1582
|
+
cont = cont[-1]
|
1583
|
+
if not isinstance(cont, dict):
|
1584
|
+
raise KeyError('There is no nest behind this key')
|
1585
|
+
return cont
|
1763
1586
|
|
1587
|
+
def append_nest_to_list(self, key: TomlKey) -> None:
|
1588
|
+
cont = self.get_or_create_nest(key[:-1])
|
1589
|
+
last_key = key[-1]
|
1590
|
+
if last_key in cont:
|
1591
|
+
list_ = cont[last_key]
|
1592
|
+
if not isinstance(list_, list):
|
1593
|
+
raise KeyError('An object other than list found behind this key')
|
1594
|
+
list_.append({})
|
1595
|
+
else:
|
1596
|
+
cont[last_key] = [{}]
|
1764
1597
|
|
1765
|
-
##
|
1766
1598
|
|
1599
|
+
class TomlOutput(ta.NamedTuple):
|
1600
|
+
data: TomlNestedDict
|
1601
|
+
flags: TomlFlags
|
1767
1602
|
|
1768
|
-
class PyremoteBootstrapDriver:
|
1769
|
-
def __init__(
|
1770
|
-
self,
|
1771
|
-
payload_src: ta.Union[str, ta.Sequence[str]],
|
1772
|
-
options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
|
1773
|
-
) -> None:
|
1774
|
-
super().__init__()
|
1775
1603
|
|
1776
|
-
|
1777
|
-
|
1604
|
+
def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
|
1605
|
+
try:
|
1606
|
+
while src[pos] in chars:
|
1607
|
+
pos += 1
|
1608
|
+
except IndexError:
|
1609
|
+
pass
|
1610
|
+
return pos
|
1778
1611
|
|
1779
|
-
self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
|
1780
|
-
self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
|
1781
1612
|
|
1782
|
-
|
1783
|
-
|
1613
|
+
def toml_skip_until(
|
1614
|
+
src: str,
|
1615
|
+
pos: TomlPos,
|
1616
|
+
expect: str,
|
1617
|
+
*,
|
1618
|
+
error_on: ta.FrozenSet[str],
|
1619
|
+
error_on_eof: bool,
|
1620
|
+
) -> TomlPos:
|
1621
|
+
try:
|
1622
|
+
new_pos = src.index(expect, pos)
|
1623
|
+
except ValueError:
|
1624
|
+
new_pos = len(src)
|
1625
|
+
if error_on_eof:
|
1626
|
+
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
1784
1627
|
|
1785
|
-
|
1786
|
-
|
1787
|
-
|
1788
|
-
|
1789
|
-
|
1790
|
-
) -> str:
|
1791
|
-
parts: ta.List[str]
|
1792
|
-
if isinstance(payload_src, str):
|
1793
|
-
parts = [payload_src]
|
1794
|
-
else:
|
1795
|
-
parts = list(payload_src)
|
1628
|
+
if not error_on.isdisjoint(src[pos:new_pos]):
|
1629
|
+
while src[pos] not in error_on:
|
1630
|
+
pos += 1
|
1631
|
+
raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
|
1632
|
+
return new_pos
|
1796
1633
|
|
1797
|
-
if (mn := options.main_name_override) is not None:
|
1798
|
-
parts.insert(0, f'__name__ = {mn!r}')
|
1799
1634
|
|
1800
|
-
|
1801
|
-
|
1802
|
-
|
1803
|
-
|
1635
|
+
def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
|
1636
|
+
try:
|
1637
|
+
char: ta.Optional[str] = src[pos]
|
1638
|
+
except IndexError:
|
1639
|
+
char = None
|
1640
|
+
if char == '#':
|
1641
|
+
return toml_skip_until(
|
1642
|
+
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
1643
|
+
)
|
1644
|
+
return pos
|
1804
1645
|
|
1805
|
-
#
|
1806
1646
|
|
1807
|
-
|
1808
|
-
|
1809
|
-
|
1647
|
+
def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
|
1648
|
+
while True:
|
1649
|
+
pos_before_skip = pos
|
1650
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1651
|
+
pos = toml_skip_comment(src, pos)
|
1652
|
+
if pos == pos_before_skip:
|
1653
|
+
return pos
|
1810
1654
|
|
1811
|
-
@dc.dataclass(frozen=True)
|
1812
|
-
class Write:
|
1813
|
-
d: bytes
|
1814
1655
|
|
1815
|
-
|
1816
|
-
|
1656
|
+
def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
1657
|
+
pos += 1 # Skip "["
|
1658
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1659
|
+
pos, key = toml_parse_key(src, pos)
|
1817
1660
|
|
1818
|
-
|
1819
|
-
|
1820
|
-
|
1821
|
-
|
1661
|
+
if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
|
1662
|
+
raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
|
1663
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
1664
|
+
try:
|
1665
|
+
out.data.get_or_create_nest(key)
|
1666
|
+
except KeyError:
|
1667
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1822
1668
|
|
1823
|
-
|
1824
|
-
|
1825
|
-
|
1669
|
+
if not src.startswith(']', pos):
|
1670
|
+
raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
|
1671
|
+
return pos + 1, key
|
1826
1672
|
|
1827
|
-
# Read pid
|
1828
|
-
d = yield from self._read(8)
|
1829
|
-
pid = struct.unpack('<Q', d)[0]
|
1830
1673
|
|
1831
|
-
|
1832
|
-
|
1833
|
-
|
1674
|
+
def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
1675
|
+
pos += 2 # Skip "[["
|
1676
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1677
|
+
pos, key = toml_parse_key(src, pos)
|
1834
1678
|
|
1835
|
-
|
1836
|
-
|
1679
|
+
if out.flags.is_(key, TomlFlags.FROZEN):
|
1680
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1681
|
+
# Free the namespace now that it points to another empty list item...
|
1682
|
+
out.flags.unset_all(key)
|
1683
|
+
# ...but this key precisely is still prohibited from table declaration
|
1684
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
1685
|
+
try:
|
1686
|
+
out.data.append_nest_to_list(key)
|
1687
|
+
except KeyError:
|
1688
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1837
1689
|
|
1838
|
-
|
1839
|
-
|
1840
|
-
|
1690
|
+
if not src.startswith(']]', pos):
|
1691
|
+
raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
|
1692
|
+
return pos + 2, key
|
1841
1693
|
|
1842
|
-
# Read third ack (after reaping child process)
|
1843
|
-
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
|
1844
1694
|
|
1845
|
-
|
1846
|
-
|
1847
|
-
|
1848
|
-
|
1849
|
-
|
1850
|
-
|
1695
|
+
def toml_key_value_rule(
|
1696
|
+
src: str,
|
1697
|
+
pos: TomlPos,
|
1698
|
+
out: TomlOutput,
|
1699
|
+
header: TomlKey,
|
1700
|
+
parse_float: TomlParseFloat,
|
1701
|
+
) -> TomlPos:
|
1702
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1703
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1704
|
+
abs_key_parent = header + key_parent
|
1851
1705
|
|
1852
|
-
|
1853
|
-
|
1706
|
+
relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
|
1707
|
+
for cont_key in relative_path_cont_keys:
|
1708
|
+
# Check that dotted key syntax does not redefine an existing table
|
1709
|
+
if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
|
1710
|
+
raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
|
1711
|
+
# Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
|
1712
|
+
# table sections.
|
1713
|
+
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
1854
1714
|
|
1855
|
-
|
1856
|
-
|
1857
|
-
|
1858
|
-
|
1715
|
+
if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
|
1716
|
+
raise toml_suffixed_err(
|
1717
|
+
src,
|
1718
|
+
pos,
|
1719
|
+
f'Cannot mutate immutable namespace {abs_key_parent}',
|
1859
1720
|
)
|
1860
1721
|
|
1861
|
-
|
1862
|
-
|
1863
|
-
|
1864
|
-
|
1865
|
-
|
1866
|
-
|
1867
|
-
|
1722
|
+
try:
|
1723
|
+
nest = out.data.get_or_create_nest(abs_key_parent)
|
1724
|
+
except KeyError:
|
1725
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1726
|
+
if key_stem in nest:
|
1727
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
1728
|
+
# Mark inline table and array namespaces recursively immutable
|
1729
|
+
if isinstance(value, (dict, list)):
|
1730
|
+
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
1731
|
+
nest[key_stem] = value
|
1732
|
+
return pos
|
1868
1733
|
|
1869
|
-
def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
|
1870
|
-
d = yield from self._read(len(e))
|
1871
|
-
if d != e:
|
1872
|
-
raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
|
1873
1734
|
|
1874
|
-
|
1875
|
-
|
1876
|
-
|
1877
|
-
|
1735
|
+
def toml_parse_key_value_pair(
|
1736
|
+
src: str,
|
1737
|
+
pos: TomlPos,
|
1738
|
+
parse_float: TomlParseFloat,
|
1739
|
+
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
1740
|
+
pos, key = toml_parse_key(src, pos)
|
1741
|
+
try:
|
1742
|
+
char: ta.Optional[str] = src[pos]
|
1743
|
+
except IndexError:
|
1744
|
+
char = None
|
1745
|
+
if char != '=':
|
1746
|
+
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
1747
|
+
pos += 1
|
1748
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1749
|
+
pos, value = toml_parse_value(src, pos, parse_float)
|
1750
|
+
return pos, key, value
|
1878
1751
|
|
1879
|
-
#
|
1880
1752
|
|
1881
|
-
|
1882
|
-
|
1753
|
+
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
1754
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
1755
|
+
key: TomlKey = (key_part,)
|
1756
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1757
|
+
while True:
|
1758
|
+
try:
|
1759
|
+
char: ta.Optional[str] = src[pos]
|
1760
|
+
except IndexError:
|
1761
|
+
char = None
|
1762
|
+
if char != '.':
|
1763
|
+
return pos, key
|
1764
|
+
pos += 1
|
1765
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1766
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
1767
|
+
key += (key_part,)
|
1768
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1769
|
+
|
1770
|
+
|
1771
|
+
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1772
|
+
try:
|
1773
|
+
char: ta.Optional[str] = src[pos]
|
1774
|
+
except IndexError:
|
1775
|
+
char = None
|
1776
|
+
if char in TOML_BARE_KEY_CHARS:
|
1777
|
+
start_pos = pos
|
1778
|
+
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
1779
|
+
return pos, src[start_pos:pos]
|
1780
|
+
if char == "'":
|
1781
|
+
return toml_parse_literal_str(src, pos)
|
1782
|
+
if char == '"':
|
1783
|
+
return toml_parse_one_line_basic_str(src, pos)
|
1784
|
+
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
1785
|
+
|
1786
|
+
|
1787
|
+
def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1788
|
+
pos += 1
|
1789
|
+
return toml_parse_basic_str(src, pos, multiline=False)
|
1790
|
+
|
1791
|
+
|
1792
|
+
def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
|
1793
|
+
pos += 1
|
1794
|
+
array: list = []
|
1795
|
+
|
1796
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1797
|
+
if src.startswith(']', pos):
|
1798
|
+
return pos + 1, array
|
1799
|
+
while True:
|
1800
|
+
pos, val = toml_parse_value(src, pos, parse_float)
|
1801
|
+
array.append(val)
|
1802
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1803
|
+
|
1804
|
+
c = src[pos:pos + 1]
|
1805
|
+
if c == ']':
|
1806
|
+
return pos + 1, array
|
1807
|
+
if c != ',':
|
1808
|
+
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
1809
|
+
pos += 1
|
1810
|
+
|
1811
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1812
|
+
if src.startswith(']', pos):
|
1813
|
+
return pos + 1, array
|
1814
|
+
|
1815
|
+
|
1816
|
+
def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
|
1817
|
+
pos += 1
|
1818
|
+
nested_dict = TomlNestedDict()
|
1819
|
+
flags = TomlFlags()
|
1820
|
+
|
1821
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1822
|
+
if src.startswith('}', pos):
|
1823
|
+
return pos + 1, nested_dict.dict
|
1824
|
+
while True:
|
1825
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1826
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1827
|
+
if flags.is_(key, TomlFlags.FROZEN):
|
1828
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1829
|
+
try:
|
1830
|
+
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
1831
|
+
except KeyError:
|
1832
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1833
|
+
if key_stem in nest:
|
1834
|
+
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
1835
|
+
nest[key_stem] = value
|
1836
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1837
|
+
c = src[pos:pos + 1]
|
1838
|
+
if c == '}':
|
1839
|
+
return pos + 1, nested_dict.dict
|
1840
|
+
if c != ',':
|
1841
|
+
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
1842
|
+
if isinstance(value, (dict, list)):
|
1843
|
+
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
1844
|
+
pos += 1
|
1845
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1846
|
+
|
1847
|
+
|
1848
|
+
def toml_parse_basic_str_escape(
|
1849
|
+
src: str,
|
1850
|
+
pos: TomlPos,
|
1851
|
+
*,
|
1852
|
+
multiline: bool = False,
|
1853
|
+
) -> ta.Tuple[TomlPos, str]:
|
1854
|
+
escape_id = src[pos:pos + 2]
|
1855
|
+
pos += 2
|
1856
|
+
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
1857
|
+
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
1858
|
+
# newline.
|
1859
|
+
if escape_id != '\\\n':
|
1860
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1861
|
+
try:
|
1862
|
+
char = src[pos]
|
1863
|
+
except IndexError:
|
1864
|
+
return pos, ''
|
1865
|
+
if char != '\n':
|
1866
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
1867
|
+
pos += 1
|
1868
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1869
|
+
return pos, ''
|
1870
|
+
if escape_id == '\\u':
|
1871
|
+
return toml_parse_hex_char(src, pos, 4)
|
1872
|
+
if escape_id == '\\U':
|
1873
|
+
return toml_parse_hex_char(src, pos, 8)
|
1874
|
+
try:
|
1875
|
+
return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
|
1876
|
+
except KeyError:
|
1877
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
1878
|
+
|
1883
1879
|
|
1884
|
-
|
1885
|
-
|
1886
|
-
try:
|
1887
|
-
if gi is not None:
|
1888
|
-
go = gen.send(gi)
|
1889
|
-
else:
|
1890
|
-
go = next(gen)
|
1891
|
-
except StopIteration as e:
|
1892
|
-
return e.value
|
1880
|
+
def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1881
|
+
return toml_parse_basic_str_escape(src, pos, multiline=True)
|
1893
1882
|
|
1894
|
-
if isinstance(go, self.Read):
|
1895
|
-
if len(gi := input.read(go.sz)) != go.sz:
|
1896
|
-
raise EOFError
|
1897
|
-
elif isinstance(go, self.Write):
|
1898
|
-
gi = None
|
1899
|
-
output.write(go.d)
|
1900
|
-
output.flush()
|
1901
|
-
else:
|
1902
|
-
raise TypeError(go)
|
1903
1883
|
|
1904
|
-
|
1905
|
-
|
1906
|
-
|
1907
|
-
|
1908
|
-
|
1909
|
-
|
1884
|
+
def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
|
1885
|
+
hex_str = src[pos:pos + hex_len]
|
1886
|
+
if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
|
1887
|
+
raise toml_suffixed_err(src, pos, 'Invalid hex value')
|
1888
|
+
pos += hex_len
|
1889
|
+
hex_int = int(hex_str, 16)
|
1890
|
+
if not toml_is_unicode_scalar_value(hex_int):
|
1891
|
+
raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
|
1892
|
+
return pos, chr(hex_int)
|
1910
1893
|
|
1911
|
-
gi: ta.Optional[bytes] = None
|
1912
|
-
while True:
|
1913
|
-
try:
|
1914
|
-
if gi is not None:
|
1915
|
-
go = gen.send(gi)
|
1916
|
-
else:
|
1917
|
-
go = next(gen)
|
1918
|
-
except StopIteration as e:
|
1919
|
-
return e.value
|
1920
1894
|
|
1921
|
-
|
1922
|
-
|
1923
|
-
|
1924
|
-
|
1925
|
-
|
1926
|
-
|
1927
|
-
|
1928
|
-
else:
|
1929
|
-
raise TypeError(go)
|
1895
|
+
def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1896
|
+
pos += 1 # Skip starting apostrophe
|
1897
|
+
start_pos = pos
|
1898
|
+
pos = toml_skip_until(
|
1899
|
+
src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
|
1900
|
+
)
|
1901
|
+
return pos + 1, src[start_pos:pos] # Skip ending apostrophe
|
1930
1902
|
|
1931
1903
|
|
1932
|
-
|
1933
|
-
|
1904
|
+
def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
|
1905
|
+
pos += 3
|
1906
|
+
if src.startswith('\n', pos):
|
1907
|
+
pos += 1
|
1934
1908
|
|
1909
|
+
if literal:
|
1910
|
+
delim = "'"
|
1911
|
+
end_pos = toml_skip_until(
|
1912
|
+
src,
|
1913
|
+
pos,
|
1914
|
+
"'''",
|
1915
|
+
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
1916
|
+
error_on_eof=True,
|
1917
|
+
)
|
1918
|
+
result = src[pos:end_pos]
|
1919
|
+
pos = end_pos + 3
|
1920
|
+
else:
|
1921
|
+
delim = '"'
|
1922
|
+
pos, result = toml_parse_basic_str(src, pos, multiline=True)
|
1935
1923
|
|
1936
|
-
|
1937
|
-
|
1938
|
-
|
1924
|
+
# Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
|
1925
|
+
if not src.startswith(delim, pos):
|
1926
|
+
return pos, result
|
1927
|
+
pos += 1
|
1928
|
+
if not src.startswith(delim, pos):
|
1929
|
+
return pos, result + delim
|
1930
|
+
pos += 1
|
1931
|
+
return pos, result + (delim * 2)
|
1939
1932
|
|
1940
|
-
self.reader = reader
|
1941
|
-
self.closed: asyncio.Future = asyncio.Future()
|
1942
1933
|
|
1943
|
-
|
1944
|
-
|
1945
|
-
|
1934
|
+
def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
|
1935
|
+
if multiline:
|
1936
|
+
error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1937
|
+
parse_escapes = toml_parse_basic_str_escape_multiline
|
1938
|
+
else:
|
1939
|
+
error_on = TOML_ILLEGAL_BASIC_STR_CHARS
|
1940
|
+
parse_escapes = toml_parse_basic_str_escape
|
1941
|
+
result = ''
|
1942
|
+
start_pos = pos
|
1943
|
+
while True:
|
1944
|
+
try:
|
1945
|
+
char = src[pos]
|
1946
|
+
except IndexError:
|
1947
|
+
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
1948
|
+
if char == '"':
|
1949
|
+
if not multiline:
|
1950
|
+
return pos + 1, result + src[start_pos:pos]
|
1951
|
+
if src.startswith('"""', pos):
|
1952
|
+
return pos + 3, result + src[start_pos:pos]
|
1953
|
+
pos += 1
|
1954
|
+
continue
|
1955
|
+
if char == '\\':
|
1956
|
+
result += src[start_pos:pos]
|
1957
|
+
pos, parsed_escape = parse_escapes(src, pos)
|
1958
|
+
result += parsed_escape
|
1959
|
+
start_pos = pos
|
1960
|
+
continue
|
1961
|
+
if char in error_on:
|
1962
|
+
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
1963
|
+
pos += 1
|
1946
1964
|
|
1947
|
-
# @ta.override
|
1948
|
-
def close(self) -> None:
|
1949
|
-
self.reader.feed_eof()
|
1950
|
-
if not self.closed.done():
|
1951
|
-
self.closed.set_result(True)
|
1952
1965
|
|
1953
|
-
|
1954
|
-
|
1955
|
-
|
1966
|
+
def toml_parse_value( # noqa: C901
|
1967
|
+
src: str,
|
1968
|
+
pos: TomlPos,
|
1969
|
+
parse_float: TomlParseFloat,
|
1970
|
+
) -> ta.Tuple[TomlPos, ta.Any]:
|
1971
|
+
try:
|
1972
|
+
char: ta.Optional[str] = src[pos]
|
1973
|
+
except IndexError:
|
1974
|
+
char = None
|
1956
1975
|
|
1976
|
+
# IMPORTANT: order conditions based on speed of checking and likelihood
|
1957
1977
|
|
1958
|
-
|
1959
|
-
|
1960
|
-
|
1961
|
-
|
1962
|
-
|
1978
|
+
# Basic strings
|
1979
|
+
if char == '"':
|
1980
|
+
if src.startswith('"""', pos):
|
1981
|
+
return toml_parse_multiline_str(src, pos, literal=False)
|
1982
|
+
return toml_parse_one_line_basic_str(src, pos)
|
1963
1983
|
|
1964
|
-
|
1965
|
-
|
1966
|
-
|
1967
|
-
|
1984
|
+
# Literal strings
|
1985
|
+
if char == "'":
|
1986
|
+
if src.startswith("'''", pos):
|
1987
|
+
return toml_parse_multiline_str(src, pos, literal=True)
|
1988
|
+
return toml_parse_literal_str(src, pos)
|
1968
1989
|
|
1969
|
-
|
1990
|
+
# Booleans
|
1991
|
+
if char == 't':
|
1992
|
+
if src.startswith('true', pos):
|
1993
|
+
return pos + 4, True
|
1994
|
+
if char == 'f':
|
1995
|
+
if src.startswith('false', pos):
|
1996
|
+
return pos + 5, False
|
1970
1997
|
|
1998
|
+
# Arrays
|
1999
|
+
if char == '[':
|
2000
|
+
return toml_parse_array(src, pos, parse_float)
|
1971
2001
|
|
1972
|
-
|
1973
|
-
|
2002
|
+
# Inline tables
|
2003
|
+
if char == '{':
|
2004
|
+
return toml_parse_inline_table(src, pos, parse_float)
|
1974
2005
|
|
2006
|
+
# Dates and times
|
2007
|
+
datetime_match = TOML_RE_DATETIME.match(src, pos)
|
2008
|
+
if datetime_match:
|
2009
|
+
try:
|
2010
|
+
datetime_obj = toml_match_to_datetime(datetime_match)
|
2011
|
+
except ValueError as e:
|
2012
|
+
raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
|
2013
|
+
return datetime_match.end(), datetime_obj
|
2014
|
+
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
2015
|
+
if localtime_match:
|
2016
|
+
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
1975
2017
|
|
1976
|
-
|
2018
|
+
# Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
|
2019
|
+
# located after handling of dates and times.
|
2020
|
+
number_match = TOML_RE_NUMBER.match(src, pos)
|
2021
|
+
if number_match:
|
2022
|
+
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
1977
2023
|
|
2024
|
+
# Special floats
|
2025
|
+
first_three = src[pos:pos + 3]
|
2026
|
+
if first_three in {'inf', 'nan'}:
|
2027
|
+
return pos + 3, parse_float(first_three)
|
2028
|
+
first_four = src[pos:pos + 4]
|
2029
|
+
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
2030
|
+
return pos + 4, parse_float(first_four)
|
1978
2031
|
|
1979
|
-
|
1980
|
-
f: ta.IO,
|
1981
|
-
loop: ta.Any = None,
|
1982
|
-
*,
|
1983
|
-
limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
|
1984
|
-
) -> asyncio.StreamReader:
|
1985
|
-
if loop is None:
|
1986
|
-
loop = asyncio.get_running_loop()
|
2032
|
+
raise toml_suffixed_err(src, pos, 'Invalid value')
|
1987
2033
|
|
1988
|
-
reader = asyncio.StreamReader(limit=limit, loop=loop)
|
1989
|
-
await loop.connect_read_pipe(
|
1990
|
-
lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
|
1991
|
-
f,
|
1992
|
-
)
|
1993
2034
|
|
1994
|
-
|
2035
|
+
def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
|
2036
|
+
"""Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
|
1995
2037
|
|
2038
|
+
def coord_repr(src: str, pos: TomlPos) -> str:
|
2039
|
+
if pos >= len(src):
|
2040
|
+
return 'end of document'
|
2041
|
+
line = src.count('\n', 0, pos) + 1
|
2042
|
+
if line == 1:
|
2043
|
+
column = pos + 1
|
2044
|
+
else:
|
2045
|
+
column = pos - src.rindex('\n', 0, pos)
|
2046
|
+
return f'line {line}, column {column}'
|
1996
2047
|
|
1997
|
-
|
1998
|
-
f: ta.IO,
|
1999
|
-
loop: ta.Any = None,
|
2000
|
-
) -> asyncio.StreamWriter:
|
2001
|
-
if loop is None:
|
2002
|
-
loop = asyncio.get_running_loop()
|
2048
|
+
return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
|
2003
2049
|
|
2004
|
-
writer_transport, writer_protocol = await loop.connect_write_pipe(
|
2005
|
-
lambda: asyncio.streams.FlowControlMixin(loop=loop),
|
2006
|
-
f,
|
2007
|
-
)
|
2008
2050
|
|
2009
|
-
|
2010
|
-
|
2011
|
-
writer_protocol,
|
2012
|
-
None,
|
2013
|
-
loop,
|
2014
|
-
)
|
2051
|
+
def toml_is_unicode_scalar_value(codepoint: int) -> bool:
|
2052
|
+
return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
|
2015
2053
|
|
2016
2054
|
|
2017
|
-
|
2018
|
-
|
2055
|
+
def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
|
2056
|
+
"""A decorator to make `parse_float` safe.
|
2057
|
+
|
2058
|
+
`parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
|
2059
|
+
thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
|
2060
|
+
"""
|
2061
|
+
# The default `float` callable never returns illegal types. Optimize it.
|
2062
|
+
if parse_float is float:
|
2063
|
+
return float
|
2019
2064
|
|
2065
|
+
def safe_parse_float(float_str: str) -> ta.Any:
|
2066
|
+
float_value = parse_float(float_str)
|
2067
|
+
if isinstance(float_value, (dict, list)):
|
2068
|
+
raise ValueError('parse_float must not return dicts or lists') # noqa
|
2069
|
+
return float_value
|
2020
2070
|
|
2021
|
-
|
2022
|
-
fut: AwaitableT,
|
2023
|
-
timeout: ta.Optional[float] = None,
|
2024
|
-
) -> AwaitableT:
|
2025
|
-
if timeout is not None:
|
2026
|
-
fut = asyncio.wait_for(fut, timeout) # type: ignore
|
2027
|
-
return fut
|
2071
|
+
return safe_parse_float
|
2028
2072
|
|
2029
2073
|
|
2030
2074
|
########################################
|
@@ -6986,30 +7030,6 @@ def build_config_named_children(
|
|
6986
7030
|
return lst
|
6987
7031
|
|
6988
7032
|
|
6989
|
-
##
|
6990
|
-
|
6991
|
-
|
6992
|
-
def render_ini_config(
|
6993
|
-
settings_by_section: IniConfigSectionSettingsMap,
|
6994
|
-
) -> str:
|
6995
|
-
out = io.StringIO()
|
6996
|
-
|
6997
|
-
for i, (section, settings) in enumerate(settings_by_section.items()):
|
6998
|
-
if i:
|
6999
|
-
out.write('\n')
|
7000
|
-
|
7001
|
-
out.write(f'[{section}]\n')
|
7002
|
-
|
7003
|
-
for k, v in settings.items():
|
7004
|
-
if isinstance(v, str):
|
7005
|
-
out.write(f'{k}={v}\n')
|
7006
|
-
else:
|
7007
|
-
for vv in v:
|
7008
|
-
out.write(f'{k}={vv}\n')
|
7009
|
-
|
7010
|
-
return out.getvalue()
|
7011
|
-
|
7012
|
-
|
7013
7033
|
########################################
|
7014
7034
|
# ../commands/marshal.py
|
7015
7035
|
|
@@ -7063,6 +7083,104 @@ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
|
|
7063
7083
|
CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
|
7064
7084
|
|
7065
7085
|
|
7086
|
+
########################################
|
7087
|
+
# ../deploy/conf/specs.py
|
7088
|
+
|
7089
|
+
|
7090
|
+
##
|
7091
|
+
|
7092
|
+
|
7093
|
+
class DeployAppConfContent(abc.ABC): # noqa
|
7094
|
+
pass
|
7095
|
+
|
7096
|
+
|
7097
|
+
#
|
7098
|
+
|
7099
|
+
|
7100
|
+
@register_single_field_type_obj_marshaler('body')
|
7101
|
+
@dc.dataclass(frozen=True)
|
7102
|
+
class RawDeployAppConfContent(DeployAppConfContent):
|
7103
|
+
body: str
|
7104
|
+
|
7105
|
+
|
7106
|
+
#
|
7107
|
+
|
7108
|
+
|
7109
|
+
@register_single_field_type_obj_marshaler('obj')
|
7110
|
+
@dc.dataclass(frozen=True)
|
7111
|
+
class JsonDeployAppConfContent(DeployAppConfContent):
|
7112
|
+
obj: ta.Any
|
7113
|
+
|
7114
|
+
|
7115
|
+
#
|
7116
|
+
|
7117
|
+
|
7118
|
+
@register_single_field_type_obj_marshaler('sections')
|
7119
|
+
@dc.dataclass(frozen=True)
|
7120
|
+
class IniDeployAppConfContent(DeployAppConfContent):
|
7121
|
+
sections: IniSectionSettingsMap
|
7122
|
+
|
7123
|
+
|
7124
|
+
#
|
7125
|
+
|
7126
|
+
|
7127
|
+
@register_single_field_type_obj_marshaler('items')
|
7128
|
+
@dc.dataclass(frozen=True)
|
7129
|
+
class NginxDeployAppConfContent(DeployAppConfContent):
|
7130
|
+
items: ta.Any
|
7131
|
+
|
7132
|
+
|
7133
|
+
##
|
7134
|
+
|
7135
|
+
|
7136
|
+
@dc.dataclass(frozen=True)
|
7137
|
+
class DeployAppConfFile:
|
7138
|
+
path: str
|
7139
|
+
content: DeployAppConfContent
|
7140
|
+
|
7141
|
+
def __post_init__(self) -> None:
|
7142
|
+
check_valid_deploy_spec_path(self.path)
|
7143
|
+
|
7144
|
+
|
7145
|
+
##
|
7146
|
+
|
7147
|
+
|
7148
|
+
@dc.dataclass(frozen=True)
|
7149
|
+
class DeployAppConfLink: # noqa
|
7150
|
+
"""
|
7151
|
+
May be either:
|
7152
|
+
- @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
|
7153
|
+
- @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
|
7154
|
+
- @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
|
7155
|
+
"""
|
7156
|
+
|
7157
|
+
src: str
|
7158
|
+
|
7159
|
+
kind: ta.Literal['current_only', 'all_active'] = 'current_only'
|
7160
|
+
|
7161
|
+
def __post_init__(self) -> None:
|
7162
|
+
check_valid_deploy_spec_path(self.src)
|
7163
|
+
if '/' in self.src:
|
7164
|
+
check.equal(self.src.count('/'), 1)
|
7165
|
+
|
7166
|
+
|
7167
|
+
##
|
7168
|
+
|
7169
|
+
|
7170
|
+
@dc.dataclass(frozen=True)
|
7171
|
+
class DeployAppConfSpec:
|
7172
|
+
files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
|
7173
|
+
|
7174
|
+
links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
|
7175
|
+
|
7176
|
+
def __post_init__(self) -> None:
|
7177
|
+
if self.files:
|
7178
|
+
seen: ta.Set[str] = set()
|
7179
|
+
for f in self.files:
|
7180
|
+
check.not_in(f.path, seen)
|
7181
|
+
seen.add(f.path)
|
7182
|
+
|
7183
|
+
|
7066
7184
|
########################################
|
7067
7185
|
# ../deploy/tags.py
|
7068
7186
|
|
@@ -7978,112 +8096,14 @@ class LocalCommandExecutor(CommandExecutor):
|
|
7978
8096
|
self,
|
7979
8097
|
*,
|
7980
8098
|
command_executors: CommandExecutorMap,
|
7981
|
-
) -> None:
|
7982
|
-
super().__init__()
|
7983
|
-
|
7984
|
-
self._command_executors = command_executors
|
7985
|
-
|
7986
|
-
async def execute(self, cmd: Command) -> Command.Output:
|
7987
|
-
ce: CommandExecutor = self._command_executors[type(cmd)]
|
7988
|
-
return await ce.execute(cmd)
|
7989
|
-
|
7990
|
-
|
7991
|
-
########################################
|
7992
|
-
# ../deploy/conf/specs.py
|
7993
|
-
|
7994
|
-
|
7995
|
-
##
|
7996
|
-
|
7997
|
-
|
7998
|
-
class DeployAppConfContent(abc.ABC): # noqa
|
7999
|
-
pass
|
8000
|
-
|
8001
|
-
|
8002
|
-
#
|
8003
|
-
|
8004
|
-
|
8005
|
-
@register_single_field_type_obj_marshaler('body')
|
8006
|
-
@dc.dataclass(frozen=True)
|
8007
|
-
class RawDeployAppConfContent(DeployAppConfContent):
|
8008
|
-
body: str
|
8009
|
-
|
8010
|
-
|
8011
|
-
#
|
8012
|
-
|
8013
|
-
|
8014
|
-
@register_single_field_type_obj_marshaler('obj')
|
8015
|
-
@dc.dataclass(frozen=True)
|
8016
|
-
class JsonDeployAppConfContent(DeployAppConfContent):
|
8017
|
-
obj: ta.Any
|
8018
|
-
|
8019
|
-
|
8020
|
-
#
|
8021
|
-
|
8022
|
-
|
8023
|
-
@register_single_field_type_obj_marshaler('sections')
|
8024
|
-
@dc.dataclass(frozen=True)
|
8025
|
-
class IniDeployAppConfContent(DeployAppConfContent):
|
8026
|
-
sections: IniConfigSectionSettingsMap
|
8027
|
-
|
8028
|
-
|
8029
|
-
#
|
8030
|
-
|
8031
|
-
|
8032
|
-
@register_single_field_type_obj_marshaler('items')
|
8033
|
-
@dc.dataclass(frozen=True)
|
8034
|
-
class NginxDeployAppConfContent(DeployAppConfContent):
|
8035
|
-
items: ta.Any
|
8036
|
-
|
8037
|
-
|
8038
|
-
##
|
8039
|
-
|
8040
|
-
|
8041
|
-
@dc.dataclass(frozen=True)
|
8042
|
-
class DeployAppConfFile:
|
8043
|
-
path: str
|
8044
|
-
content: DeployAppConfContent
|
8045
|
-
|
8046
|
-
def __post_init__(self) -> None:
|
8047
|
-
check_valid_deploy_spec_path(self.path)
|
8048
|
-
|
8049
|
-
|
8050
|
-
##
|
8051
|
-
|
8052
|
-
|
8053
|
-
@dc.dataclass(frozen=True)
|
8054
|
-
class DeployAppConfLink: # noqa
|
8055
|
-
"""
|
8056
|
-
May be either:
|
8057
|
-
- @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
|
8058
|
-
- @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
|
8059
|
-
- @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
|
8060
|
-
"""
|
8061
|
-
|
8062
|
-
src: str
|
8063
|
-
|
8064
|
-
kind: ta.Literal['current_only', 'all_active'] = 'current_only'
|
8065
|
-
|
8066
|
-
def __post_init__(self) -> None:
|
8067
|
-
check_valid_deploy_spec_path(self.src)
|
8068
|
-
if '/' in self.src:
|
8069
|
-
check.equal(self.src.count('/'), 1)
|
8070
|
-
|
8071
|
-
|
8072
|
-
##
|
8073
|
-
|
8074
|
-
|
8075
|
-
@dc.dataclass(frozen=True)
|
8076
|
-
class DeployAppConfSpec:
|
8077
|
-
files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
|
8078
|
-
|
8079
|
-
links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
|
8080
|
-
|
8081
|
-
def __post_init__(self) -> None:
|
8082
|
-
if self.files:
|
8083
|
-
seen: ta.Set[str] = set()
|
8084
|
-
for f in self.files:
|
8085
|
-
check.not_in(f.path, seen)
|
8086
|
-
seen.add(f.path)
|
8099
|
+
) -> None:
|
8100
|
+
super().__init__()
|
8101
|
+
|
8102
|
+
self._command_executors = command_executors
|
8103
|
+
|
8104
|
+
async def execute(self, cmd: Command) -> Command.Output:
|
8105
|
+
ce: CommandExecutor = self._command_executors[type(cmd)]
|
8106
|
+
return await ce.execute(cmd)
|
8087
8107
|
|
8088
8108
|
|
8089
8109
|
########################################
|
@@ -8288,6 +8308,127 @@ class DeployPath(DeployPathRenderable):
|
|
8288
8308
|
return cls(tuple(DeployPathPart.parse(p) for p in ps))
|
8289
8309
|
|
8290
8310
|
|
8311
|
+
########################################
|
8312
|
+
# ../deploy/specs.py
|
8313
|
+
|
8314
|
+
|
8315
|
+
##
|
8316
|
+
|
8317
|
+
|
8318
|
+
class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
|
8319
|
+
@cached_nullary
|
8320
|
+
def _key_str(self) -> str:
|
8321
|
+
return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
|
8322
|
+
|
8323
|
+
@abc.abstractmethod
|
8324
|
+
def key(self) -> KeyDeployTagT:
|
8325
|
+
raise NotImplementedError
|
8326
|
+
|
8327
|
+
|
8328
|
+
##
|
8329
|
+
|
8330
|
+
|
8331
|
+
@dc.dataclass(frozen=True)
|
8332
|
+
class DeployGitRepo:
|
8333
|
+
host: ta.Optional[str] = None
|
8334
|
+
username: ta.Optional[str] = None
|
8335
|
+
path: ta.Optional[str] = None
|
8336
|
+
|
8337
|
+
def __post_init__(self) -> None:
|
8338
|
+
check.not_in('..', check.non_empty_str(self.host))
|
8339
|
+
check.not_in('.', check.non_empty_str(self.path))
|
8340
|
+
|
8341
|
+
|
8342
|
+
@dc.dataclass(frozen=True)
|
8343
|
+
class DeployGitSpec:
|
8344
|
+
repo: DeployGitRepo
|
8345
|
+
rev: DeployRev
|
8346
|
+
|
8347
|
+
subtrees: ta.Optional[ta.Sequence[str]] = None
|
8348
|
+
|
8349
|
+
def __post_init__(self) -> None:
|
8350
|
+
check.non_empty_str(self.rev)
|
8351
|
+
if self.subtrees is not None:
|
8352
|
+
for st in self.subtrees:
|
8353
|
+
check.non_empty_str(st)
|
8354
|
+
|
8355
|
+
|
8356
|
+
##
|
8357
|
+
|
8358
|
+
|
8359
|
+
@dc.dataclass(frozen=True)
|
8360
|
+
class DeployVenvSpec:
|
8361
|
+
interp: ta.Optional[str] = None
|
8362
|
+
|
8363
|
+
requirements_files: ta.Optional[ta.Sequence[str]] = None
|
8364
|
+
extra_dependencies: ta.Optional[ta.Sequence[str]] = None
|
8365
|
+
|
8366
|
+
use_uv: bool = False
|
8367
|
+
|
8368
|
+
|
8369
|
+
##
|
8370
|
+
|
8371
|
+
|
8372
|
+
@dc.dataclass(frozen=True)
|
8373
|
+
class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
|
8374
|
+
app: DeployApp
|
8375
|
+
|
8376
|
+
git: DeployGitSpec
|
8377
|
+
|
8378
|
+
venv: ta.Optional[DeployVenvSpec] = None
|
8379
|
+
|
8380
|
+
conf: ta.Optional[DeployAppConfSpec] = None
|
8381
|
+
|
8382
|
+
# @ta.override
|
8383
|
+
def key(self) -> DeployAppKey:
|
8384
|
+
return DeployAppKey(self._key_str())
|
8385
|
+
|
8386
|
+
|
8387
|
+
@dc.dataclass(frozen=True)
|
8388
|
+
class DeployAppLinksSpec:
|
8389
|
+
apps: ta.Sequence[DeployApp] = ()
|
8390
|
+
|
8391
|
+
removed_apps: ta.Sequence[DeployApp] = ()
|
8392
|
+
|
8393
|
+
exclude_unspecified: bool = False
|
8394
|
+
|
8395
|
+
|
8396
|
+
##
|
8397
|
+
|
8398
|
+
|
8399
|
+
@dc.dataclass(frozen=True)
|
8400
|
+
class DeploySystemdSpec:
|
8401
|
+
# ~/.config/systemd/user/
|
8402
|
+
unit_dir: ta.Optional[str] = None
|
8403
|
+
|
8404
|
+
|
8405
|
+
##
|
8406
|
+
|
8407
|
+
|
8408
|
+
@dc.dataclass(frozen=True)
|
8409
|
+
class DeploySpec(DeploySpecKeyed[DeployKey]):
|
8410
|
+
home: DeployHome
|
8411
|
+
|
8412
|
+
apps: ta.Sequence[DeployAppSpec] = ()
|
8413
|
+
|
8414
|
+
app_links: DeployAppLinksSpec = DeployAppLinksSpec()
|
8415
|
+
|
8416
|
+
systemd: ta.Optional[DeploySystemdSpec] = None
|
8417
|
+
|
8418
|
+
def __post_init__(self) -> None:
|
8419
|
+
check.non_empty_str(self.home)
|
8420
|
+
|
8421
|
+
seen: ta.Set[DeployApp] = set()
|
8422
|
+
for a in self.apps:
|
8423
|
+
if a.app in seen:
|
8424
|
+
raise KeyError(a.app)
|
8425
|
+
seen.add(a.app)
|
8426
|
+
|
8427
|
+
# @ta.override
|
8428
|
+
def key(self) -> DeployKey:
|
8429
|
+
return DeployKey(self._key_str())
|
8430
|
+
|
8431
|
+
|
8291
8432
|
########################################
|
8292
8433
|
# ../remote/execution.py
|
8293
8434
|
"""
|
@@ -9320,7 +9461,7 @@ class DeployConfManager:
|
|
9320
9461
|
|
9321
9462
|
elif isinstance(ac, IniDeployAppConfContent):
|
9322
9463
|
ini_sections = pcc(ac.sections)
|
9323
|
-
return strip_with_newline(
|
9464
|
+
return strip_with_newline(render_ini_sections(ini_sections))
|
9324
9465
|
|
9325
9466
|
elif isinstance(ac, NginxDeployAppConfContent):
|
9326
9467
|
nginx_items = NginxConfigItems.of(pcc(ac.items))
|
@@ -9353,9 +9494,15 @@ class DeployConfManager:
|
|
9353
9494
|
self,
|
9354
9495
|
spec: DeployAppConfSpec,
|
9355
9496
|
app_conf_dir: str,
|
9497
|
+
*,
|
9498
|
+
string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
9356
9499
|
) -> None:
|
9357
|
-
|
9358
|
-
|
9500
|
+
process_str: ta.Any
|
9501
|
+
if string_ns is not None:
|
9502
|
+
def process_str(s: str) -> str:
|
9503
|
+
return s.format(**string_ns)
|
9504
|
+
else:
|
9505
|
+
process_str = None
|
9359
9506
|
|
9360
9507
|
for acf in spec.files or []:
|
9361
9508
|
await self._write_app_conf_file(
|
@@ -9528,125 +9675,6 @@ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
|
|
9528
9675
|
return self._owned_deploy_paths
|
9529
9676
|
|
9530
9677
|
|
9531
|
-
########################################
|
9532
|
-
# ../deploy/specs.py
|
9533
|
-
|
9534
|
-
|
9535
|
-
##
|
9536
|
-
|
9537
|
-
|
9538
|
-
class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
|
9539
|
-
@cached_nullary
|
9540
|
-
def _key_str(self) -> str:
|
9541
|
-
return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
|
9542
|
-
|
9543
|
-
@abc.abstractmethod
|
9544
|
-
def key(self) -> KeyDeployTagT:
|
9545
|
-
raise NotImplementedError
|
9546
|
-
|
9547
|
-
|
9548
|
-
##
|
9549
|
-
|
9550
|
-
|
9551
|
-
@dc.dataclass(frozen=True)
|
9552
|
-
class DeployGitRepo:
|
9553
|
-
host: ta.Optional[str] = None
|
9554
|
-
username: ta.Optional[str] = None
|
9555
|
-
path: ta.Optional[str] = None
|
9556
|
-
|
9557
|
-
def __post_init__(self) -> None:
|
9558
|
-
check.not_in('..', check.non_empty_str(self.host))
|
9559
|
-
check.not_in('.', check.non_empty_str(self.path))
|
9560
|
-
|
9561
|
-
|
9562
|
-
@dc.dataclass(frozen=True)
|
9563
|
-
class DeployGitSpec:
|
9564
|
-
repo: DeployGitRepo
|
9565
|
-
rev: DeployRev
|
9566
|
-
|
9567
|
-
subtrees: ta.Optional[ta.Sequence[str]] = None
|
9568
|
-
|
9569
|
-
def __post_init__(self) -> None:
|
9570
|
-
check.non_empty_str(self.rev)
|
9571
|
-
if self.subtrees is not None:
|
9572
|
-
for st in self.subtrees:
|
9573
|
-
check.non_empty_str(st)
|
9574
|
-
|
9575
|
-
|
9576
|
-
##
|
9577
|
-
|
9578
|
-
|
9579
|
-
@dc.dataclass(frozen=True)
|
9580
|
-
class DeployVenvSpec:
|
9581
|
-
interp: ta.Optional[str] = None
|
9582
|
-
|
9583
|
-
requirements_files: ta.Optional[ta.Sequence[str]] = None
|
9584
|
-
extra_dependencies: ta.Optional[ta.Sequence[str]] = None
|
9585
|
-
|
9586
|
-
use_uv: bool = False
|
9587
|
-
|
9588
|
-
|
9589
|
-
##
|
9590
|
-
|
9591
|
-
|
9592
|
-
@dc.dataclass(frozen=True)
|
9593
|
-
class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
|
9594
|
-
app: DeployApp
|
9595
|
-
|
9596
|
-
git: DeployGitSpec
|
9597
|
-
|
9598
|
-
venv: ta.Optional[DeployVenvSpec] = None
|
9599
|
-
|
9600
|
-
conf: ta.Optional[DeployAppConfSpec] = None
|
9601
|
-
|
9602
|
-
# @ta.override
|
9603
|
-
def key(self) -> DeployAppKey:
|
9604
|
-
return DeployAppKey(self._key_str())
|
9605
|
-
|
9606
|
-
|
9607
|
-
@dc.dataclass(frozen=True)
|
9608
|
-
class DeployAppLinksSpec:
|
9609
|
-
apps: ta.Sequence[DeployApp] = ()
|
9610
|
-
|
9611
|
-
exclude_unspecified: bool = False
|
9612
|
-
|
9613
|
-
|
9614
|
-
##
|
9615
|
-
|
9616
|
-
|
9617
|
-
@dc.dataclass(frozen=True)
|
9618
|
-
class DeploySystemdSpec:
|
9619
|
-
# ~/.config/systemd/user/
|
9620
|
-
unit_dir: ta.Optional[str] = None
|
9621
|
-
|
9622
|
-
|
9623
|
-
##
|
9624
|
-
|
9625
|
-
|
9626
|
-
@dc.dataclass(frozen=True)
|
9627
|
-
class DeploySpec(DeploySpecKeyed[DeployKey]):
|
9628
|
-
home: DeployHome
|
9629
|
-
|
9630
|
-
apps: ta.Sequence[DeployAppSpec] = ()
|
9631
|
-
|
9632
|
-
app_links: DeployAppLinksSpec = DeployAppLinksSpec()
|
9633
|
-
|
9634
|
-
systemd: ta.Optional[DeploySystemdSpec] = None
|
9635
|
-
|
9636
|
-
def __post_init__(self) -> None:
|
9637
|
-
check.non_empty_str(self.home)
|
9638
|
-
|
9639
|
-
seen: ta.Set[DeployApp] = set()
|
9640
|
-
for a in self.apps:
|
9641
|
-
if a.app in seen:
|
9642
|
-
raise KeyError(a.app)
|
9643
|
-
seen.add(a.app)
|
9644
|
-
|
9645
|
-
# @ta.override
|
9646
|
-
def key(self) -> DeployKey:
|
9647
|
-
return DeployKey(self._key_str())
|
9648
|
-
|
9649
|
-
|
9650
9678
|
########################################
|
9651
9679
|
# ../remote/_main.py
|
9652
9680
|
|
@@ -10967,7 +10995,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
|
|
10967
10995
|
|
10968
10996
|
async def fetch(self, rev: DeployRev) -> None:
|
10969
10997
|
await self.init()
|
10970
|
-
|
10998
|
+
|
10999
|
+
# This fetch shouldn't be depth=1 - git doesn't reuse local data with shallow fetches.
|
11000
|
+
await self._call('git', 'fetch', 'origin', rev)
|
10971
11001
|
|
10972
11002
|
#
|
10973
11003
|
|
@@ -11125,7 +11155,7 @@ class DeploySystemdManager:
|
|
11125
11155
|
|
11126
11156
|
#
|
11127
11157
|
|
11128
|
-
if sys.platform == 'linux':
|
11158
|
+
if sys.platform == 'linux' and shutil.which('systemctl') is not None:
|
11129
11159
|
async def reload() -> None:
|
11130
11160
|
await asyncio_subprocesses.check_call('systemctl', '--user', 'daemon-reload')
|
11131
11161
|
|
@@ -11614,6 +11644,8 @@ class DeployAppManager(DeployPathOwner):
|
|
11614
11644
|
spec: DeployAppSpec,
|
11615
11645
|
home: DeployHome,
|
11616
11646
|
tags: DeployTagMap,
|
11647
|
+
*,
|
11648
|
+
conf_string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
11617
11649
|
) -> PreparedApp:
|
11618
11650
|
spec_json = json_dumps_pretty(self._msh.marshal_obj(spec))
|
11619
11651
|
|
@@ -11670,9 +11702,17 @@ class DeployAppManager(DeployPathOwner):
|
|
11670
11702
|
if spec.conf is not None:
|
11671
11703
|
conf_dir = os.path.join(app_dir, 'conf')
|
11672
11704
|
rkw.update(conf_dir=conf_dir)
|
11705
|
+
|
11706
|
+
conf_ns: ta.Dict[str, ta.Any] = dict(
|
11707
|
+
**(conf_string_ns or {}),
|
11708
|
+
app=spec.app.s,
|
11709
|
+
app_dir=app_dir.rstrip('/'),
|
11710
|
+
)
|
11711
|
+
|
11673
11712
|
await self._conf.write_app_conf(
|
11674
11713
|
spec.conf,
|
11675
11714
|
conf_dir,
|
11715
|
+
string_ns=conf_ns,
|
11676
11716
|
)
|
11677
11717
|
|
11678
11718
|
#
|
@@ -11735,7 +11775,7 @@ class DeployAppManager(DeployPathOwner):
|
|
11735
11775
|
##
|
11736
11776
|
|
11737
11777
|
|
11738
|
-
DEPLOY_TAG_DATETIME_FMT = '%Y
|
11778
|
+
DEPLOY_TAG_DATETIME_FMT = '%Y-%m-%d-T-%H-%M-%S-%f-Z'
|
11739
11779
|
|
11740
11780
|
|
11741
11781
|
DeployManagerUtcClock = ta.NewType('DeployManagerUtcClock', Func0[datetime.datetime])
|
@@ -11888,8 +11928,9 @@ class DeployDriver:
|
|
11888
11928
|
|
11889
11929
|
das: ta.Set[DeployApp] = {a.app for a in self._spec.apps}
|
11890
11930
|
las: ta.Set[DeployApp] = set(self._spec.app_links.apps)
|
11891
|
-
|
11892
|
-
|
11931
|
+
ras: ta.Set[DeployApp] = set(self._spec.app_links.removed_apps)
|
11932
|
+
check.empty(das & (las | ras))
|
11933
|
+
check.empty(las & ras)
|
11893
11934
|
|
11894
11935
|
#
|
11895
11936
|
|
@@ -11935,10 +11976,10 @@ class DeployDriver:
|
|
11935
11976
|
cad = abs_real_path(os.path.join(current_link, 'apps'))
|
11936
11977
|
if os.path.exists(cad):
|
11937
11978
|
for d in os.listdir(cad):
|
11938
|
-
if (da := DeployApp(d)) not in das:
|
11979
|
+
if (da := DeployApp(d)) not in das and da not in ras:
|
11939
11980
|
las.add(da)
|
11940
11981
|
|
11941
|
-
for la in
|
11982
|
+
for la in las:
|
11942
11983
|
await self._drive_app_link(
|
11943
11984
|
la,
|
11944
11985
|
current_link,
|
@@ -11968,10 +12009,16 @@ class DeployDriver:
|
|
11968
12009
|
#
|
11969
12010
|
|
11970
12011
|
async def _drive_app_deploy(self, app: DeployAppSpec) -> None:
|
12012
|
+
current_deploy_link = os.path.join(self._home, self._deploys.CURRENT_DEPLOY_LINK.render())
|
12013
|
+
|
11971
12014
|
pa = await self._apps.prepare_app(
|
11972
12015
|
app,
|
11973
12016
|
self._home,
|
11974
12017
|
self.tags,
|
12018
|
+
conf_string_ns=dict(
|
12019
|
+
deploy_home=self._home,
|
12020
|
+
current_deploy_link=current_deploy_link,
|
12021
|
+
),
|
11975
12022
|
)
|
11976
12023
|
|
11977
12024
|
#
|