ominfra 0.0.0.dev191__py3-none-any.whl → 0.0.0.dev193__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
ominfra/scripts/manage.py CHANGED
@@ -16,6 +16,7 @@ import asyncio.subprocess
16
16
  import base64
17
17
  import collections
18
18
  import collections.abc
19
+ import configparser
19
20
  import contextlib
20
21
  import contextvars
21
22
  import ctypes as ct
@@ -73,17 +74,20 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
73
74
  VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
74
75
  VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
75
76
 
76
- # ../../omdev/toml/parser.py
77
- TomlParseFloat = ta.Callable[[str], ta.Any]
78
- TomlKey = ta.Tuple[str, ...]
79
- TomlPos = int # ta.TypeAlias
80
-
81
77
  # deploy/paths/types.py
82
78
  DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
83
79
 
84
80
  # ../../omlish/asyncs/asyncio/timeouts.py
85
81
  AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
86
82
 
83
+ # ../../omlish/formats/ini/sections.py
84
+ IniSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
85
+
86
+ # ../../omlish/formats/toml/parser.py
87
+ TomlParseFloat = ta.Callable[[str], ta.Any]
88
+ TomlKey = ta.Tuple[str, ...]
89
+ TomlPos = int # ta.TypeAlias
90
+
87
91
  # ../../omlish/lite/cached.py
88
92
  T = ta.TypeVar('T')
89
93
  CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
@@ -129,7 +133,6 @@ AtomicPathSwapState = ta.Literal['open', 'committed', 'aborted'] # ta.TypeAlias
129
133
 
130
134
  # ../configs.py
131
135
  ConfigMapping = ta.Mapping[str, ta.Any]
132
- IniConfigSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
133
136
 
134
137
  # ../../omlish/subprocesses.py
135
138
  SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
@@ -549,1482 +552,1523 @@ def canonicalize_version(
549
552
 
550
553
 
551
554
  ########################################
552
- # ../../../omdev/toml/parser.py
553
- # SPDX-License-Identifier: MIT
554
- # SPDX-FileCopyrightText: 2021 Taneli Hukkinen
555
- # Licensed to PSF under a Contributor Agreement.
556
- #
557
- # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
558
- # --------------------------------------------
559
- #
560
- # 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
561
- # ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
562
- # documentation.
563
- #
564
- # 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
565
- # royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
566
- # works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
567
- # Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
568
- # 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
569
- # Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
570
- #
571
- # 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
572
- # wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
573
- # any such work a brief summary of the changes made to Python.
574
- #
575
- # 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
576
- # EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
577
- # OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
578
- # RIGHTS.
579
- #
580
- # 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
581
- # DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
582
- # ADVISED OF THE POSSIBILITY THEREOF.
583
- #
584
- # 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
585
- #
586
- # 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
587
- # venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
588
- # name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
589
- #
590
- # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
591
- # License Agreement.
592
- #
593
- # https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
555
+ # ../config.py
594
556
 
595
557
 
596
- ##
558
+ @dc.dataclass(frozen=True)
559
+ class MainConfig:
560
+ log_level: ta.Optional[str] = 'INFO'
597
561
 
562
+ debug: bool = False
598
563
 
599
- _TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
600
564
 
601
- TOML_RE_NUMBER = re.compile(
602
- r"""
603
- 0
604
- (?:
605
- x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
606
- |
607
- b[01](?:_?[01])* # bin
608
- |
609
- o[0-7](?:_?[0-7])* # oct
610
- )
611
- |
612
- [+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
613
- (?P<floatpart>
614
- (?:\.[0-9](?:_?[0-9])*)? # optional fractional part
615
- (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
616
- )
617
- """,
618
- flags=re.VERBOSE,
619
- )
620
- TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
621
- TOML_RE_DATETIME = re.compile(
622
- rf"""
623
- ([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
624
- (?:
625
- [Tt ]
626
- {_TOML_TIME_RE_STR}
627
- (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
628
- )?
629
- """,
630
- flags=re.VERBOSE,
631
- )
565
+ ########################################
566
+ # ../deploy/config.py
632
567
 
633
568
 
634
- def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
635
- """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
569
+ ##
636
570
 
637
- Raises ValueError if the match does not correspond to a valid date or datetime.
638
- """
639
- (
640
- year_str,
641
- month_str,
642
- day_str,
643
- hour_str,
644
- minute_str,
645
- sec_str,
646
- micros_str,
647
- zulu_time,
648
- offset_sign_str,
649
- offset_hour_str,
650
- offset_minute_str,
651
- ) = match.groups()
652
- year, month, day = int(year_str), int(month_str), int(day_str)
653
- if hour_str is None:
654
- return datetime.date(year, month, day)
655
- hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
656
- micros = int(micros_str.ljust(6, '0')) if micros_str else 0
657
- if offset_sign_str:
658
- tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
659
- offset_hour_str, offset_minute_str, offset_sign_str,
660
- )
661
- elif zulu_time:
662
- tz = datetime.UTC
663
- else: # local date-time
664
- tz = None
665
- return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
666
571
 
572
+ @dc.dataclass(frozen=True)
573
+ class DeployConfig:
574
+ pass
667
575
 
668
- @functools.lru_cache() # noqa
669
- def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
670
- sign = 1 if sign_str == '+' else -1
671
- return datetime.timezone(
672
- datetime.timedelta(
673
- hours=sign * int(hour_str),
674
- minutes=sign * int(minute_str),
675
- ),
676
- )
677
576
 
577
+ ########################################
578
+ # ../deploy/paths/types.py
678
579
 
679
- def toml_match_to_localtime(match: re.Match) -> datetime.time:
680
- hour_str, minute_str, sec_str, micros_str = match.groups()
681
- micros = int(micros_str.ljust(6, '0')) if micros_str else 0
682
- return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
683
580
 
581
+ ##
684
582
 
685
- def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
686
- if match.group('floatpart'):
687
- return parse_float(match.group())
688
- return int(match.group(), 0)
689
583
 
584
+ ########################################
585
+ # ../deploy/types.py
690
586
 
691
- TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
692
587
 
693
- # Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
694
- # functions.
695
- TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
696
- TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
588
+ ##
697
589
 
698
- TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
699
- TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
700
590
 
701
- TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
591
+ DeployHome = ta.NewType('DeployHome', str)
702
592
 
703
- TOML_WS = frozenset(' \t')
704
- TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
705
- TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
706
- TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
707
- TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
593
+ DeployRev = ta.NewType('DeployRev', str)
708
594
 
709
- TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
710
- {
711
- '\\b': '\u0008', # backspace
712
- '\\t': '\u0009', # tab
713
- '\\n': '\u000A', # linefeed
714
- '\\f': '\u000C', # form feed
715
- '\\r': '\u000D', # carriage return
716
- '\\"': '\u0022', # quote
717
- '\\\\': '\u005C', # backslash
718
- },
719
- )
720
595
 
596
+ ########################################
597
+ # ../../pyremote.py
598
+ """
599
+ Basically this: https://mitogen.networkgenomics.com/howitworks.html
721
600
 
722
- class TomlDecodeError(ValueError):
723
- """An error raised if a document is not valid TOML."""
601
+ TODO:
602
+ - log: ta.Optional[logging.Logger] = None + log.debug's
603
+ """
724
604
 
725
605
 
726
- def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
727
- """Parse TOML from a binary file object."""
728
- b = fp.read()
729
- try:
730
- s = b.decode()
731
- except AttributeError:
732
- raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
733
- return toml_loads(s, parse_float=parse_float)
606
+ ##
734
607
 
735
608
 
736
- def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
737
- """Parse TOML from a string."""
609
+ @dc.dataclass(frozen=True)
610
+ class PyremoteBootstrapOptions:
611
+ debug: bool = False
738
612
 
739
- # The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
740
- try:
741
- src = s.replace('\r\n', '\n')
742
- except (AttributeError, TypeError):
743
- raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
744
- pos = 0
745
- out = TomlOutput(TomlNestedDict(), TomlFlags())
746
- header: TomlKey = ()
747
- parse_float = toml_make_safe_parse_float(parse_float)
613
+ DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
614
+ main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
748
615
 
749
- # Parse one statement at a time (typically means one line in TOML source)
750
- while True:
751
- # 1. Skip line leading whitespace
752
- pos = toml_skip_chars(src, pos, TOML_WS)
753
616
 
754
- # 2. Parse rules. Expect one of the following:
755
- # - end of file
756
- # - end of line
757
- # - comment
758
- # - key/value pair
759
- # - append dict to list (and move to its namespace)
760
- # - create dict (and move to its namespace)
761
- # Skip trailing whitespace when applicable.
762
- try:
763
- char = src[pos]
764
- except IndexError:
765
- break
766
- if char == '\n':
767
- pos += 1
768
- continue
769
- if char in TOML_KEY_INITIAL_CHARS:
770
- pos = toml_key_value_rule(src, pos, out, header, parse_float)
771
- pos = toml_skip_chars(src, pos, TOML_WS)
772
- elif char == '[':
773
- try:
774
- second_char: ta.Optional[str] = src[pos + 1]
775
- except IndexError:
776
- second_char = None
777
- out.flags.finalize_pending()
778
- if second_char == '[':
779
- pos, header = toml_create_list_rule(src, pos, out)
780
- else:
781
- pos, header = toml_create_dict_rule(src, pos, out)
782
- pos = toml_skip_chars(src, pos, TOML_WS)
783
- elif char != '#':
784
- raise toml_suffixed_err(src, pos, 'Invalid statement')
617
+ ##
785
618
 
786
- # 3. Skip comment
787
- pos = toml_skip_comment(src, pos)
788
619
 
789
- # 4. Expect end of line or end of file
790
- try:
791
- char = src[pos]
792
- except IndexError:
793
- break
794
- if char != '\n':
795
- raise toml_suffixed_err(
796
- src, pos, 'Expected newline or end of document after a statement',
797
- )
798
- pos += 1
620
+ @dc.dataclass(frozen=True)
621
+ class PyremoteEnvInfo:
622
+ sys_base_prefix: str
623
+ sys_byteorder: str
624
+ sys_defaultencoding: str
625
+ sys_exec_prefix: str
626
+ sys_executable: str
627
+ sys_implementation_name: str
628
+ sys_path: ta.List[str]
629
+ sys_platform: str
630
+ sys_prefix: str
631
+ sys_version: str
632
+ sys_version_info: ta.List[ta.Union[int, str]]
799
633
 
800
- return out.data.dict
634
+ platform_architecture: ta.List[str]
635
+ platform_machine: str
636
+ platform_platform: str
637
+ platform_processor: str
638
+ platform_system: str
639
+ platform_release: str
640
+ platform_version: str
801
641
 
642
+ site_userbase: str
802
643
 
803
- class TomlFlags:
804
- """Flags that map to parsed keys/namespaces."""
644
+ os_cwd: str
645
+ os_gid: int
646
+ os_loadavg: ta.List[float]
647
+ os_login: ta.Optional[str]
648
+ os_pgrp: int
649
+ os_pid: int
650
+ os_ppid: int
651
+ os_uid: int
805
652
 
806
- # Marks an immutable namespace (inline array or inline table).
807
- FROZEN = 0
808
- # Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
809
- EXPLICIT_NEST = 1
653
+ pw_name: str
654
+ pw_uid: int
655
+ pw_gid: int
656
+ pw_gecos: str
657
+ pw_dir: str
658
+ pw_shell: str
810
659
 
811
- def __init__(self) -> None:
812
- self._flags: ta.Dict[str, dict] = {}
813
- self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
660
+ env_path: ta.Optional[str]
814
661
 
815
- def add_pending(self, key: TomlKey, flag: int) -> None:
816
- self._pending_flags.add((key, flag))
817
662
 
818
- def finalize_pending(self) -> None:
819
- for key, flag in self._pending_flags:
820
- self.set(key, flag, recursive=False)
821
- self._pending_flags.clear()
663
+ def _get_pyremote_env_info() -> PyremoteEnvInfo:
664
+ os_uid = os.getuid()
822
665
 
823
- def unset_all(self, key: TomlKey) -> None:
824
- cont = self._flags
825
- for k in key[:-1]:
826
- if k not in cont:
827
- return
828
- cont = cont[k]['nested']
829
- cont.pop(key[-1], None)
666
+ pw = pwd.getpwuid(os_uid)
830
667
 
831
- def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
832
- cont = self._flags
833
- key_parent, key_stem = key[:-1], key[-1]
834
- for k in key_parent:
835
- if k not in cont:
836
- cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
837
- cont = cont[k]['nested']
838
- if key_stem not in cont:
839
- cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
840
- cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
668
+ os_login: ta.Optional[str]
669
+ try:
670
+ os_login = os.getlogin()
671
+ except OSError:
672
+ os_login = None
841
673
 
842
- def is_(self, key: TomlKey, flag: int) -> bool:
843
- if not key:
844
- return False # document root has no flags
845
- cont = self._flags
846
- for k in key[:-1]:
847
- if k not in cont:
848
- return False
849
- inner_cont = cont[k]
850
- if flag in inner_cont['recursive_flags']:
851
- return True
852
- cont = inner_cont['nested']
853
- key_stem = key[-1]
854
- if key_stem in cont:
855
- cont = cont[key_stem]
856
- return flag in cont['flags'] or flag in cont['recursive_flags']
857
- return False
674
+ return PyremoteEnvInfo(
675
+ sys_base_prefix=sys.base_prefix,
676
+ sys_byteorder=sys.byteorder,
677
+ sys_defaultencoding=sys.getdefaultencoding(),
678
+ sys_exec_prefix=sys.exec_prefix,
679
+ sys_executable=sys.executable,
680
+ sys_implementation_name=sys.implementation.name,
681
+ sys_path=sys.path,
682
+ sys_platform=sys.platform,
683
+ sys_prefix=sys.prefix,
684
+ sys_version=sys.version,
685
+ sys_version_info=list(sys.version_info),
858
686
 
687
+ platform_architecture=list(platform.architecture()),
688
+ platform_machine=platform.machine(),
689
+ platform_platform=platform.platform(),
690
+ platform_processor=platform.processor(),
691
+ platform_system=platform.system(),
692
+ platform_release=platform.release(),
693
+ platform_version=platform.version(),
859
694
 
860
- class TomlNestedDict:
861
- def __init__(self) -> None:
862
- # The parsed content of the TOML document
863
- self.dict: ta.Dict[str, ta.Any] = {}
695
+ site_userbase=site.getuserbase(),
864
696
 
865
- def get_or_create_nest(
866
- self,
867
- key: TomlKey,
868
- *,
869
- access_lists: bool = True,
870
- ) -> dict:
871
- cont: ta.Any = self.dict
872
- for k in key:
873
- if k not in cont:
874
- cont[k] = {}
875
- cont = cont[k]
876
- if access_lists and isinstance(cont, list):
877
- cont = cont[-1]
878
- if not isinstance(cont, dict):
879
- raise KeyError('There is no nest behind this key')
880
- return cont
697
+ os_cwd=os.getcwd(),
698
+ os_gid=os.getgid(),
699
+ os_loadavg=list(os.getloadavg()),
700
+ os_login=os_login,
701
+ os_pgrp=os.getpgrp(),
702
+ os_pid=os.getpid(),
703
+ os_ppid=os.getppid(),
704
+ os_uid=os_uid,
881
705
 
882
- def append_nest_to_list(self, key: TomlKey) -> None:
883
- cont = self.get_or_create_nest(key[:-1])
884
- last_key = key[-1]
885
- if last_key in cont:
886
- list_ = cont[last_key]
887
- if not isinstance(list_, list):
888
- raise KeyError('An object other than list found behind this key')
889
- list_.append({})
890
- else:
891
- cont[last_key] = [{}]
706
+ pw_name=pw.pw_name,
707
+ pw_uid=pw.pw_uid,
708
+ pw_gid=pw.pw_gid,
709
+ pw_gecos=pw.pw_gecos,
710
+ pw_dir=pw.pw_dir,
711
+ pw_shell=pw.pw_shell,
892
712
 
713
+ env_path=os.environ.get('PATH'),
714
+ )
893
715
 
894
- class TomlOutput(ta.NamedTuple):
895
- data: TomlNestedDict
896
- flags: TomlFlags
897
716
 
717
+ ##
898
718
 
899
- def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
900
- try:
901
- while src[pos] in chars:
902
- pos += 1
903
- except IndexError:
904
- pass
905
- return pos
906
719
 
720
+ _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
721
+ _PYREMOTE_BOOTSTRAP_SRC_FD = 101
907
722
 
908
- def toml_skip_until(
909
- src: str,
910
- pos: TomlPos,
911
- expect: str,
912
- *,
913
- error_on: ta.FrozenSet[str],
914
- error_on_eof: bool,
915
- ) -> TomlPos:
916
- try:
917
- new_pos = src.index(expect, pos)
918
- except ValueError:
919
- new_pos = len(src)
920
- if error_on_eof:
921
- raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
723
+ _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
724
+ _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
725
+ _PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
726
+ _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
727
+ _PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
922
728
 
923
- if not error_on.isdisjoint(src[pos:new_pos]):
924
- while src[pos] not in error_on:
925
- pos += 1
926
- raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
927
- return new_pos
729
+ _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
730
+ _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
731
+ _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
732
+ _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
928
733
 
734
+ _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
929
735
 
930
- def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
931
- try:
932
- char: ta.Optional[str] = src[pos]
933
- except IndexError:
934
- char = None
935
- if char == '#':
936
- return toml_skip_until(
937
- src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
938
- )
939
- return pos
736
+ _PYREMOTE_BOOTSTRAP_IMPORTS = [
737
+ 'base64',
738
+ 'os',
739
+ 'struct',
740
+ 'sys',
741
+ 'zlib',
742
+ ]
940
743
 
941
744
 
942
- def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
943
- while True:
944
- pos_before_skip = pos
945
- pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
946
- pos = toml_skip_comment(src, pos)
947
- if pos == pos_before_skip:
948
- return pos
745
+ def _pyremote_bootstrap_main(context_name: str) -> None:
746
+ # Get pid
747
+ pid = os.getpid()
949
748
 
749
+ # Two copies of payload src to be sent to parent
750
+ r0, w0 = os.pipe()
751
+ r1, w1 = os.pipe()
950
752
 
951
- def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
952
- pos += 1 # Skip "["
953
- pos = toml_skip_chars(src, pos, TOML_WS)
954
- pos, key = toml_parse_key(src, pos)
753
+ if (cp := os.fork()):
754
+ # Parent process
955
755
 
956
- if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
957
- raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
958
- out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
959
- try:
960
- out.data.get_or_create_nest(key)
961
- except KeyError:
962
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
756
+ # Dup original stdin to comm_fd for use as comm channel
757
+ os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
963
758
 
964
- if not src.startswith(']', pos):
965
- raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
966
- return pos + 1, key
759
+ # Overwrite stdin (fed to python repl) with first copy of src
760
+ os.dup2(r0, 0)
967
761
 
762
+ # Dup second copy of src to src_fd to recover after launch
763
+ os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
968
764
 
969
- def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
970
- pos += 2 # Skip "[["
971
- pos = toml_skip_chars(src, pos, TOML_WS)
972
- pos, key = toml_parse_key(src, pos)
973
-
974
- if out.flags.is_(key, TomlFlags.FROZEN):
975
- raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
976
- # Free the namespace now that it points to another empty list item...
977
- out.flags.unset_all(key)
978
- # ...but this key precisely is still prohibited from table declaration
979
- out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
980
- try:
981
- out.data.append_nest_to_list(key)
982
- except KeyError:
983
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
765
+ # Close remaining fd's
766
+ for f in [r0, w0, r1, w1]:
767
+ os.close(f)
984
768
 
985
- if not src.startswith(']]', pos):
986
- raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
987
- return pos + 2, key
769
+ # Save vars
770
+ env = os.environ
771
+ exe = sys.executable
772
+ env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
773
+ env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
774
+ env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
988
775
 
776
+ # Start repl reading stdin from r0
777
+ os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
989
778
 
990
- def toml_key_value_rule(
991
- src: str,
992
- pos: TomlPos,
993
- out: TomlOutput,
994
- header: TomlKey,
995
- parse_float: TomlParseFloat,
996
- ) -> TomlPos:
997
- pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
998
- key_parent, key_stem = key[:-1], key[-1]
999
- abs_key_parent = header + key_parent
779
+ else:
780
+ # Child process
1000
781
 
1001
- relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
1002
- for cont_key in relative_path_cont_keys:
1003
- # Check that dotted key syntax does not redefine an existing table
1004
- if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
1005
- raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
1006
- # Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
1007
- # table sections.
1008
- out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
782
+ # Write first ack
783
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
1009
784
 
1010
- if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
1011
- raise toml_suffixed_err(
1012
- src,
1013
- pos,
1014
- f'Cannot mutate immutable namespace {abs_key_parent}',
1015
- )
785
+ # Write pid
786
+ os.write(1, struct.pack('<Q', pid))
1016
787
 
1017
- try:
1018
- nest = out.data.get_or_create_nest(abs_key_parent)
1019
- except KeyError:
1020
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1021
- if key_stem in nest:
1022
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
1023
- # Mark inline table and array namespaces recursively immutable
1024
- if isinstance(value, (dict, list)):
1025
- out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
1026
- nest[key_stem] = value
1027
- return pos
788
+ # Read payload src from stdin
789
+ payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
790
+ if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
791
+ raise EOFError
792
+ payload_src = zlib.decompress(payload_z)
1028
793
 
794
+ # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
795
+ # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
796
+ for w in [w0, w1]:
797
+ fp = os.fdopen(w, 'wb', 0)
798
+ fp.write(payload_src)
799
+ fp.close()
1029
800
 
1030
- def toml_parse_key_value_pair(
1031
- src: str,
1032
- pos: TomlPos,
1033
- parse_float: TomlParseFloat,
1034
- ) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
1035
- pos, key = toml_parse_key(src, pos)
1036
- try:
1037
- char: ta.Optional[str] = src[pos]
1038
- except IndexError:
1039
- char = None
1040
- if char != '=':
1041
- raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
1042
- pos += 1
1043
- pos = toml_skip_chars(src, pos, TOML_WS)
1044
- pos, value = toml_parse_value(src, pos, parse_float)
1045
- return pos, key, value
801
+ # Write second ack
802
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
1046
803
 
804
+ # Exit child
805
+ sys.exit(0)
1047
806
 
1048
- def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
1049
- pos, key_part = toml_parse_key_part(src, pos)
1050
- key: TomlKey = (key_part,)
1051
- pos = toml_skip_chars(src, pos, TOML_WS)
1052
- while True:
1053
- try:
1054
- char: ta.Optional[str] = src[pos]
1055
- except IndexError:
1056
- char = None
1057
- if char != '.':
1058
- return pos, key
1059
- pos += 1
1060
- pos = toml_skip_chars(src, pos, TOML_WS)
1061
- pos, key_part = toml_parse_key_part(src, pos)
1062
- key += (key_part,)
1063
- pos = toml_skip_chars(src, pos, TOML_WS)
1064
807
 
808
+ ##
1065
809
 
1066
- def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1067
- try:
1068
- char: ta.Optional[str] = src[pos]
1069
- except IndexError:
1070
- char = None
1071
- if char in TOML_BARE_KEY_CHARS:
1072
- start_pos = pos
1073
- pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
1074
- return pos, src[start_pos:pos]
1075
- if char == "'":
1076
- return toml_parse_literal_str(src, pos)
1077
- if char == '"':
1078
- return toml_parse_one_line_basic_str(src, pos)
1079
- raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
1080
810
 
811
+ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
812
+ if any(c in context_name for c in '\'"'):
813
+ raise NameError(context_name)
1081
814
 
1082
- def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1083
- pos += 1
1084
- return toml_parse_basic_str(src, pos, multiline=False)
815
+ import inspect
816
+ import textwrap
817
+ bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
1085
818
 
819
+ for gl in [
820
+ '_PYREMOTE_BOOTSTRAP_INPUT_FD',
821
+ '_PYREMOTE_BOOTSTRAP_SRC_FD',
1086
822
 
1087
- def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
1088
- pos += 1
1089
- array: list = []
823
+ '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
824
+ '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
825
+ '_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
1090
826
 
1091
- pos = toml_skip_comments_and_array_ws(src, pos)
1092
- if src.startswith(']', pos):
1093
- return pos + 1, array
1094
- while True:
1095
- pos, val = toml_parse_value(src, pos, parse_float)
1096
- array.append(val)
1097
- pos = toml_skip_comments_and_array_ws(src, pos)
827
+ '_PYREMOTE_BOOTSTRAP_ACK0',
828
+ '_PYREMOTE_BOOTSTRAP_ACK1',
1098
829
 
1099
- c = src[pos:pos + 1]
1100
- if c == ']':
1101
- return pos + 1, array
1102
- if c != ',':
1103
- raise toml_suffixed_err(src, pos, 'Unclosed array')
1104
- pos += 1
830
+ '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
831
+ ]:
832
+ bs_src = bs_src.replace(gl, repr(globals()[gl]))
1105
833
 
1106
- pos = toml_skip_comments_and_array_ws(src, pos)
1107
- if src.startswith(']', pos):
1108
- return pos + 1, array
834
+ bs_src = '\n'.join(
835
+ cl
836
+ for l in bs_src.splitlines()
837
+ if (cl := (l.split('#')[0]).rstrip())
838
+ if cl.strip()
839
+ )
1109
840
 
841
+ bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
842
+ bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
843
+ if b'"' in bs_z85:
844
+ raise ValueError(bs_z85)
1110
845
 
1111
- def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
1112
- pos += 1
1113
- nested_dict = TomlNestedDict()
1114
- flags = TomlFlags()
846
+ stmts = [
847
+ f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
848
+ f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
849
+ f'_pyremote_bootstrap_main("{context_name}")',
850
+ ]
1115
851
 
1116
- pos = toml_skip_chars(src, pos, TOML_WS)
1117
- if src.startswith('}', pos):
1118
- return pos + 1, nested_dict.dict
1119
- while True:
1120
- pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1121
- key_parent, key_stem = key[:-1], key[-1]
1122
- if flags.is_(key, TomlFlags.FROZEN):
1123
- raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1124
- try:
1125
- nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
1126
- except KeyError:
1127
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1128
- if key_stem in nest:
1129
- raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
1130
- nest[key_stem] = value
1131
- pos = toml_skip_chars(src, pos, TOML_WS)
1132
- c = src[pos:pos + 1]
1133
- if c == '}':
1134
- return pos + 1, nested_dict.dict
1135
- if c != ',':
1136
- raise toml_suffixed_err(src, pos, 'Unclosed inline table')
1137
- if isinstance(value, (dict, list)):
1138
- flags.set(key, TomlFlags.FROZEN, recursive=True)
1139
- pos += 1
1140
- pos = toml_skip_chars(src, pos, TOML_WS)
852
+ cmd = '; '.join(stmts)
853
+ return cmd
1141
854
 
1142
855
 
1143
- def toml_parse_basic_str_escape(
1144
- src: str,
1145
- pos: TomlPos,
1146
- *,
1147
- multiline: bool = False,
1148
- ) -> ta.Tuple[TomlPos, str]:
1149
- escape_id = src[pos:pos + 2]
1150
- pos += 2
1151
- if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
1152
- # Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
1153
- # newline.
1154
- if escape_id != '\\\n':
1155
- pos = toml_skip_chars(src, pos, TOML_WS)
1156
- try:
1157
- char = src[pos]
1158
- except IndexError:
1159
- return pos, ''
1160
- if char != '\n':
1161
- raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
1162
- pos += 1
1163
- pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1164
- return pos, ''
1165
- if escape_id == '\\u':
1166
- return toml_parse_hex_char(src, pos, 4)
1167
- if escape_id == '\\U':
1168
- return toml_parse_hex_char(src, pos, 8)
1169
- try:
1170
- return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
1171
- except KeyError:
1172
- raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
856
+ ##
1173
857
 
1174
858
 
1175
- def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1176
- return toml_parse_basic_str_escape(src, pos, multiline=True)
859
+ @dc.dataclass(frozen=True)
860
+ class PyremotePayloadRuntime:
861
+ input: ta.BinaryIO
862
+ output: ta.BinaryIO
863
+ context_name: str
864
+ payload_src: str
865
+ options: PyremoteBootstrapOptions
866
+ env_info: PyremoteEnvInfo
1177
867
 
1178
868
 
1179
- def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
1180
- hex_str = src[pos:pos + hex_len]
1181
- if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
1182
- raise toml_suffixed_err(src, pos, 'Invalid hex value')
1183
- pos += hex_len
1184
- hex_int = int(hex_str, 16)
1185
- if not toml_is_unicode_scalar_value(hex_int):
1186
- raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
1187
- return pos, chr(hex_int)
869
+ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
870
+ # If src file var is not present we need to do initial finalization
871
+ if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
872
+ # Read second copy of payload src
873
+ r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
874
+ payload_src = r1.read().decode('utf-8')
875
+ r1.close()
876
+
877
+ # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
878
+ # a pipe at once.
879
+ os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
1188
880
 
881
+ # Read options
882
+ options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
883
+ if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
884
+ raise EOFError
885
+ options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
1189
886
 
1190
- def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1191
- pos += 1 # Skip starting apostrophe
1192
- start_pos = pos
1193
- pos = toml_skip_until(
1194
- src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
1195
- )
1196
- return pos + 1, src[start_pos:pos] # Skip ending apostrophe
887
+ # If debugging, re-exec as file
888
+ if options.debug:
889
+ # Write temp source file
890
+ import tempfile
891
+ tfd, tfn = tempfile.mkstemp('-pyremote.py')
892
+ os.write(tfd, payload_src.encode('utf-8'))
893
+ os.close(tfd)
1197
894
 
895
+ # Set vars
896
+ os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
897
+ os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
1198
898
 
1199
- def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
1200
- pos += 3
1201
- if src.startswith('\n', pos):
1202
- pos += 1
899
+ # Re-exec temp file
900
+ exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
901
+ context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
902
+ os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
1203
903
 
1204
- if literal:
1205
- delim = "'"
1206
- end_pos = toml_skip_until(
1207
- src,
1208
- pos,
1209
- "'''",
1210
- error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
1211
- error_on_eof=True,
1212
- )
1213
- result = src[pos:end_pos]
1214
- pos = end_pos + 3
1215
904
  else:
1216
- delim = '"'
1217
- pos, result = toml_parse_basic_str(src, pos, multiline=True)
905
+ # Load options json var
906
+ options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
907
+ options = PyremoteBootstrapOptions(**json.loads(options_json_str))
1218
908
 
1219
- # Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
1220
- if not src.startswith(delim, pos):
1221
- return pos, result
1222
- pos += 1
1223
- if not src.startswith(delim, pos):
1224
- return pos, result + delim
1225
- pos += 1
1226
- return pos, result + (delim * 2)
909
+ # Read temp source file
910
+ with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
911
+ payload_src = sf.read()
1227
912
 
913
+ # Restore vars
914
+ sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
915
+ context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
1228
916
 
1229
- def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
1230
- if multiline:
1231
- error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1232
- parse_escapes = toml_parse_basic_str_escape_multiline
1233
- else:
1234
- error_on = TOML_ILLEGAL_BASIC_STR_CHARS
1235
- parse_escapes = toml_parse_basic_str_escape
1236
- result = ''
1237
- start_pos = pos
1238
- while True:
1239
- try:
1240
- char = src[pos]
1241
- except IndexError:
1242
- raise toml_suffixed_err(src, pos, 'Unterminated string') from None
1243
- if char == '"':
1244
- if not multiline:
1245
- return pos + 1, result + src[start_pos:pos]
1246
- if src.startswith('"""', pos):
1247
- return pos + 3, result + src[start_pos:pos]
1248
- pos += 1
1249
- continue
1250
- if char == '\\':
1251
- result += src[start_pos:pos]
1252
- pos, parsed_escape = parse_escapes(src, pos)
1253
- result += parsed_escape
1254
- start_pos = pos
1255
- continue
1256
- if char in error_on:
1257
- raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
1258
- pos += 1
917
+ # Write third ack
918
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
1259
919
 
920
+ # Write env info
921
+ env_info = _get_pyremote_env_info()
922
+ env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
923
+ os.write(1, struct.pack('<I', len(env_info_json)))
924
+ os.write(1, env_info_json.encode('utf-8'))
1260
925
 
1261
- def toml_parse_value( # noqa: C901
1262
- src: str,
1263
- pos: TomlPos,
1264
- parse_float: TomlParseFloat,
1265
- ) -> ta.Tuple[TomlPos, ta.Any]:
1266
- try:
1267
- char: ta.Optional[str] = src[pos]
1268
- except IndexError:
1269
- char = None
926
+ # Setup IO
927
+ input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
928
+ output = os.fdopen(os.dup(1), 'wb', 0) # noqa
929
+ os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
930
+ os.close(nfd)
1270
931
 
1271
- # IMPORTANT: order conditions based on speed of checking and likelihood
932
+ if (mn := options.main_name_override) is not None:
933
+ # Inspections like typing.get_type_hints need an entry in sys.modules.
934
+ sys.modules[mn] = sys.modules['__main__']
1272
935
 
1273
- # Basic strings
1274
- if char == '"':
1275
- if src.startswith('"""', pos):
1276
- return toml_parse_multiline_str(src, pos, literal=False)
1277
- return toml_parse_one_line_basic_str(src, pos)
936
+ # Write fourth ack
937
+ output.write(_PYREMOTE_BOOTSTRAP_ACK3)
1278
938
 
1279
- # Literal strings
1280
- if char == "'":
1281
- if src.startswith("'''", pos):
1282
- return toml_parse_multiline_str(src, pos, literal=True)
1283
- return toml_parse_literal_str(src, pos)
939
+ # Return
940
+ return PyremotePayloadRuntime(
941
+ input=input,
942
+ output=output,
943
+ context_name=context_name,
944
+ payload_src=payload_src,
945
+ options=options,
946
+ env_info=env_info,
947
+ )
1284
948
 
1285
- # Booleans
1286
- if char == 't':
1287
- if src.startswith('true', pos):
1288
- return pos + 4, True
1289
- if char == 'f':
1290
- if src.startswith('false', pos):
1291
- return pos + 5, False
1292
949
 
1293
- # Arrays
1294
- if char == '[':
1295
- return toml_parse_array(src, pos, parse_float)
950
+ ##
1296
951
 
1297
- # Inline tables
1298
- if char == '{':
1299
- return toml_parse_inline_table(src, pos, parse_float)
1300
952
 
1301
- # Dates and times
1302
- datetime_match = TOML_RE_DATETIME.match(src, pos)
1303
- if datetime_match:
1304
- try:
1305
- datetime_obj = toml_match_to_datetime(datetime_match)
1306
- except ValueError as e:
1307
- raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
1308
- return datetime_match.end(), datetime_obj
1309
- localtime_match = TOML_RE_LOCALTIME.match(src, pos)
1310
- if localtime_match:
1311
- return localtime_match.end(), toml_match_to_localtime(localtime_match)
953
+ class PyremoteBootstrapDriver:
954
+ def __init__(
955
+ self,
956
+ payload_src: ta.Union[str, ta.Sequence[str]],
957
+ options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
958
+ ) -> None:
959
+ super().__init__()
1312
960
 
1313
- # Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
1314
- # located after handling of dates and times.
1315
- number_match = TOML_RE_NUMBER.match(src, pos)
1316
- if number_match:
1317
- return number_match.end(), toml_match_to_number(number_match, parse_float)
961
+ self._payload_src = payload_src
962
+ self._options = options
1318
963
 
1319
- # Special floats
1320
- first_three = src[pos:pos + 3]
1321
- if first_three in {'inf', 'nan'}:
1322
- return pos + 3, parse_float(first_three)
1323
- first_four = src[pos:pos + 4]
1324
- if first_four in {'-inf', '+inf', '-nan', '+nan'}:
1325
- return pos + 4, parse_float(first_four)
964
+ self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
965
+ self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
1326
966
 
1327
- raise toml_suffixed_err(src, pos, 'Invalid value')
967
+ self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
968
+ #
1328
969
 
970
+ @classmethod
971
+ def _prepare_payload_src(
972
+ cls,
973
+ payload_src: ta.Union[str, ta.Sequence[str]],
974
+ options: PyremoteBootstrapOptions,
975
+ ) -> str:
976
+ parts: ta.List[str]
977
+ if isinstance(payload_src, str):
978
+ parts = [payload_src]
979
+ else:
980
+ parts = list(payload_src)
1329
981
 
1330
- def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
1331
- """Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
982
+ if (mn := options.main_name_override) is not None:
983
+ parts.insert(0, f'__name__ = {mn!r}')
1332
984
 
1333
- def coord_repr(src: str, pos: TomlPos) -> str:
1334
- if pos >= len(src):
1335
- return 'end of document'
1336
- line = src.count('\n', 0, pos) + 1
1337
- if line == 1:
1338
- column = pos + 1
985
+ if len(parts) == 1:
986
+ return parts[0]
1339
987
  else:
1340
- column = pos - src.rindex('\n', 0, pos)
1341
- return f'line {line}, column {column}'
988
+ return '\n\n'.join(parts)
1342
989
 
1343
- return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
990
+ #
1344
991
 
992
+ @dc.dataclass(frozen=True)
993
+ class Read:
994
+ sz: int
1345
995
 
1346
- def toml_is_unicode_scalar_value(codepoint: int) -> bool:
1347
- return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
996
+ @dc.dataclass(frozen=True)
997
+ class Write:
998
+ d: bytes
1348
999
 
1000
+ class ProtocolError(Exception):
1001
+ pass
1349
1002
 
1350
- def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
1351
- """A decorator to make `parse_float` safe.
1003
+ @dc.dataclass(frozen=True)
1004
+ class Result:
1005
+ pid: int
1006
+ env_info: PyremoteEnvInfo
1352
1007
 
1353
- `parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
1354
- thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
1355
- """
1356
- # The default `float` callable never returns illegal types. Optimize it.
1357
- if parse_float is float:
1358
- return float
1008
+ def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
1009
+ # Read first ack (after fork)
1010
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
1359
1011
 
1360
- def safe_parse_float(float_str: str) -> ta.Any:
1361
- float_value = parse_float(float_str)
1362
- if isinstance(float_value, (dict, list)):
1363
- raise ValueError('parse_float must not return dicts or lists') # noqa
1364
- return float_value
1012
+ # Read pid
1013
+ d = yield from self._read(8)
1014
+ pid = struct.unpack('<Q', d)[0]
1365
1015
 
1366
- return safe_parse_float
1016
+ # Write payload src
1017
+ yield from self._write(struct.pack('<I', len(self._payload_z)))
1018
+ yield from self._write(self._payload_z)
1367
1019
 
1020
+ # Read second ack (after writing src copies)
1021
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
1368
1022
 
1369
- ########################################
1370
- # ../config.py
1023
+ # Write options
1024
+ yield from self._write(struct.pack('<I', len(self._options_json)))
1025
+ yield from self._write(self._options_json)
1371
1026
 
1027
+ # Read third ack (after reaping child process)
1028
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
1372
1029
 
1373
- @dc.dataclass(frozen=True)
1374
- class MainConfig:
1375
- log_level: ta.Optional[str] = 'INFO'
1030
+ # Read env info
1031
+ d = yield from self._read(4)
1032
+ env_info_json_len = struct.unpack('<I', d)[0]
1033
+ d = yield from self._read(env_info_json_len)
1034
+ env_info_json = d.decode('utf-8')
1035
+ env_info = PyremoteEnvInfo(**json.loads(env_info_json))
1376
1036
 
1377
- debug: bool = False
1037
+ # Read fourth ack (after finalization completed)
1038
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
1378
1039
 
1040
+ # Return
1041
+ return self.Result(
1042
+ pid=pid,
1043
+ env_info=env_info,
1044
+ )
1379
1045
 
1380
- ########################################
1381
- # ../deploy/config.py
1046
+ def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
1047
+ d = yield self.Read(sz)
1048
+ if not isinstance(d, bytes):
1049
+ raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
1050
+ if len(d) != sz:
1051
+ raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
1052
+ return d
1382
1053
 
1054
+ def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
1055
+ d = yield from self._read(len(e))
1056
+ if d != e:
1057
+ raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
1383
1058
 
1384
- ##
1059
+ def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
1060
+ i = yield self.Write(d)
1061
+ if i is not None:
1062
+ raise self.ProtocolError('Unexpected input after write')
1385
1063
 
1064
+ #
1386
1065
 
1387
- @dc.dataclass(frozen=True)
1388
- class DeployConfig:
1389
- pass
1066
+ def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
1067
+ gen = self.gen()
1390
1068
 
1069
+ gi: ta.Optional[bytes] = None
1070
+ while True:
1071
+ try:
1072
+ if gi is not None:
1073
+ go = gen.send(gi)
1074
+ else:
1075
+ go = next(gen)
1076
+ except StopIteration as e:
1077
+ return e.value
1391
1078
 
1392
- ########################################
1393
- # ../deploy/paths/types.py
1079
+ if isinstance(go, self.Read):
1080
+ if len(gi := input.read(go.sz)) != go.sz:
1081
+ raise EOFError
1082
+ elif isinstance(go, self.Write):
1083
+ gi = None
1084
+ output.write(go.d)
1085
+ output.flush()
1086
+ else:
1087
+ raise TypeError(go)
1394
1088
 
1089
+ async def async_run(
1090
+ self,
1091
+ input: ta.Any, # asyncio.StreamWriter # noqa
1092
+ output: ta.Any, # asyncio.StreamReader
1093
+ ) -> Result:
1094
+ gen = self.gen()
1395
1095
 
1396
- ##
1096
+ gi: ta.Optional[bytes] = None
1097
+ while True:
1098
+ try:
1099
+ if gi is not None:
1100
+ go = gen.send(gi)
1101
+ else:
1102
+ go = next(gen)
1103
+ except StopIteration as e:
1104
+ return e.value
1105
+
1106
+ if isinstance(go, self.Read):
1107
+ if len(gi := await input.read(go.sz)) != go.sz:
1108
+ raise EOFError
1109
+ elif isinstance(go, self.Write):
1110
+ gi = None
1111
+ output.write(go.d)
1112
+ await output.drain()
1113
+ else:
1114
+ raise TypeError(go)
1397
1115
 
1398
1116
 
1399
1117
  ########################################
1400
- # ../deploy/types.py
1118
+ # ../../../omlish/asyncs/asyncio/channels.py
1401
1119
 
1402
1120
 
1403
- ##
1121
+ class AsyncioBytesChannelTransport(asyncio.Transport):
1122
+ def __init__(self, reader: asyncio.StreamReader) -> None:
1123
+ super().__init__()
1404
1124
 
1125
+ self.reader = reader
1126
+ self.closed: asyncio.Future = asyncio.Future()
1405
1127
 
1406
- DeployHome = ta.NewType('DeployHome', str)
1128
+ # @ta.override
1129
+ def write(self, data: bytes) -> None:
1130
+ self.reader.feed_data(data)
1407
1131
 
1408
- DeployRev = ta.NewType('DeployRev', str)
1132
+ # @ta.override
1133
+ def close(self) -> None:
1134
+ self.reader.feed_eof()
1135
+ if not self.closed.done():
1136
+ self.closed.set_result(True)
1409
1137
 
1138
+ # @ta.override
1139
+ def is_closing(self) -> bool:
1140
+ return self.closed.done()
1410
1141
 
1411
- ########################################
1412
- # ../../pyremote.py
1413
- """
1414
- Basically this: https://mitogen.networkgenomics.com/howitworks.html
1415
1142
 
1416
- TODO:
1417
- - log: ta.Optional[logging.Logger] = None + log.debug's
1418
- """
1143
+ def asyncio_create_bytes_channel(
1144
+ loop: ta.Any = None,
1145
+ ) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
1146
+ if loop is None:
1147
+ loop = asyncio.get_running_loop()
1419
1148
 
1149
+ reader = asyncio.StreamReader()
1150
+ protocol = asyncio.StreamReaderProtocol(reader)
1151
+ transport = AsyncioBytesChannelTransport(reader)
1152
+ writer = asyncio.StreamWriter(transport, protocol, reader, loop)
1420
1153
 
1421
- ##
1154
+ return reader, writer
1422
1155
 
1423
1156
 
1424
- @dc.dataclass(frozen=True)
1425
- class PyremoteBootstrapOptions:
1426
- debug: bool = False
1157
+ ########################################
1158
+ # ../../../omlish/asyncs/asyncio/streams.py
1427
1159
 
1428
- DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
1429
- main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
1430
1160
 
1161
+ ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
1431
1162
 
1432
- ##
1433
1163
 
1164
+ async def asyncio_open_stream_reader(
1165
+ f: ta.IO,
1166
+ loop: ta.Any = None,
1167
+ *,
1168
+ limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1169
+ ) -> asyncio.StreamReader:
1170
+ if loop is None:
1171
+ loop = asyncio.get_running_loop()
1434
1172
 
1435
- @dc.dataclass(frozen=True)
1436
- class PyremoteEnvInfo:
1437
- sys_base_prefix: str
1438
- sys_byteorder: str
1439
- sys_defaultencoding: str
1440
- sys_exec_prefix: str
1441
- sys_executable: str
1442
- sys_implementation_name: str
1443
- sys_path: ta.List[str]
1444
- sys_platform: str
1445
- sys_prefix: str
1446
- sys_version: str
1447
- sys_version_info: ta.List[ta.Union[int, str]]
1173
+ reader = asyncio.StreamReader(limit=limit, loop=loop)
1174
+ await loop.connect_read_pipe(
1175
+ lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1176
+ f,
1177
+ )
1448
1178
 
1449
- platform_architecture: ta.List[str]
1450
- platform_machine: str
1451
- platform_platform: str
1452
- platform_processor: str
1453
- platform_system: str
1454
- platform_release: str
1455
- platform_version: str
1179
+ return reader
1456
1180
 
1457
- site_userbase: str
1458
1181
 
1459
- os_cwd: str
1460
- os_gid: int
1461
- os_loadavg: ta.List[float]
1462
- os_login: ta.Optional[str]
1463
- os_pgrp: int
1464
- os_pid: int
1465
- os_ppid: int
1466
- os_uid: int
1182
+ async def asyncio_open_stream_writer(
1183
+ f: ta.IO,
1184
+ loop: ta.Any = None,
1185
+ ) -> asyncio.StreamWriter:
1186
+ if loop is None:
1187
+ loop = asyncio.get_running_loop()
1467
1188
 
1468
- pw_name: str
1469
- pw_uid: int
1470
- pw_gid: int
1471
- pw_gecos: str
1472
- pw_dir: str
1473
- pw_shell: str
1189
+ writer_transport, writer_protocol = await loop.connect_write_pipe(
1190
+ lambda: asyncio.streams.FlowControlMixin(loop=loop),
1191
+ f,
1192
+ )
1474
1193
 
1475
- env_path: ta.Optional[str]
1194
+ return asyncio.streams.StreamWriter(
1195
+ writer_transport,
1196
+ writer_protocol,
1197
+ None,
1198
+ loop,
1199
+ )
1476
1200
 
1477
1201
 
1478
- def _get_pyremote_env_info() -> PyremoteEnvInfo:
1479
- os_uid = os.getuid()
1202
+ ########################################
1203
+ # ../../../omlish/asyncs/asyncio/timeouts.py
1480
1204
 
1481
- pw = pwd.getpwuid(os_uid)
1482
1205
 
1483
- os_login: ta.Optional[str]
1484
- try:
1485
- os_login = os.getlogin()
1486
- except OSError:
1487
- os_login = None
1206
+ def asyncio_maybe_timeout(
1207
+ fut: AwaitableT,
1208
+ timeout: ta.Optional[float] = None,
1209
+ ) -> AwaitableT:
1210
+ if timeout is not None:
1211
+ fut = asyncio.wait_for(fut, timeout) # type: ignore
1212
+ return fut
1488
1213
 
1489
- return PyremoteEnvInfo(
1490
- sys_base_prefix=sys.base_prefix,
1491
- sys_byteorder=sys.byteorder,
1492
- sys_defaultencoding=sys.getdefaultencoding(),
1493
- sys_exec_prefix=sys.exec_prefix,
1494
- sys_executable=sys.executable,
1495
- sys_implementation_name=sys.implementation.name,
1496
- sys_path=sys.path,
1497
- sys_platform=sys.platform,
1498
- sys_prefix=sys.prefix,
1499
- sys_version=sys.version,
1500
- sys_version_info=list(sys.version_info),
1501
1214
 
1502
- platform_architecture=list(platform.architecture()),
1503
- platform_machine=platform.machine(),
1504
- platform_platform=platform.platform(),
1505
- platform_processor=platform.processor(),
1506
- platform_system=platform.system(),
1507
- platform_release=platform.release(),
1508
- platform_version=platform.version(),
1215
+ ########################################
1216
+ # ../../../omlish/formats/ini/sections.py
1509
1217
 
1510
- site_userbase=site.getuserbase(),
1511
1218
 
1512
- os_cwd=os.getcwd(),
1513
- os_gid=os.getgid(),
1514
- os_loadavg=list(os.getloadavg()),
1515
- os_login=os_login,
1516
- os_pgrp=os.getpgrp(),
1517
- os_pid=os.getpid(),
1518
- os_ppid=os.getppid(),
1519
- os_uid=os_uid,
1219
+ ##
1520
1220
 
1521
- pw_name=pw.pw_name,
1522
- pw_uid=pw.pw_uid,
1523
- pw_gid=pw.pw_gid,
1524
- pw_gecos=pw.pw_gecos,
1525
- pw_dir=pw.pw_dir,
1526
- pw_shell=pw.pw_shell,
1527
1221
 
1528
- env_path=os.environ.get('PATH'),
1529
- )
1222
+ def extract_ini_sections(cp: configparser.ConfigParser) -> IniSectionSettingsMap:
1223
+ config_dct: ta.Dict[str, ta.Any] = {}
1224
+ for sec in cp.sections():
1225
+ cd = config_dct
1226
+ for k in sec.split('.'):
1227
+ cd = cd.setdefault(k, {})
1228
+ cd.update(cp.items(sec))
1229
+ return config_dct
1530
1230
 
1531
1231
 
1532
1232
  ##
1533
1233
 
1534
1234
 
1535
- _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
1536
- _PYREMOTE_BOOTSTRAP_SRC_FD = 101
1537
-
1538
- _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
1539
- _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
1540
- _PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
1541
- _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
1542
- _PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
1543
-
1544
- _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
1545
- _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
1546
- _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
1547
- _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
1548
-
1549
- _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
1235
+ def render_ini_sections(
1236
+ settings_by_section: IniSectionSettingsMap,
1237
+ ) -> str:
1238
+ out = io.StringIO()
1550
1239
 
1551
- _PYREMOTE_BOOTSTRAP_IMPORTS = [
1552
- 'base64',
1553
- 'os',
1554
- 'struct',
1555
- 'sys',
1556
- 'zlib',
1557
- ]
1240
+ for i, (section, settings) in enumerate(settings_by_section.items()):
1241
+ if i:
1242
+ out.write('\n')
1558
1243
 
1244
+ out.write(f'[{section}]\n')
1559
1245
 
1560
- def _pyremote_bootstrap_main(context_name: str) -> None:
1561
- # Get pid
1562
- pid = os.getpid()
1246
+ for k, v in settings.items():
1247
+ if isinstance(v, str):
1248
+ out.write(f'{k}={v}\n')
1249
+ else:
1250
+ for vv in v:
1251
+ out.write(f'{k}={vv}\n')
1563
1252
 
1564
- # Two copies of payload src to be sent to parent
1565
- r0, w0 = os.pipe()
1566
- r1, w1 = os.pipe()
1253
+ return out.getvalue()
1567
1254
 
1568
- if (cp := os.fork()):
1569
- # Parent process
1570
1255
 
1571
- # Dup original stdin to comm_fd for use as comm channel
1572
- os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
1256
+ ########################################
1257
+ # ../../../omlish/formats/toml/parser.py
1258
+ # SPDX-License-Identifier: MIT
1259
+ # SPDX-FileCopyrightText: 2021 Taneli Hukkinen
1260
+ # Licensed to PSF under a Contributor Agreement.
1261
+ #
1262
+ # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
1263
+ # --------------------------------------------
1264
+ #
1265
+ # 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
1266
+ # ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
1267
+ # documentation.
1268
+ #
1269
+ # 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
1270
+ # royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
1271
+ # works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
1272
+ # Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
1273
+ # 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
1274
+ # Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
1275
+ #
1276
+ # 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
1277
+ # wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
1278
+ # any such work a brief summary of the changes made to Python.
1279
+ #
1280
+ # 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
1281
+ # EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
1282
+ # OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
1283
+ # RIGHTS.
1284
+ #
1285
+ # 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
1286
+ # DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
1287
+ # ADVISED OF THE POSSIBILITY THEREOF.
1288
+ #
1289
+ # 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
1290
+ #
1291
+ # 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
1292
+ # venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
1293
+ # name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
1294
+ #
1295
+ # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
1296
+ # License Agreement.
1297
+ #
1298
+ # https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
1573
1299
 
1574
- # Overwrite stdin (fed to python repl) with first copy of src
1575
- os.dup2(r0, 0)
1576
1300
 
1577
- # Dup second copy of src to src_fd to recover after launch
1578
- os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
1301
+ ##
1579
1302
 
1580
- # Close remaining fd's
1581
- for f in [r0, w0, r1, w1]:
1582
- os.close(f)
1583
1303
 
1584
- # Save vars
1585
- env = os.environ
1586
- exe = sys.executable
1587
- env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
1588
- env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
1589
- env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
1304
+ _TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
1590
1305
 
1591
- # Start repl reading stdin from r0
1592
- os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
1306
+ TOML_RE_NUMBER = re.compile(
1307
+ r"""
1308
+ 0
1309
+ (?:
1310
+ x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
1311
+ |
1312
+ b[01](?:_?[01])* # bin
1313
+ |
1314
+ o[0-7](?:_?[0-7])* # oct
1315
+ )
1316
+ |
1317
+ [+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
1318
+ (?P<floatpart>
1319
+ (?:\.[0-9](?:_?[0-9])*)? # optional fractional part
1320
+ (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
1321
+ )
1322
+ """,
1323
+ flags=re.VERBOSE,
1324
+ )
1325
+ TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
1326
+ TOML_RE_DATETIME = re.compile(
1327
+ rf"""
1328
+ ([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
1329
+ (?:
1330
+ [Tt ]
1331
+ {_TOML_TIME_RE_STR}
1332
+ (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
1333
+ )?
1334
+ """,
1335
+ flags=re.VERBOSE,
1336
+ )
1593
1337
 
1594
- else:
1595
- # Child process
1596
1338
 
1597
- # Write first ack
1598
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
1339
+ def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
1340
+ """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
1599
1341
 
1600
- # Write pid
1601
- os.write(1, struct.pack('<Q', pid))
1342
+ Raises ValueError if the match does not correspond to a valid date or datetime.
1343
+ """
1344
+ (
1345
+ year_str,
1346
+ month_str,
1347
+ day_str,
1348
+ hour_str,
1349
+ minute_str,
1350
+ sec_str,
1351
+ micros_str,
1352
+ zulu_time,
1353
+ offset_sign_str,
1354
+ offset_hour_str,
1355
+ offset_minute_str,
1356
+ ) = match.groups()
1357
+ year, month, day = int(year_str), int(month_str), int(day_str)
1358
+ if hour_str is None:
1359
+ return datetime.date(year, month, day)
1360
+ hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
1361
+ micros = int(micros_str.ljust(6, '0')) if micros_str else 0
1362
+ if offset_sign_str:
1363
+ tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
1364
+ offset_hour_str, offset_minute_str, offset_sign_str,
1365
+ )
1366
+ elif zulu_time:
1367
+ tz = datetime.UTC
1368
+ else: # local date-time
1369
+ tz = None
1370
+ return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
1602
1371
 
1603
- # Read payload src from stdin
1604
- payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
1605
- if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
1606
- raise EOFError
1607
- payload_src = zlib.decompress(payload_z)
1608
1372
 
1609
- # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
1610
- # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
1611
- for w in [w0, w1]:
1612
- fp = os.fdopen(w, 'wb', 0)
1613
- fp.write(payload_src)
1614
- fp.close()
1373
+ @functools.lru_cache() # noqa
1374
+ def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
1375
+ sign = 1 if sign_str == '+' else -1
1376
+ return datetime.timezone(
1377
+ datetime.timedelta(
1378
+ hours=sign * int(hour_str),
1379
+ minutes=sign * int(minute_str),
1380
+ ),
1381
+ )
1615
1382
 
1616
- # Write second ack
1617
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
1618
1383
 
1619
- # Exit child
1620
- sys.exit(0)
1384
+ def toml_match_to_localtime(match: re.Match) -> datetime.time:
1385
+ hour_str, minute_str, sec_str, micros_str = match.groups()
1386
+ micros = int(micros_str.ljust(6, '0')) if micros_str else 0
1387
+ return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
1621
1388
 
1622
1389
 
1623
- ##
1390
+ def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
1391
+ if match.group('floatpart'):
1392
+ return parse_float(match.group())
1393
+ return int(match.group(), 0)
1624
1394
 
1625
1395
 
1626
- def pyremote_build_bootstrap_cmd(context_name: str) -> str:
1627
- if any(c in context_name for c in '\'"'):
1628
- raise NameError(context_name)
1396
+ TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
1629
1397
 
1630
- import inspect
1631
- import textwrap
1632
- bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
1398
+ # Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
1399
+ # functions.
1400
+ TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
1401
+ TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
1633
1402
 
1634
- for gl in [
1635
- '_PYREMOTE_BOOTSTRAP_INPUT_FD',
1636
- '_PYREMOTE_BOOTSTRAP_SRC_FD',
1403
+ TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
1404
+ TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1637
1405
 
1638
- '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
1639
- '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
1640
- '_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
1406
+ TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
1641
1407
 
1642
- '_PYREMOTE_BOOTSTRAP_ACK0',
1643
- '_PYREMOTE_BOOTSTRAP_ACK1',
1408
+ TOML_WS = frozenset(' \t')
1409
+ TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
1410
+ TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
1411
+ TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
1412
+ TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
1644
1413
 
1645
- '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
1646
- ]:
1647
- bs_src = bs_src.replace(gl, repr(globals()[gl]))
1414
+ TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
1415
+ {
1416
+ '\\b': '\u0008', # backspace
1417
+ '\\t': '\u0009', # tab
1418
+ '\\n': '\u000A', # linefeed
1419
+ '\\f': '\u000C', # form feed
1420
+ '\\r': '\u000D', # carriage return
1421
+ '\\"': '\u0022', # quote
1422
+ '\\\\': '\u005C', # backslash
1423
+ },
1424
+ )
1648
1425
 
1649
- bs_src = '\n'.join(
1650
- cl
1651
- for l in bs_src.splitlines()
1652
- if (cl := (l.split('#')[0]).rstrip())
1653
- if cl.strip()
1654
- )
1655
1426
 
1656
- bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
1657
- bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
1658
- if b'"' in bs_z85:
1659
- raise ValueError(bs_z85)
1427
+ class TomlDecodeError(ValueError):
1428
+ """An error raised if a document is not valid TOML."""
1660
1429
 
1661
- stmts = [
1662
- f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
1663
- f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
1664
- f'_pyremote_bootstrap_main("{context_name}")',
1665
- ]
1666
1430
 
1667
- cmd = '; '.join(stmts)
1668
- return cmd
1431
+ def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
1432
+ """Parse TOML from a binary file object."""
1433
+ b = fp.read()
1434
+ try:
1435
+ s = b.decode()
1436
+ except AttributeError:
1437
+ raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
1438
+ return toml_loads(s, parse_float=parse_float)
1669
1439
 
1670
1440
 
1671
- ##
1441
+ def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
1442
+ """Parse TOML from a string."""
1672
1443
 
1444
+ # The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
1445
+ try:
1446
+ src = s.replace('\r\n', '\n')
1447
+ except (AttributeError, TypeError):
1448
+ raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
1449
+ pos = 0
1450
+ out = TomlOutput(TomlNestedDict(), TomlFlags())
1451
+ header: TomlKey = ()
1452
+ parse_float = toml_make_safe_parse_float(parse_float)
1673
1453
 
1674
- @dc.dataclass(frozen=True)
1675
- class PyremotePayloadRuntime:
1676
- input: ta.BinaryIO
1677
- output: ta.BinaryIO
1678
- context_name: str
1679
- payload_src: str
1680
- options: PyremoteBootstrapOptions
1681
- env_info: PyremoteEnvInfo
1454
+ # Parse one statement at a time (typically means one line in TOML source)
1455
+ while True:
1456
+ # 1. Skip line leading whitespace
1457
+ pos = toml_skip_chars(src, pos, TOML_WS)
1682
1458
 
1459
+ # 2. Parse rules. Expect one of the following:
1460
+ # - end of file
1461
+ # - end of line
1462
+ # - comment
1463
+ # - key/value pair
1464
+ # - append dict to list (and move to its namespace)
1465
+ # - create dict (and move to its namespace)
1466
+ # Skip trailing whitespace when applicable.
1467
+ try:
1468
+ char = src[pos]
1469
+ except IndexError:
1470
+ break
1471
+ if char == '\n':
1472
+ pos += 1
1473
+ continue
1474
+ if char in TOML_KEY_INITIAL_CHARS:
1475
+ pos = toml_key_value_rule(src, pos, out, header, parse_float)
1476
+ pos = toml_skip_chars(src, pos, TOML_WS)
1477
+ elif char == '[':
1478
+ try:
1479
+ second_char: ta.Optional[str] = src[pos + 1]
1480
+ except IndexError:
1481
+ second_char = None
1482
+ out.flags.finalize_pending()
1483
+ if second_char == '[':
1484
+ pos, header = toml_create_list_rule(src, pos, out)
1485
+ else:
1486
+ pos, header = toml_create_dict_rule(src, pos, out)
1487
+ pos = toml_skip_chars(src, pos, TOML_WS)
1488
+ elif char != '#':
1489
+ raise toml_suffixed_err(src, pos, 'Invalid statement')
1683
1490
 
1684
- def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
1685
- # If src file var is not present we need to do initial finalization
1686
- if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
1687
- # Read second copy of payload src
1688
- r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
1689
- payload_src = r1.read().decode('utf-8')
1690
- r1.close()
1491
+ # 3. Skip comment
1492
+ pos = toml_skip_comment(src, pos)
1691
1493
 
1692
- # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
1693
- # a pipe at once.
1694
- os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
1494
+ # 4. Expect end of line or end of file
1495
+ try:
1496
+ char = src[pos]
1497
+ except IndexError:
1498
+ break
1499
+ if char != '\n':
1500
+ raise toml_suffixed_err(
1501
+ src, pos, 'Expected newline or end of document after a statement',
1502
+ )
1503
+ pos += 1
1695
1504
 
1696
- # Read options
1697
- options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
1698
- if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
1699
- raise EOFError
1700
- options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
1505
+ return out.data.dict
1701
1506
 
1702
- # If debugging, re-exec as file
1703
- if options.debug:
1704
- # Write temp source file
1705
- import tempfile
1706
- tfd, tfn = tempfile.mkstemp('-pyremote.py')
1707
- os.write(tfd, payload_src.encode('utf-8'))
1708
- os.close(tfd)
1709
1507
 
1710
- # Set vars
1711
- os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
1712
- os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
1508
+ class TomlFlags:
1509
+ """Flags that map to parsed keys/namespaces."""
1713
1510
 
1714
- # Re-exec temp file
1715
- exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
1716
- context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
1717
- os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
1511
+ # Marks an immutable namespace (inline array or inline table).
1512
+ FROZEN = 0
1513
+ # Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
1514
+ EXPLICIT_NEST = 1
1718
1515
 
1719
- else:
1720
- # Load options json var
1721
- options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
1722
- options = PyremoteBootstrapOptions(**json.loads(options_json_str))
1516
+ def __init__(self) -> None:
1517
+ self._flags: ta.Dict[str, dict] = {}
1518
+ self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
1723
1519
 
1724
- # Read temp source file
1725
- with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
1726
- payload_src = sf.read()
1520
+ def add_pending(self, key: TomlKey, flag: int) -> None:
1521
+ self._pending_flags.add((key, flag))
1727
1522
 
1728
- # Restore vars
1729
- sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
1730
- context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
1523
+ def finalize_pending(self) -> None:
1524
+ for key, flag in self._pending_flags:
1525
+ self.set(key, flag, recursive=False)
1526
+ self._pending_flags.clear()
1731
1527
 
1732
- # Write third ack
1733
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
1528
+ def unset_all(self, key: TomlKey) -> None:
1529
+ cont = self._flags
1530
+ for k in key[:-1]:
1531
+ if k not in cont:
1532
+ return
1533
+ cont = cont[k]['nested']
1534
+ cont.pop(key[-1], None)
1734
1535
 
1735
- # Write env info
1736
- env_info = _get_pyremote_env_info()
1737
- env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
1738
- os.write(1, struct.pack('<I', len(env_info_json)))
1739
- os.write(1, env_info_json.encode('utf-8'))
1536
+ def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
1537
+ cont = self._flags
1538
+ key_parent, key_stem = key[:-1], key[-1]
1539
+ for k in key_parent:
1540
+ if k not in cont:
1541
+ cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
1542
+ cont = cont[k]['nested']
1543
+ if key_stem not in cont:
1544
+ cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
1545
+ cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
1740
1546
 
1741
- # Setup IO
1742
- input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
1743
- output = os.fdopen(os.dup(1), 'wb', 0) # noqa
1744
- os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
1745
- os.close(nfd)
1547
+ def is_(self, key: TomlKey, flag: int) -> bool:
1548
+ if not key:
1549
+ return False # document root has no flags
1550
+ cont = self._flags
1551
+ for k in key[:-1]:
1552
+ if k not in cont:
1553
+ return False
1554
+ inner_cont = cont[k]
1555
+ if flag in inner_cont['recursive_flags']:
1556
+ return True
1557
+ cont = inner_cont['nested']
1558
+ key_stem = key[-1]
1559
+ if key_stem in cont:
1560
+ cont = cont[key_stem]
1561
+ return flag in cont['flags'] or flag in cont['recursive_flags']
1562
+ return False
1746
1563
 
1747
- if (mn := options.main_name_override) is not None:
1748
- # Inspections like typing.get_type_hints need an entry in sys.modules.
1749
- sys.modules[mn] = sys.modules['__main__']
1750
1564
 
1751
- # Write fourth ack
1752
- output.write(_PYREMOTE_BOOTSTRAP_ACK3)
1565
+ class TomlNestedDict:
1566
+ def __init__(self) -> None:
1567
+ # The parsed content of the TOML document
1568
+ self.dict: ta.Dict[str, ta.Any] = {}
1753
1569
 
1754
- # Return
1755
- return PyremotePayloadRuntime(
1756
- input=input,
1757
- output=output,
1758
- context_name=context_name,
1759
- payload_src=payload_src,
1760
- options=options,
1761
- env_info=env_info,
1762
- )
1570
+ def get_or_create_nest(
1571
+ self,
1572
+ key: TomlKey,
1573
+ *,
1574
+ access_lists: bool = True,
1575
+ ) -> dict:
1576
+ cont: ta.Any = self.dict
1577
+ for k in key:
1578
+ if k not in cont:
1579
+ cont[k] = {}
1580
+ cont = cont[k]
1581
+ if access_lists and isinstance(cont, list):
1582
+ cont = cont[-1]
1583
+ if not isinstance(cont, dict):
1584
+ raise KeyError('There is no nest behind this key')
1585
+ return cont
1763
1586
 
1587
+ def append_nest_to_list(self, key: TomlKey) -> None:
1588
+ cont = self.get_or_create_nest(key[:-1])
1589
+ last_key = key[-1]
1590
+ if last_key in cont:
1591
+ list_ = cont[last_key]
1592
+ if not isinstance(list_, list):
1593
+ raise KeyError('An object other than list found behind this key')
1594
+ list_.append({})
1595
+ else:
1596
+ cont[last_key] = [{}]
1764
1597
 
1765
- ##
1766
1598
 
1599
+ class TomlOutput(ta.NamedTuple):
1600
+ data: TomlNestedDict
1601
+ flags: TomlFlags
1767
1602
 
1768
- class PyremoteBootstrapDriver:
1769
- def __init__(
1770
- self,
1771
- payload_src: ta.Union[str, ta.Sequence[str]],
1772
- options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
1773
- ) -> None:
1774
- super().__init__()
1775
1603
 
1776
- self._payload_src = payload_src
1777
- self._options = options
1604
+ def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
1605
+ try:
1606
+ while src[pos] in chars:
1607
+ pos += 1
1608
+ except IndexError:
1609
+ pass
1610
+ return pos
1778
1611
 
1779
- self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
1780
- self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
1781
1612
 
1782
- self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
1783
- #
1613
+ def toml_skip_until(
1614
+ src: str,
1615
+ pos: TomlPos,
1616
+ expect: str,
1617
+ *,
1618
+ error_on: ta.FrozenSet[str],
1619
+ error_on_eof: bool,
1620
+ ) -> TomlPos:
1621
+ try:
1622
+ new_pos = src.index(expect, pos)
1623
+ except ValueError:
1624
+ new_pos = len(src)
1625
+ if error_on_eof:
1626
+ raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
1784
1627
 
1785
- @classmethod
1786
- def _prepare_payload_src(
1787
- cls,
1788
- payload_src: ta.Union[str, ta.Sequence[str]],
1789
- options: PyremoteBootstrapOptions,
1790
- ) -> str:
1791
- parts: ta.List[str]
1792
- if isinstance(payload_src, str):
1793
- parts = [payload_src]
1794
- else:
1795
- parts = list(payload_src)
1628
+ if not error_on.isdisjoint(src[pos:new_pos]):
1629
+ while src[pos] not in error_on:
1630
+ pos += 1
1631
+ raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
1632
+ return new_pos
1796
1633
 
1797
- if (mn := options.main_name_override) is not None:
1798
- parts.insert(0, f'__name__ = {mn!r}')
1799
1634
 
1800
- if len(parts) == 1:
1801
- return parts[0]
1802
- else:
1803
- return '\n\n'.join(parts)
1635
+ def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
1636
+ try:
1637
+ char: ta.Optional[str] = src[pos]
1638
+ except IndexError:
1639
+ char = None
1640
+ if char == '#':
1641
+ return toml_skip_until(
1642
+ src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
1643
+ )
1644
+ return pos
1804
1645
 
1805
- #
1806
1646
 
1807
- @dc.dataclass(frozen=True)
1808
- class Read:
1809
- sz: int
1647
+ def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
1648
+ while True:
1649
+ pos_before_skip = pos
1650
+ pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1651
+ pos = toml_skip_comment(src, pos)
1652
+ if pos == pos_before_skip:
1653
+ return pos
1810
1654
 
1811
- @dc.dataclass(frozen=True)
1812
- class Write:
1813
- d: bytes
1814
1655
 
1815
- class ProtocolError(Exception):
1816
- pass
1656
+ def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
1657
+ pos += 1 # Skip "["
1658
+ pos = toml_skip_chars(src, pos, TOML_WS)
1659
+ pos, key = toml_parse_key(src, pos)
1817
1660
 
1818
- @dc.dataclass(frozen=True)
1819
- class Result:
1820
- pid: int
1821
- env_info: PyremoteEnvInfo
1661
+ if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
1662
+ raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
1663
+ out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
1664
+ try:
1665
+ out.data.get_or_create_nest(key)
1666
+ except KeyError:
1667
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1822
1668
 
1823
- def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
1824
- # Read first ack (after fork)
1825
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
1669
+ if not src.startswith(']', pos):
1670
+ raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
1671
+ return pos + 1, key
1826
1672
 
1827
- # Read pid
1828
- d = yield from self._read(8)
1829
- pid = struct.unpack('<Q', d)[0]
1830
1673
 
1831
- # Write payload src
1832
- yield from self._write(struct.pack('<I', len(self._payload_z)))
1833
- yield from self._write(self._payload_z)
1674
+ def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
1675
+ pos += 2 # Skip "[["
1676
+ pos = toml_skip_chars(src, pos, TOML_WS)
1677
+ pos, key = toml_parse_key(src, pos)
1834
1678
 
1835
- # Read second ack (after writing src copies)
1836
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
1679
+ if out.flags.is_(key, TomlFlags.FROZEN):
1680
+ raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1681
+ # Free the namespace now that it points to another empty list item...
1682
+ out.flags.unset_all(key)
1683
+ # ...but this key precisely is still prohibited from table declaration
1684
+ out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
1685
+ try:
1686
+ out.data.append_nest_to_list(key)
1687
+ except KeyError:
1688
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1837
1689
 
1838
- # Write options
1839
- yield from self._write(struct.pack('<I', len(self._options_json)))
1840
- yield from self._write(self._options_json)
1690
+ if not src.startswith(']]', pos):
1691
+ raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
1692
+ return pos + 2, key
1841
1693
 
1842
- # Read third ack (after reaping child process)
1843
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
1844
1694
 
1845
- # Read env info
1846
- d = yield from self._read(4)
1847
- env_info_json_len = struct.unpack('<I', d)[0]
1848
- d = yield from self._read(env_info_json_len)
1849
- env_info_json = d.decode('utf-8')
1850
- env_info = PyremoteEnvInfo(**json.loads(env_info_json))
1695
+ def toml_key_value_rule(
1696
+ src: str,
1697
+ pos: TomlPos,
1698
+ out: TomlOutput,
1699
+ header: TomlKey,
1700
+ parse_float: TomlParseFloat,
1701
+ ) -> TomlPos:
1702
+ pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1703
+ key_parent, key_stem = key[:-1], key[-1]
1704
+ abs_key_parent = header + key_parent
1851
1705
 
1852
- # Read fourth ack (after finalization completed)
1853
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
1706
+ relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
1707
+ for cont_key in relative_path_cont_keys:
1708
+ # Check that dotted key syntax does not redefine an existing table
1709
+ if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
1710
+ raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
1711
+ # Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
1712
+ # table sections.
1713
+ out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
1854
1714
 
1855
- # Return
1856
- return self.Result(
1857
- pid=pid,
1858
- env_info=env_info,
1715
+ if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
1716
+ raise toml_suffixed_err(
1717
+ src,
1718
+ pos,
1719
+ f'Cannot mutate immutable namespace {abs_key_parent}',
1859
1720
  )
1860
1721
 
1861
- def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
1862
- d = yield self.Read(sz)
1863
- if not isinstance(d, bytes):
1864
- raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
1865
- if len(d) != sz:
1866
- raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
1867
- return d
1722
+ try:
1723
+ nest = out.data.get_or_create_nest(abs_key_parent)
1724
+ except KeyError:
1725
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1726
+ if key_stem in nest:
1727
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
1728
+ # Mark inline table and array namespaces recursively immutable
1729
+ if isinstance(value, (dict, list)):
1730
+ out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
1731
+ nest[key_stem] = value
1732
+ return pos
1868
1733
 
1869
- def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
1870
- d = yield from self._read(len(e))
1871
- if d != e:
1872
- raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
1873
1734
 
1874
- def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
1875
- i = yield self.Write(d)
1876
- if i is not None:
1877
- raise self.ProtocolError('Unexpected input after write')
1735
+ def toml_parse_key_value_pair(
1736
+ src: str,
1737
+ pos: TomlPos,
1738
+ parse_float: TomlParseFloat,
1739
+ ) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
1740
+ pos, key = toml_parse_key(src, pos)
1741
+ try:
1742
+ char: ta.Optional[str] = src[pos]
1743
+ except IndexError:
1744
+ char = None
1745
+ if char != '=':
1746
+ raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
1747
+ pos += 1
1748
+ pos = toml_skip_chars(src, pos, TOML_WS)
1749
+ pos, value = toml_parse_value(src, pos, parse_float)
1750
+ return pos, key, value
1878
1751
 
1879
- #
1880
1752
 
1881
- def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
1882
- gen = self.gen()
1753
+ def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
1754
+ pos, key_part = toml_parse_key_part(src, pos)
1755
+ key: TomlKey = (key_part,)
1756
+ pos = toml_skip_chars(src, pos, TOML_WS)
1757
+ while True:
1758
+ try:
1759
+ char: ta.Optional[str] = src[pos]
1760
+ except IndexError:
1761
+ char = None
1762
+ if char != '.':
1763
+ return pos, key
1764
+ pos += 1
1765
+ pos = toml_skip_chars(src, pos, TOML_WS)
1766
+ pos, key_part = toml_parse_key_part(src, pos)
1767
+ key += (key_part,)
1768
+ pos = toml_skip_chars(src, pos, TOML_WS)
1769
+
1770
+
1771
+ def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1772
+ try:
1773
+ char: ta.Optional[str] = src[pos]
1774
+ except IndexError:
1775
+ char = None
1776
+ if char in TOML_BARE_KEY_CHARS:
1777
+ start_pos = pos
1778
+ pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
1779
+ return pos, src[start_pos:pos]
1780
+ if char == "'":
1781
+ return toml_parse_literal_str(src, pos)
1782
+ if char == '"':
1783
+ return toml_parse_one_line_basic_str(src, pos)
1784
+ raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
1785
+
1786
+
1787
+ def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1788
+ pos += 1
1789
+ return toml_parse_basic_str(src, pos, multiline=False)
1790
+
1791
+
1792
+ def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
1793
+ pos += 1
1794
+ array: list = []
1795
+
1796
+ pos = toml_skip_comments_and_array_ws(src, pos)
1797
+ if src.startswith(']', pos):
1798
+ return pos + 1, array
1799
+ while True:
1800
+ pos, val = toml_parse_value(src, pos, parse_float)
1801
+ array.append(val)
1802
+ pos = toml_skip_comments_and_array_ws(src, pos)
1803
+
1804
+ c = src[pos:pos + 1]
1805
+ if c == ']':
1806
+ return pos + 1, array
1807
+ if c != ',':
1808
+ raise toml_suffixed_err(src, pos, 'Unclosed array')
1809
+ pos += 1
1810
+
1811
+ pos = toml_skip_comments_and_array_ws(src, pos)
1812
+ if src.startswith(']', pos):
1813
+ return pos + 1, array
1814
+
1815
+
1816
+ def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
1817
+ pos += 1
1818
+ nested_dict = TomlNestedDict()
1819
+ flags = TomlFlags()
1820
+
1821
+ pos = toml_skip_chars(src, pos, TOML_WS)
1822
+ if src.startswith('}', pos):
1823
+ return pos + 1, nested_dict.dict
1824
+ while True:
1825
+ pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1826
+ key_parent, key_stem = key[:-1], key[-1]
1827
+ if flags.is_(key, TomlFlags.FROZEN):
1828
+ raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1829
+ try:
1830
+ nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
1831
+ except KeyError:
1832
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1833
+ if key_stem in nest:
1834
+ raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
1835
+ nest[key_stem] = value
1836
+ pos = toml_skip_chars(src, pos, TOML_WS)
1837
+ c = src[pos:pos + 1]
1838
+ if c == '}':
1839
+ return pos + 1, nested_dict.dict
1840
+ if c != ',':
1841
+ raise toml_suffixed_err(src, pos, 'Unclosed inline table')
1842
+ if isinstance(value, (dict, list)):
1843
+ flags.set(key, TomlFlags.FROZEN, recursive=True)
1844
+ pos += 1
1845
+ pos = toml_skip_chars(src, pos, TOML_WS)
1846
+
1847
+
1848
+ def toml_parse_basic_str_escape(
1849
+ src: str,
1850
+ pos: TomlPos,
1851
+ *,
1852
+ multiline: bool = False,
1853
+ ) -> ta.Tuple[TomlPos, str]:
1854
+ escape_id = src[pos:pos + 2]
1855
+ pos += 2
1856
+ if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
1857
+ # Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
1858
+ # newline.
1859
+ if escape_id != '\\\n':
1860
+ pos = toml_skip_chars(src, pos, TOML_WS)
1861
+ try:
1862
+ char = src[pos]
1863
+ except IndexError:
1864
+ return pos, ''
1865
+ if char != '\n':
1866
+ raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
1867
+ pos += 1
1868
+ pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1869
+ return pos, ''
1870
+ if escape_id == '\\u':
1871
+ return toml_parse_hex_char(src, pos, 4)
1872
+ if escape_id == '\\U':
1873
+ return toml_parse_hex_char(src, pos, 8)
1874
+ try:
1875
+ return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
1876
+ except KeyError:
1877
+ raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
1878
+
1883
1879
 
1884
- gi: ta.Optional[bytes] = None
1885
- while True:
1886
- try:
1887
- if gi is not None:
1888
- go = gen.send(gi)
1889
- else:
1890
- go = next(gen)
1891
- except StopIteration as e:
1892
- return e.value
1880
+ def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1881
+ return toml_parse_basic_str_escape(src, pos, multiline=True)
1893
1882
 
1894
- if isinstance(go, self.Read):
1895
- if len(gi := input.read(go.sz)) != go.sz:
1896
- raise EOFError
1897
- elif isinstance(go, self.Write):
1898
- gi = None
1899
- output.write(go.d)
1900
- output.flush()
1901
- else:
1902
- raise TypeError(go)
1903
1883
 
1904
- async def async_run(
1905
- self,
1906
- input: ta.Any, # asyncio.StreamWriter # noqa
1907
- output: ta.Any, # asyncio.StreamReader
1908
- ) -> Result:
1909
- gen = self.gen()
1884
+ def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
1885
+ hex_str = src[pos:pos + hex_len]
1886
+ if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
1887
+ raise toml_suffixed_err(src, pos, 'Invalid hex value')
1888
+ pos += hex_len
1889
+ hex_int = int(hex_str, 16)
1890
+ if not toml_is_unicode_scalar_value(hex_int):
1891
+ raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
1892
+ return pos, chr(hex_int)
1910
1893
 
1911
- gi: ta.Optional[bytes] = None
1912
- while True:
1913
- try:
1914
- if gi is not None:
1915
- go = gen.send(gi)
1916
- else:
1917
- go = next(gen)
1918
- except StopIteration as e:
1919
- return e.value
1920
1894
 
1921
- if isinstance(go, self.Read):
1922
- if len(gi := await input.read(go.sz)) != go.sz:
1923
- raise EOFError
1924
- elif isinstance(go, self.Write):
1925
- gi = None
1926
- output.write(go.d)
1927
- await output.drain()
1928
- else:
1929
- raise TypeError(go)
1895
+ def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1896
+ pos += 1 # Skip starting apostrophe
1897
+ start_pos = pos
1898
+ pos = toml_skip_until(
1899
+ src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
1900
+ )
1901
+ return pos + 1, src[start_pos:pos] # Skip ending apostrophe
1930
1902
 
1931
1903
 
1932
- ########################################
1933
- # ../../../omlish/asyncs/asyncio/channels.py
1904
+ def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
1905
+ pos += 3
1906
+ if src.startswith('\n', pos):
1907
+ pos += 1
1934
1908
 
1909
+ if literal:
1910
+ delim = "'"
1911
+ end_pos = toml_skip_until(
1912
+ src,
1913
+ pos,
1914
+ "'''",
1915
+ error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
1916
+ error_on_eof=True,
1917
+ )
1918
+ result = src[pos:end_pos]
1919
+ pos = end_pos + 3
1920
+ else:
1921
+ delim = '"'
1922
+ pos, result = toml_parse_basic_str(src, pos, multiline=True)
1935
1923
 
1936
- class AsyncioBytesChannelTransport(asyncio.Transport):
1937
- def __init__(self, reader: asyncio.StreamReader) -> None:
1938
- super().__init__()
1924
+ # Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
1925
+ if not src.startswith(delim, pos):
1926
+ return pos, result
1927
+ pos += 1
1928
+ if not src.startswith(delim, pos):
1929
+ return pos, result + delim
1930
+ pos += 1
1931
+ return pos, result + (delim * 2)
1939
1932
 
1940
- self.reader = reader
1941
- self.closed: asyncio.Future = asyncio.Future()
1942
1933
 
1943
- # @ta.override
1944
- def write(self, data: bytes) -> None:
1945
- self.reader.feed_data(data)
1934
+ def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
1935
+ if multiline:
1936
+ error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1937
+ parse_escapes = toml_parse_basic_str_escape_multiline
1938
+ else:
1939
+ error_on = TOML_ILLEGAL_BASIC_STR_CHARS
1940
+ parse_escapes = toml_parse_basic_str_escape
1941
+ result = ''
1942
+ start_pos = pos
1943
+ while True:
1944
+ try:
1945
+ char = src[pos]
1946
+ except IndexError:
1947
+ raise toml_suffixed_err(src, pos, 'Unterminated string') from None
1948
+ if char == '"':
1949
+ if not multiline:
1950
+ return pos + 1, result + src[start_pos:pos]
1951
+ if src.startswith('"""', pos):
1952
+ return pos + 3, result + src[start_pos:pos]
1953
+ pos += 1
1954
+ continue
1955
+ if char == '\\':
1956
+ result += src[start_pos:pos]
1957
+ pos, parsed_escape = parse_escapes(src, pos)
1958
+ result += parsed_escape
1959
+ start_pos = pos
1960
+ continue
1961
+ if char in error_on:
1962
+ raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
1963
+ pos += 1
1946
1964
 
1947
- # @ta.override
1948
- def close(self) -> None:
1949
- self.reader.feed_eof()
1950
- if not self.closed.done():
1951
- self.closed.set_result(True)
1952
1965
 
1953
- # @ta.override
1954
- def is_closing(self) -> bool:
1955
- return self.closed.done()
1966
+ def toml_parse_value( # noqa: C901
1967
+ src: str,
1968
+ pos: TomlPos,
1969
+ parse_float: TomlParseFloat,
1970
+ ) -> ta.Tuple[TomlPos, ta.Any]:
1971
+ try:
1972
+ char: ta.Optional[str] = src[pos]
1973
+ except IndexError:
1974
+ char = None
1956
1975
 
1976
+ # IMPORTANT: order conditions based on speed of checking and likelihood
1957
1977
 
1958
- def asyncio_create_bytes_channel(
1959
- loop: ta.Any = None,
1960
- ) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
1961
- if loop is None:
1962
- loop = asyncio.get_running_loop()
1978
+ # Basic strings
1979
+ if char == '"':
1980
+ if src.startswith('"""', pos):
1981
+ return toml_parse_multiline_str(src, pos, literal=False)
1982
+ return toml_parse_one_line_basic_str(src, pos)
1963
1983
 
1964
- reader = asyncio.StreamReader()
1965
- protocol = asyncio.StreamReaderProtocol(reader)
1966
- transport = AsyncioBytesChannelTransport(reader)
1967
- writer = asyncio.StreamWriter(transport, protocol, reader, loop)
1984
+ # Literal strings
1985
+ if char == "'":
1986
+ if src.startswith("'''", pos):
1987
+ return toml_parse_multiline_str(src, pos, literal=True)
1988
+ return toml_parse_literal_str(src, pos)
1968
1989
 
1969
- return reader, writer
1990
+ # Booleans
1991
+ if char == 't':
1992
+ if src.startswith('true', pos):
1993
+ return pos + 4, True
1994
+ if char == 'f':
1995
+ if src.startswith('false', pos):
1996
+ return pos + 5, False
1970
1997
 
1998
+ # Arrays
1999
+ if char == '[':
2000
+ return toml_parse_array(src, pos, parse_float)
1971
2001
 
1972
- ########################################
1973
- # ../../../omlish/asyncs/asyncio/streams.py
2002
+ # Inline tables
2003
+ if char == '{':
2004
+ return toml_parse_inline_table(src, pos, parse_float)
1974
2005
 
2006
+ # Dates and times
2007
+ datetime_match = TOML_RE_DATETIME.match(src, pos)
2008
+ if datetime_match:
2009
+ try:
2010
+ datetime_obj = toml_match_to_datetime(datetime_match)
2011
+ except ValueError as e:
2012
+ raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
2013
+ return datetime_match.end(), datetime_obj
2014
+ localtime_match = TOML_RE_LOCALTIME.match(src, pos)
2015
+ if localtime_match:
2016
+ return localtime_match.end(), toml_match_to_localtime(localtime_match)
1975
2017
 
1976
- ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
2018
+ # Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
2019
+ # located after handling of dates and times.
2020
+ number_match = TOML_RE_NUMBER.match(src, pos)
2021
+ if number_match:
2022
+ return number_match.end(), toml_match_to_number(number_match, parse_float)
1977
2023
 
2024
+ # Special floats
2025
+ first_three = src[pos:pos + 3]
2026
+ if first_three in {'inf', 'nan'}:
2027
+ return pos + 3, parse_float(first_three)
2028
+ first_four = src[pos:pos + 4]
2029
+ if first_four in {'-inf', '+inf', '-nan', '+nan'}:
2030
+ return pos + 4, parse_float(first_four)
1978
2031
 
1979
- async def asyncio_open_stream_reader(
1980
- f: ta.IO,
1981
- loop: ta.Any = None,
1982
- *,
1983
- limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1984
- ) -> asyncio.StreamReader:
1985
- if loop is None:
1986
- loop = asyncio.get_running_loop()
2032
+ raise toml_suffixed_err(src, pos, 'Invalid value')
1987
2033
 
1988
- reader = asyncio.StreamReader(limit=limit, loop=loop)
1989
- await loop.connect_read_pipe(
1990
- lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1991
- f,
1992
- )
1993
2034
 
1994
- return reader
2035
+ def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
2036
+ """Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
1995
2037
 
2038
+ def coord_repr(src: str, pos: TomlPos) -> str:
2039
+ if pos >= len(src):
2040
+ return 'end of document'
2041
+ line = src.count('\n', 0, pos) + 1
2042
+ if line == 1:
2043
+ column = pos + 1
2044
+ else:
2045
+ column = pos - src.rindex('\n', 0, pos)
2046
+ return f'line {line}, column {column}'
1996
2047
 
1997
- async def asyncio_open_stream_writer(
1998
- f: ta.IO,
1999
- loop: ta.Any = None,
2000
- ) -> asyncio.StreamWriter:
2001
- if loop is None:
2002
- loop = asyncio.get_running_loop()
2048
+ return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
2003
2049
 
2004
- writer_transport, writer_protocol = await loop.connect_write_pipe(
2005
- lambda: asyncio.streams.FlowControlMixin(loop=loop),
2006
- f,
2007
- )
2008
2050
 
2009
- return asyncio.streams.StreamWriter(
2010
- writer_transport,
2011
- writer_protocol,
2012
- None,
2013
- loop,
2014
- )
2051
+ def toml_is_unicode_scalar_value(codepoint: int) -> bool:
2052
+ return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
2015
2053
 
2016
2054
 
2017
- ########################################
2018
- # ../../../omlish/asyncs/asyncio/timeouts.py
2055
+ def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
2056
+ """A decorator to make `parse_float` safe.
2057
+
2058
+ `parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
2059
+ thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
2060
+ """
2061
+ # The default `float` callable never returns illegal types. Optimize it.
2062
+ if parse_float is float:
2063
+ return float
2019
2064
 
2065
+ def safe_parse_float(float_str: str) -> ta.Any:
2066
+ float_value = parse_float(float_str)
2067
+ if isinstance(float_value, (dict, list)):
2068
+ raise ValueError('parse_float must not return dicts or lists') # noqa
2069
+ return float_value
2020
2070
 
2021
- def asyncio_maybe_timeout(
2022
- fut: AwaitableT,
2023
- timeout: ta.Optional[float] = None,
2024
- ) -> AwaitableT:
2025
- if timeout is not None:
2026
- fut = asyncio.wait_for(fut, timeout) # type: ignore
2027
- return fut
2071
+ return safe_parse_float
2028
2072
 
2029
2073
 
2030
2074
  ########################################
@@ -6986,30 +7030,6 @@ def build_config_named_children(
6986
7030
  return lst
6987
7031
 
6988
7032
 
6989
- ##
6990
-
6991
-
6992
- def render_ini_config(
6993
- settings_by_section: IniConfigSectionSettingsMap,
6994
- ) -> str:
6995
- out = io.StringIO()
6996
-
6997
- for i, (section, settings) in enumerate(settings_by_section.items()):
6998
- if i:
6999
- out.write('\n')
7000
-
7001
- out.write(f'[{section}]\n')
7002
-
7003
- for k, v in settings.items():
7004
- if isinstance(v, str):
7005
- out.write(f'{k}={v}\n')
7006
- else:
7007
- for vv in v:
7008
- out.write(f'{k}={vv}\n')
7009
-
7010
- return out.getvalue()
7011
-
7012
-
7013
7033
  ########################################
7014
7034
  # ../commands/marshal.py
7015
7035
 
@@ -7063,6 +7083,104 @@ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
7063
7083
  CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
7064
7084
 
7065
7085
 
7086
+ ########################################
7087
+ # ../deploy/conf/specs.py
7088
+
7089
+
7090
+ ##
7091
+
7092
+
7093
+ class DeployAppConfContent(abc.ABC): # noqa
7094
+ pass
7095
+
7096
+
7097
+ #
7098
+
7099
+
7100
+ @register_single_field_type_obj_marshaler('body')
7101
+ @dc.dataclass(frozen=True)
7102
+ class RawDeployAppConfContent(DeployAppConfContent):
7103
+ body: str
7104
+
7105
+
7106
+ #
7107
+
7108
+
7109
+ @register_single_field_type_obj_marshaler('obj')
7110
+ @dc.dataclass(frozen=True)
7111
+ class JsonDeployAppConfContent(DeployAppConfContent):
7112
+ obj: ta.Any
7113
+
7114
+
7115
+ #
7116
+
7117
+
7118
+ @register_single_field_type_obj_marshaler('sections')
7119
+ @dc.dataclass(frozen=True)
7120
+ class IniDeployAppConfContent(DeployAppConfContent):
7121
+ sections: IniSectionSettingsMap
7122
+
7123
+
7124
+ #
7125
+
7126
+
7127
+ @register_single_field_type_obj_marshaler('items')
7128
+ @dc.dataclass(frozen=True)
7129
+ class NginxDeployAppConfContent(DeployAppConfContent):
7130
+ items: ta.Any
7131
+
7132
+
7133
+ ##
7134
+
7135
+
7136
+ @dc.dataclass(frozen=True)
7137
+ class DeployAppConfFile:
7138
+ path: str
7139
+ content: DeployAppConfContent
7140
+
7141
+ def __post_init__(self) -> None:
7142
+ check_valid_deploy_spec_path(self.path)
7143
+
7144
+
7145
+ ##
7146
+
7147
+
7148
+ @dc.dataclass(frozen=True)
7149
+ class DeployAppConfLink: # noqa
7150
+ """
7151
+ May be either:
7152
+ - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
7153
+ - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
7154
+ - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
7155
+ """
7156
+
7157
+ src: str
7158
+
7159
+ kind: ta.Literal['current_only', 'all_active'] = 'current_only'
7160
+
7161
+ def __post_init__(self) -> None:
7162
+ check_valid_deploy_spec_path(self.src)
7163
+ if '/' in self.src:
7164
+ check.equal(self.src.count('/'), 1)
7165
+
7166
+
7167
+ ##
7168
+
7169
+
7170
+ @dc.dataclass(frozen=True)
7171
+ class DeployAppConfSpec:
7172
+ files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
7173
+
7174
+ links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
7175
+
7176
+ def __post_init__(self) -> None:
7177
+ if self.files:
7178
+ seen: ta.Set[str] = set()
7179
+ for f in self.files:
7180
+ check.not_in(f.path, seen)
7181
+ seen.add(f.path)
7182
+
7183
+
7066
7184
  ########################################
7067
7185
  # ../deploy/tags.py
7068
7186
 
@@ -7978,112 +8096,14 @@ class LocalCommandExecutor(CommandExecutor):
7978
8096
  self,
7979
8097
  *,
7980
8098
  command_executors: CommandExecutorMap,
7981
- ) -> None:
7982
- super().__init__()
7983
-
7984
- self._command_executors = command_executors
7985
-
7986
- async def execute(self, cmd: Command) -> Command.Output:
7987
- ce: CommandExecutor = self._command_executors[type(cmd)]
7988
- return await ce.execute(cmd)
7989
-
7990
-
7991
- ########################################
7992
- # ../deploy/conf/specs.py
7993
-
7994
-
7995
- ##
7996
-
7997
-
7998
- class DeployAppConfContent(abc.ABC): # noqa
7999
- pass
8000
-
8001
-
8002
- #
8003
-
8004
-
8005
- @register_single_field_type_obj_marshaler('body')
8006
- @dc.dataclass(frozen=True)
8007
- class RawDeployAppConfContent(DeployAppConfContent):
8008
- body: str
8009
-
8010
-
8011
- #
8012
-
8013
-
8014
- @register_single_field_type_obj_marshaler('obj')
8015
- @dc.dataclass(frozen=True)
8016
- class JsonDeployAppConfContent(DeployAppConfContent):
8017
- obj: ta.Any
8018
-
8019
-
8020
- #
8021
-
8022
-
8023
- @register_single_field_type_obj_marshaler('sections')
8024
- @dc.dataclass(frozen=True)
8025
- class IniDeployAppConfContent(DeployAppConfContent):
8026
- sections: IniConfigSectionSettingsMap
8027
-
8028
-
8029
- #
8030
-
8031
-
8032
- @register_single_field_type_obj_marshaler('items')
8033
- @dc.dataclass(frozen=True)
8034
- class NginxDeployAppConfContent(DeployAppConfContent):
8035
- items: ta.Any
8036
-
8037
-
8038
- ##
8039
-
8040
-
8041
- @dc.dataclass(frozen=True)
8042
- class DeployAppConfFile:
8043
- path: str
8044
- content: DeployAppConfContent
8045
-
8046
- def __post_init__(self) -> None:
8047
- check_valid_deploy_spec_path(self.path)
8048
-
8049
-
8050
- ##
8051
-
8052
-
8053
- @dc.dataclass(frozen=True)
8054
- class DeployAppConfLink: # noqa
8055
- """
8056
- May be either:
8057
- - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
8058
- - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
8059
- - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
8060
- """
8061
-
8062
- src: str
8063
-
8064
- kind: ta.Literal['current_only', 'all_active'] = 'current_only'
8065
-
8066
- def __post_init__(self) -> None:
8067
- check_valid_deploy_spec_path(self.src)
8068
- if '/' in self.src:
8069
- check.equal(self.src.count('/'), 1)
8070
-
8071
-
8072
- ##
8073
-
8074
-
8075
- @dc.dataclass(frozen=True)
8076
- class DeployAppConfSpec:
8077
- files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
8078
-
8079
- links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
8080
-
8081
- def __post_init__(self) -> None:
8082
- if self.files:
8083
- seen: ta.Set[str] = set()
8084
- for f in self.files:
8085
- check.not_in(f.path, seen)
8086
- seen.add(f.path)
8099
+ ) -> None:
8100
+ super().__init__()
8101
+
8102
+ self._command_executors = command_executors
8103
+
8104
+ async def execute(self, cmd: Command) -> Command.Output:
8105
+ ce: CommandExecutor = self._command_executors[type(cmd)]
8106
+ return await ce.execute(cmd)
8087
8107
 
8088
8108
 
8089
8109
  ########################################
@@ -8288,6 +8308,127 @@ class DeployPath(DeployPathRenderable):
8288
8308
  return cls(tuple(DeployPathPart.parse(p) for p in ps))
8289
8309
 
8290
8310
 
8311
+ ########################################
8312
+ # ../deploy/specs.py
8313
+
8314
+
8315
+ ##
8316
+
8317
+
8318
+ class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
8319
+ @cached_nullary
8320
+ def _key_str(self) -> str:
8321
+ return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
8322
+
8323
+ @abc.abstractmethod
8324
+ def key(self) -> KeyDeployTagT:
8325
+ raise NotImplementedError
8326
+
8327
+
8328
+ ##
8329
+
8330
+
8331
+ @dc.dataclass(frozen=True)
8332
+ class DeployGitRepo:
8333
+ host: ta.Optional[str] = None
8334
+ username: ta.Optional[str] = None
8335
+ path: ta.Optional[str] = None
8336
+
8337
+ def __post_init__(self) -> None:
8338
+ check.not_in('..', check.non_empty_str(self.host))
8339
+ check.not_in('.', check.non_empty_str(self.path))
8340
+
8341
+
8342
+ @dc.dataclass(frozen=True)
8343
+ class DeployGitSpec:
8344
+ repo: DeployGitRepo
8345
+ rev: DeployRev
8346
+
8347
+ subtrees: ta.Optional[ta.Sequence[str]] = None
8348
+
8349
+ def __post_init__(self) -> None:
8350
+ check.non_empty_str(self.rev)
8351
+ if self.subtrees is not None:
8352
+ for st in self.subtrees:
8353
+ check.non_empty_str(st)
8354
+
8355
+
8356
+ ##
8357
+
8358
+
8359
+ @dc.dataclass(frozen=True)
8360
+ class DeployVenvSpec:
8361
+ interp: ta.Optional[str] = None
8362
+
8363
+ requirements_files: ta.Optional[ta.Sequence[str]] = None
8364
+ extra_dependencies: ta.Optional[ta.Sequence[str]] = None
8365
+
8366
+ use_uv: bool = False
8367
+
8368
+
8369
+ ##
8370
+
8371
+
8372
+ @dc.dataclass(frozen=True)
8373
+ class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
8374
+ app: DeployApp
8375
+
8376
+ git: DeployGitSpec
8377
+
8378
+ venv: ta.Optional[DeployVenvSpec] = None
8379
+
8380
+ conf: ta.Optional[DeployAppConfSpec] = None
8381
+
8382
+ # @ta.override
8383
+ def key(self) -> DeployAppKey:
8384
+ return DeployAppKey(self._key_str())
8385
+
8386
+
8387
+ @dc.dataclass(frozen=True)
8388
+ class DeployAppLinksSpec:
8389
+ apps: ta.Sequence[DeployApp] = ()
8390
+
8391
+ removed_apps: ta.Sequence[DeployApp] = ()
8392
+
8393
+ exclude_unspecified: bool = False
8394
+
8395
+
8396
+ ##
8397
+
8398
+
8399
+ @dc.dataclass(frozen=True)
8400
+ class DeploySystemdSpec:
8401
+ # ~/.config/systemd/user/
8402
+ unit_dir: ta.Optional[str] = None
8403
+
8404
+
8405
+ ##
8406
+
8407
+
8408
+ @dc.dataclass(frozen=True)
8409
+ class DeploySpec(DeploySpecKeyed[DeployKey]):
8410
+ home: DeployHome
8411
+
8412
+ apps: ta.Sequence[DeployAppSpec] = ()
8413
+
8414
+ app_links: DeployAppLinksSpec = DeployAppLinksSpec()
8415
+
8416
+ systemd: ta.Optional[DeploySystemdSpec] = None
8417
+
8418
+ def __post_init__(self) -> None:
8419
+ check.non_empty_str(self.home)
8420
+
8421
+ seen: ta.Set[DeployApp] = set()
8422
+ for a in self.apps:
8423
+ if a.app in seen:
8424
+ raise KeyError(a.app)
8425
+ seen.add(a.app)
8426
+
8427
+ # @ta.override
8428
+ def key(self) -> DeployKey:
8429
+ return DeployKey(self._key_str())
8430
+
8431
+
8291
8432
  ########################################
8292
8433
  # ../remote/execution.py
8293
8434
  """
@@ -9320,7 +9461,7 @@ class DeployConfManager:
9320
9461
 
9321
9462
  elif isinstance(ac, IniDeployAppConfContent):
9322
9463
  ini_sections = pcc(ac.sections)
9323
- return strip_with_newline(render_ini_config(ini_sections))
9464
+ return strip_with_newline(render_ini_sections(ini_sections))
9324
9465
 
9325
9466
  elif isinstance(ac, NginxDeployAppConfContent):
9326
9467
  nginx_items = NginxConfigItems.of(pcc(ac.items))
@@ -9353,9 +9494,15 @@ class DeployConfManager:
9353
9494
  self,
9354
9495
  spec: DeployAppConfSpec,
9355
9496
  app_conf_dir: str,
9497
+ *,
9498
+ string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
9356
9499
  ) -> None:
9357
- def process_str(s: str) -> str:
9358
- return s
9500
+ process_str: ta.Any
9501
+ if string_ns is not None:
9502
+ def process_str(s: str) -> str:
9503
+ return s.format(**string_ns)
9504
+ else:
9505
+ process_str = None
9359
9506
 
9360
9507
  for acf in spec.files or []:
9361
9508
  await self._write_app_conf_file(
@@ -9528,125 +9675,6 @@ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
9528
9675
  return self._owned_deploy_paths
9529
9676
 
9530
9677
 
9531
- ########################################
9532
- # ../deploy/specs.py
9533
-
9534
-
9535
- ##
9536
-
9537
-
9538
- class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
9539
- @cached_nullary
9540
- def _key_str(self) -> str:
9541
- return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
9542
-
9543
- @abc.abstractmethod
9544
- def key(self) -> KeyDeployTagT:
9545
- raise NotImplementedError
9546
-
9547
-
9548
- ##
9549
-
9550
-
9551
- @dc.dataclass(frozen=True)
9552
- class DeployGitRepo:
9553
- host: ta.Optional[str] = None
9554
- username: ta.Optional[str] = None
9555
- path: ta.Optional[str] = None
9556
-
9557
- def __post_init__(self) -> None:
9558
- check.not_in('..', check.non_empty_str(self.host))
9559
- check.not_in('.', check.non_empty_str(self.path))
9560
-
9561
-
9562
- @dc.dataclass(frozen=True)
9563
- class DeployGitSpec:
9564
- repo: DeployGitRepo
9565
- rev: DeployRev
9566
-
9567
- subtrees: ta.Optional[ta.Sequence[str]] = None
9568
-
9569
- def __post_init__(self) -> None:
9570
- check.non_empty_str(self.rev)
9571
- if self.subtrees is not None:
9572
- for st in self.subtrees:
9573
- check.non_empty_str(st)
9574
-
9575
-
9576
- ##
9577
-
9578
-
9579
- @dc.dataclass(frozen=True)
9580
- class DeployVenvSpec:
9581
- interp: ta.Optional[str] = None
9582
-
9583
- requirements_files: ta.Optional[ta.Sequence[str]] = None
9584
- extra_dependencies: ta.Optional[ta.Sequence[str]] = None
9585
-
9586
- use_uv: bool = False
9587
-
9588
-
9589
- ##
9590
-
9591
-
9592
- @dc.dataclass(frozen=True)
9593
- class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
9594
- app: DeployApp
9595
-
9596
- git: DeployGitSpec
9597
-
9598
- venv: ta.Optional[DeployVenvSpec] = None
9599
-
9600
- conf: ta.Optional[DeployAppConfSpec] = None
9601
-
9602
- # @ta.override
9603
- def key(self) -> DeployAppKey:
9604
- return DeployAppKey(self._key_str())
9605
-
9606
-
9607
- @dc.dataclass(frozen=True)
9608
- class DeployAppLinksSpec:
9609
- apps: ta.Sequence[DeployApp] = ()
9610
-
9611
- exclude_unspecified: bool = False
9612
-
9613
-
9614
- ##
9615
-
9616
-
9617
- @dc.dataclass(frozen=True)
9618
- class DeploySystemdSpec:
9619
- # ~/.config/systemd/user/
9620
- unit_dir: ta.Optional[str] = None
9621
-
9622
-
9623
- ##
9624
-
9625
-
9626
- @dc.dataclass(frozen=True)
9627
- class DeploySpec(DeploySpecKeyed[DeployKey]):
9628
- home: DeployHome
9629
-
9630
- apps: ta.Sequence[DeployAppSpec] = ()
9631
-
9632
- app_links: DeployAppLinksSpec = DeployAppLinksSpec()
9633
-
9634
- systemd: ta.Optional[DeploySystemdSpec] = None
9635
-
9636
- def __post_init__(self) -> None:
9637
- check.non_empty_str(self.home)
9638
-
9639
- seen: ta.Set[DeployApp] = set()
9640
- for a in self.apps:
9641
- if a.app in seen:
9642
- raise KeyError(a.app)
9643
- seen.add(a.app)
9644
-
9645
- # @ta.override
9646
- def key(self) -> DeployKey:
9647
- return DeployKey(self._key_str())
9648
-
9649
-
9650
9678
  ########################################
9651
9679
  # ../remote/_main.py
9652
9680
 
@@ -10967,7 +10995,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
10967
10995
 
10968
10996
  async def fetch(self, rev: DeployRev) -> None:
10969
10997
  await self.init()
10970
- await self._call('git', 'fetch', '--depth=1', 'origin', rev)
10998
+
10999
+ # This fetch shouldn't be depth=1 - git doesn't reuse local data with shallow fetches.
11000
+ await self._call('git', 'fetch', 'origin', rev)
10971
11001
 
10972
11002
  #
10973
11003
 
@@ -11125,7 +11155,7 @@ class DeploySystemdManager:
11125
11155
 
11126
11156
  #
11127
11157
 
11128
- if sys.platform == 'linux':
11158
+ if sys.platform == 'linux' and shutil.which('systemctl') is not None:
11129
11159
  async def reload() -> None:
11130
11160
  await asyncio_subprocesses.check_call('systemctl', '--user', 'daemon-reload')
11131
11161
 
@@ -11614,6 +11644,8 @@ class DeployAppManager(DeployPathOwner):
11614
11644
  spec: DeployAppSpec,
11615
11645
  home: DeployHome,
11616
11646
  tags: DeployTagMap,
11647
+ *,
11648
+ conf_string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
11617
11649
  ) -> PreparedApp:
11618
11650
  spec_json = json_dumps_pretty(self._msh.marshal_obj(spec))
11619
11651
 
@@ -11670,9 +11702,17 @@ class DeployAppManager(DeployPathOwner):
11670
11702
  if spec.conf is not None:
11671
11703
  conf_dir = os.path.join(app_dir, 'conf')
11672
11704
  rkw.update(conf_dir=conf_dir)
11705
+
11706
+ conf_ns: ta.Dict[str, ta.Any] = dict(
11707
+ **(conf_string_ns or {}),
11708
+ app=spec.app.s,
11709
+ app_dir=app_dir.rstrip('/'),
11710
+ )
11711
+
11673
11712
  await self._conf.write_app_conf(
11674
11713
  spec.conf,
11675
11714
  conf_dir,
11715
+ string_ns=conf_ns,
11676
11716
  )
11677
11717
 
11678
11718
  #
@@ -11735,7 +11775,7 @@ class DeployAppManager(DeployPathOwner):
11735
11775
  ##
11736
11776
 
11737
11777
 
11738
- DEPLOY_TAG_DATETIME_FMT = '%Y%m%dT%H%M%SZ'
11778
+ DEPLOY_TAG_DATETIME_FMT = '%Y-%m-%d-T-%H-%M-%S-%f-Z'
11739
11779
 
11740
11780
 
11741
11781
  DeployManagerUtcClock = ta.NewType('DeployManagerUtcClock', Func0[datetime.datetime])
@@ -11888,8 +11928,9 @@ class DeployDriver:
11888
11928
 
11889
11929
  das: ta.Set[DeployApp] = {a.app for a in self._spec.apps}
11890
11930
  las: ta.Set[DeployApp] = set(self._spec.app_links.apps)
11891
- if (ras := das & las):
11892
- raise RuntimeError(f'Must not specify apps as both deploy and link: {sorted(a.s for a in ras)}')
11931
+ ras: ta.Set[DeployApp] = set(self._spec.app_links.removed_apps)
11932
+ check.empty(das & (las | ras))
11933
+ check.empty(las & ras)
11893
11934
 
11894
11935
  #
11895
11936
 
@@ -11935,10 +11976,10 @@ class DeployDriver:
11935
11976
  cad = abs_real_path(os.path.join(current_link, 'apps'))
11936
11977
  if os.path.exists(cad):
11937
11978
  for d in os.listdir(cad):
11938
- if (da := DeployApp(d)) not in das:
11979
+ if (da := DeployApp(d)) not in das and da not in ras:
11939
11980
  las.add(da)
11940
11981
 
11941
- for la in self._spec.app_links.apps:
11982
+ for la in las:
11942
11983
  await self._drive_app_link(
11943
11984
  la,
11944
11985
  current_link,
@@ -11968,10 +12009,16 @@ class DeployDriver:
11968
12009
  #
11969
12010
 
11970
12011
  async def _drive_app_deploy(self, app: DeployAppSpec) -> None:
12012
+ current_deploy_link = os.path.join(self._home, self._deploys.CURRENT_DEPLOY_LINK.render())
12013
+
11971
12014
  pa = await self._apps.prepare_app(
11972
12015
  app,
11973
12016
  self._home,
11974
12017
  self.tags,
12018
+ conf_string_ns=dict(
12019
+ deploy_home=self._home,
12020
+ current_deploy_link=current_deploy_link,
12021
+ ),
11975
12022
  )
11976
12023
 
11977
12024
  #