ominfra 0.0.0.dev191__py3-none-any.whl → 0.0.0.dev193__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ominfra/scripts/manage.py CHANGED
@@ -16,6 +16,7 @@ import asyncio.subprocess
16
16
  import base64
17
17
  import collections
18
18
  import collections.abc
19
+ import configparser
19
20
  import contextlib
20
21
  import contextvars
21
22
  import ctypes as ct
@@ -73,17 +74,20 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
73
74
  VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
74
75
  VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
75
76
 
76
- # ../../omdev/toml/parser.py
77
- TomlParseFloat = ta.Callable[[str], ta.Any]
78
- TomlKey = ta.Tuple[str, ...]
79
- TomlPos = int # ta.TypeAlias
80
-
81
77
  # deploy/paths/types.py
82
78
  DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
83
79
 
84
80
  # ../../omlish/asyncs/asyncio/timeouts.py
85
81
  AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
86
82
 
83
+ # ../../omlish/formats/ini/sections.py
84
+ IniSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
85
+
86
+ # ../../omlish/formats/toml/parser.py
87
+ TomlParseFloat = ta.Callable[[str], ta.Any]
88
+ TomlKey = ta.Tuple[str, ...]
89
+ TomlPos = int # ta.TypeAlias
90
+
87
91
  # ../../omlish/lite/cached.py
88
92
  T = ta.TypeVar('T')
89
93
  CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
@@ -129,7 +133,6 @@ AtomicPathSwapState = ta.Literal['open', 'committed', 'aborted'] # ta.TypeAlias
129
133
 
130
134
  # ../configs.py
131
135
  ConfigMapping = ta.Mapping[str, ta.Any]
132
- IniConfigSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
133
136
 
134
137
  # ../../omlish/subprocesses.py
135
138
  SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
@@ -549,1482 +552,1523 @@ def canonicalize_version(
549
552
 
550
553
 
551
554
  ########################################
552
- # ../../../omdev/toml/parser.py
553
- # SPDX-License-Identifier: MIT
554
- # SPDX-FileCopyrightText: 2021 Taneli Hukkinen
555
- # Licensed to PSF under a Contributor Agreement.
556
- #
557
- # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
558
- # --------------------------------------------
559
- #
560
- # 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
561
- # ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
562
- # documentation.
563
- #
564
- # 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
565
- # royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
566
- # works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
567
- # Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
568
- # 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
569
- # Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
570
- #
571
- # 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
572
- # wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
573
- # any such work a brief summary of the changes made to Python.
574
- #
575
- # 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
576
- # EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
577
- # OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
578
- # RIGHTS.
579
- #
580
- # 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
581
- # DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
582
- # ADVISED OF THE POSSIBILITY THEREOF.
583
- #
584
- # 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
585
- #
586
- # 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
587
- # venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
588
- # name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
589
- #
590
- # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
591
- # License Agreement.
592
- #
593
- # https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
555
+ # ../config.py
594
556
 
595
557
 
596
- ##
558
+ @dc.dataclass(frozen=True)
559
+ class MainConfig:
560
+ log_level: ta.Optional[str] = 'INFO'
597
561
 
562
+ debug: bool = False
598
563
 
599
- _TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
600
564
 
601
- TOML_RE_NUMBER = re.compile(
602
- r"""
603
- 0
604
- (?:
605
- x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
606
- |
607
- b[01](?:_?[01])* # bin
608
- |
609
- o[0-7](?:_?[0-7])* # oct
610
- )
611
- |
612
- [+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
613
- (?P<floatpart>
614
- (?:\.[0-9](?:_?[0-9])*)? # optional fractional part
615
- (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
616
- )
617
- """,
618
- flags=re.VERBOSE,
619
- )
620
- TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
621
- TOML_RE_DATETIME = re.compile(
622
- rf"""
623
- ([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
624
- (?:
625
- [Tt ]
626
- {_TOML_TIME_RE_STR}
627
- (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
628
- )?
629
- """,
630
- flags=re.VERBOSE,
631
- )
565
+ ########################################
566
+ # ../deploy/config.py
632
567
 
633
568
 
634
- def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
635
- """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
569
+ ##
636
570
 
637
- Raises ValueError if the match does not correspond to a valid date or datetime.
638
- """
639
- (
640
- year_str,
641
- month_str,
642
- day_str,
643
- hour_str,
644
- minute_str,
645
- sec_str,
646
- micros_str,
647
- zulu_time,
648
- offset_sign_str,
649
- offset_hour_str,
650
- offset_minute_str,
651
- ) = match.groups()
652
- year, month, day = int(year_str), int(month_str), int(day_str)
653
- if hour_str is None:
654
- return datetime.date(year, month, day)
655
- hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
656
- micros = int(micros_str.ljust(6, '0')) if micros_str else 0
657
- if offset_sign_str:
658
- tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
659
- offset_hour_str, offset_minute_str, offset_sign_str,
660
- )
661
- elif zulu_time:
662
- tz = datetime.UTC
663
- else: # local date-time
664
- tz = None
665
- return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
666
571
 
572
+ @dc.dataclass(frozen=True)
573
+ class DeployConfig:
574
+ pass
667
575
 
668
- @functools.lru_cache() # noqa
669
- def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
670
- sign = 1 if sign_str == '+' else -1
671
- return datetime.timezone(
672
- datetime.timedelta(
673
- hours=sign * int(hour_str),
674
- minutes=sign * int(minute_str),
675
- ),
676
- )
677
576
 
577
+ ########################################
578
+ # ../deploy/paths/types.py
678
579
 
679
- def toml_match_to_localtime(match: re.Match) -> datetime.time:
680
- hour_str, minute_str, sec_str, micros_str = match.groups()
681
- micros = int(micros_str.ljust(6, '0')) if micros_str else 0
682
- return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
683
580
 
581
+ ##
684
582
 
685
- def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
686
- if match.group('floatpart'):
687
- return parse_float(match.group())
688
- return int(match.group(), 0)
689
583
 
584
+ ########################################
585
+ # ../deploy/types.py
690
586
 
691
- TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
692
587
 
693
- # Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
694
- # functions.
695
- TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
696
- TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
588
+ ##
697
589
 
698
- TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
699
- TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
700
590
 
701
- TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
591
+ DeployHome = ta.NewType('DeployHome', str)
702
592
 
703
- TOML_WS = frozenset(' \t')
704
- TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
705
- TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
706
- TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
707
- TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
593
+ DeployRev = ta.NewType('DeployRev', str)
708
594
 
709
- TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
710
- {
711
- '\\b': '\u0008', # backspace
712
- '\\t': '\u0009', # tab
713
- '\\n': '\u000A', # linefeed
714
- '\\f': '\u000C', # form feed
715
- '\\r': '\u000D', # carriage return
716
- '\\"': '\u0022', # quote
717
- '\\\\': '\u005C', # backslash
718
- },
719
- )
720
595
 
596
+ ########################################
597
+ # ../../pyremote.py
598
+ """
599
+ Basically this: https://mitogen.networkgenomics.com/howitworks.html
721
600
 
722
- class TomlDecodeError(ValueError):
723
- """An error raised if a document is not valid TOML."""
601
+ TODO:
602
+ - log: ta.Optional[logging.Logger] = None + log.debug's
603
+ """
724
604
 
725
605
 
726
- def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
727
- """Parse TOML from a binary file object."""
728
- b = fp.read()
729
- try:
730
- s = b.decode()
731
- except AttributeError:
732
- raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
733
- return toml_loads(s, parse_float=parse_float)
606
+ ##
734
607
 
735
608
 
736
- def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
737
- """Parse TOML from a string."""
609
+ @dc.dataclass(frozen=True)
610
+ class PyremoteBootstrapOptions:
611
+ debug: bool = False
738
612
 
739
- # The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
740
- try:
741
- src = s.replace('\r\n', '\n')
742
- except (AttributeError, TypeError):
743
- raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
744
- pos = 0
745
- out = TomlOutput(TomlNestedDict(), TomlFlags())
746
- header: TomlKey = ()
747
- parse_float = toml_make_safe_parse_float(parse_float)
613
+ DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
614
+ main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
748
615
 
749
- # Parse one statement at a time (typically means one line in TOML source)
750
- while True:
751
- # 1. Skip line leading whitespace
752
- pos = toml_skip_chars(src, pos, TOML_WS)
753
616
 
754
- # 2. Parse rules. Expect one of the following:
755
- # - end of file
756
- # - end of line
757
- # - comment
758
- # - key/value pair
759
- # - append dict to list (and move to its namespace)
760
- # - create dict (and move to its namespace)
761
- # Skip trailing whitespace when applicable.
762
- try:
763
- char = src[pos]
764
- except IndexError:
765
- break
766
- if char == '\n':
767
- pos += 1
768
- continue
769
- if char in TOML_KEY_INITIAL_CHARS:
770
- pos = toml_key_value_rule(src, pos, out, header, parse_float)
771
- pos = toml_skip_chars(src, pos, TOML_WS)
772
- elif char == '[':
773
- try:
774
- second_char: ta.Optional[str] = src[pos + 1]
775
- except IndexError:
776
- second_char = None
777
- out.flags.finalize_pending()
778
- if second_char == '[':
779
- pos, header = toml_create_list_rule(src, pos, out)
780
- else:
781
- pos, header = toml_create_dict_rule(src, pos, out)
782
- pos = toml_skip_chars(src, pos, TOML_WS)
783
- elif char != '#':
784
- raise toml_suffixed_err(src, pos, 'Invalid statement')
617
+ ##
785
618
 
786
- # 3. Skip comment
787
- pos = toml_skip_comment(src, pos)
788
619
 
789
- # 4. Expect end of line or end of file
790
- try:
791
- char = src[pos]
792
- except IndexError:
793
- break
794
- if char != '\n':
795
- raise toml_suffixed_err(
796
- src, pos, 'Expected newline or end of document after a statement',
797
- )
798
- pos += 1
620
+ @dc.dataclass(frozen=True)
621
+ class PyremoteEnvInfo:
622
+ sys_base_prefix: str
623
+ sys_byteorder: str
624
+ sys_defaultencoding: str
625
+ sys_exec_prefix: str
626
+ sys_executable: str
627
+ sys_implementation_name: str
628
+ sys_path: ta.List[str]
629
+ sys_platform: str
630
+ sys_prefix: str
631
+ sys_version: str
632
+ sys_version_info: ta.List[ta.Union[int, str]]
799
633
 
800
- return out.data.dict
634
+ platform_architecture: ta.List[str]
635
+ platform_machine: str
636
+ platform_platform: str
637
+ platform_processor: str
638
+ platform_system: str
639
+ platform_release: str
640
+ platform_version: str
801
641
 
642
+ site_userbase: str
802
643
 
803
- class TomlFlags:
804
- """Flags that map to parsed keys/namespaces."""
644
+ os_cwd: str
645
+ os_gid: int
646
+ os_loadavg: ta.List[float]
647
+ os_login: ta.Optional[str]
648
+ os_pgrp: int
649
+ os_pid: int
650
+ os_ppid: int
651
+ os_uid: int
805
652
 
806
- # Marks an immutable namespace (inline array or inline table).
807
- FROZEN = 0
808
- # Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
809
- EXPLICIT_NEST = 1
653
+ pw_name: str
654
+ pw_uid: int
655
+ pw_gid: int
656
+ pw_gecos: str
657
+ pw_dir: str
658
+ pw_shell: str
810
659
 
811
- def __init__(self) -> None:
812
- self._flags: ta.Dict[str, dict] = {}
813
- self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
660
+ env_path: ta.Optional[str]
814
661
 
815
- def add_pending(self, key: TomlKey, flag: int) -> None:
816
- self._pending_flags.add((key, flag))
817
662
 
818
- def finalize_pending(self) -> None:
819
- for key, flag in self._pending_flags:
820
- self.set(key, flag, recursive=False)
821
- self._pending_flags.clear()
663
+ def _get_pyremote_env_info() -> PyremoteEnvInfo:
664
+ os_uid = os.getuid()
822
665
 
823
- def unset_all(self, key: TomlKey) -> None:
824
- cont = self._flags
825
- for k in key[:-1]:
826
- if k not in cont:
827
- return
828
- cont = cont[k]['nested']
829
- cont.pop(key[-1], None)
666
+ pw = pwd.getpwuid(os_uid)
830
667
 
831
- def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
832
- cont = self._flags
833
- key_parent, key_stem = key[:-1], key[-1]
834
- for k in key_parent:
835
- if k not in cont:
836
- cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
837
- cont = cont[k]['nested']
838
- if key_stem not in cont:
839
- cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
840
- cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
668
+ os_login: ta.Optional[str]
669
+ try:
670
+ os_login = os.getlogin()
671
+ except OSError:
672
+ os_login = None
841
673
 
842
- def is_(self, key: TomlKey, flag: int) -> bool:
843
- if not key:
844
- return False # document root has no flags
845
- cont = self._flags
846
- for k in key[:-1]:
847
- if k not in cont:
848
- return False
849
- inner_cont = cont[k]
850
- if flag in inner_cont['recursive_flags']:
851
- return True
852
- cont = inner_cont['nested']
853
- key_stem = key[-1]
854
- if key_stem in cont:
855
- cont = cont[key_stem]
856
- return flag in cont['flags'] or flag in cont['recursive_flags']
857
- return False
674
+ return PyremoteEnvInfo(
675
+ sys_base_prefix=sys.base_prefix,
676
+ sys_byteorder=sys.byteorder,
677
+ sys_defaultencoding=sys.getdefaultencoding(),
678
+ sys_exec_prefix=sys.exec_prefix,
679
+ sys_executable=sys.executable,
680
+ sys_implementation_name=sys.implementation.name,
681
+ sys_path=sys.path,
682
+ sys_platform=sys.platform,
683
+ sys_prefix=sys.prefix,
684
+ sys_version=sys.version,
685
+ sys_version_info=list(sys.version_info),
858
686
 
687
+ platform_architecture=list(platform.architecture()),
688
+ platform_machine=platform.machine(),
689
+ platform_platform=platform.platform(),
690
+ platform_processor=platform.processor(),
691
+ platform_system=platform.system(),
692
+ platform_release=platform.release(),
693
+ platform_version=platform.version(),
859
694
 
860
- class TomlNestedDict:
861
- def __init__(self) -> None:
862
- # The parsed content of the TOML document
863
- self.dict: ta.Dict[str, ta.Any] = {}
695
+ site_userbase=site.getuserbase(),
864
696
 
865
- def get_or_create_nest(
866
- self,
867
- key: TomlKey,
868
- *,
869
- access_lists: bool = True,
870
- ) -> dict:
871
- cont: ta.Any = self.dict
872
- for k in key:
873
- if k not in cont:
874
- cont[k] = {}
875
- cont = cont[k]
876
- if access_lists and isinstance(cont, list):
877
- cont = cont[-1]
878
- if not isinstance(cont, dict):
879
- raise KeyError('There is no nest behind this key')
880
- return cont
697
+ os_cwd=os.getcwd(),
698
+ os_gid=os.getgid(),
699
+ os_loadavg=list(os.getloadavg()),
700
+ os_login=os_login,
701
+ os_pgrp=os.getpgrp(),
702
+ os_pid=os.getpid(),
703
+ os_ppid=os.getppid(),
704
+ os_uid=os_uid,
881
705
 
882
- def append_nest_to_list(self, key: TomlKey) -> None:
883
- cont = self.get_or_create_nest(key[:-1])
884
- last_key = key[-1]
885
- if last_key in cont:
886
- list_ = cont[last_key]
887
- if not isinstance(list_, list):
888
- raise KeyError('An object other than list found behind this key')
889
- list_.append({})
890
- else:
891
- cont[last_key] = [{}]
706
+ pw_name=pw.pw_name,
707
+ pw_uid=pw.pw_uid,
708
+ pw_gid=pw.pw_gid,
709
+ pw_gecos=pw.pw_gecos,
710
+ pw_dir=pw.pw_dir,
711
+ pw_shell=pw.pw_shell,
892
712
 
713
+ env_path=os.environ.get('PATH'),
714
+ )
893
715
 
894
- class TomlOutput(ta.NamedTuple):
895
- data: TomlNestedDict
896
- flags: TomlFlags
897
716
 
717
+ ##
898
718
 
899
- def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
900
- try:
901
- while src[pos] in chars:
902
- pos += 1
903
- except IndexError:
904
- pass
905
- return pos
906
719
 
720
+ _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
721
+ _PYREMOTE_BOOTSTRAP_SRC_FD = 101
907
722
 
908
- def toml_skip_until(
909
- src: str,
910
- pos: TomlPos,
911
- expect: str,
912
- *,
913
- error_on: ta.FrozenSet[str],
914
- error_on_eof: bool,
915
- ) -> TomlPos:
916
- try:
917
- new_pos = src.index(expect, pos)
918
- except ValueError:
919
- new_pos = len(src)
920
- if error_on_eof:
921
- raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
723
+ _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
724
+ _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
725
+ _PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
726
+ _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
727
+ _PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
922
728
 
923
- if not error_on.isdisjoint(src[pos:new_pos]):
924
- while src[pos] not in error_on:
925
- pos += 1
926
- raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
927
- return new_pos
729
+ _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
730
+ _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
731
+ _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
732
+ _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
928
733
 
734
+ _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
929
735
 
930
- def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
931
- try:
932
- char: ta.Optional[str] = src[pos]
933
- except IndexError:
934
- char = None
935
- if char == '#':
936
- return toml_skip_until(
937
- src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
938
- )
939
- return pos
736
+ _PYREMOTE_BOOTSTRAP_IMPORTS = [
737
+ 'base64',
738
+ 'os',
739
+ 'struct',
740
+ 'sys',
741
+ 'zlib',
742
+ ]
940
743
 
941
744
 
942
- def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
943
- while True:
944
- pos_before_skip = pos
945
- pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
946
- pos = toml_skip_comment(src, pos)
947
- if pos == pos_before_skip:
948
- return pos
745
+ def _pyremote_bootstrap_main(context_name: str) -> None:
746
+ # Get pid
747
+ pid = os.getpid()
949
748
 
749
+ # Two copies of payload src to be sent to parent
750
+ r0, w0 = os.pipe()
751
+ r1, w1 = os.pipe()
950
752
 
951
- def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
952
- pos += 1 # Skip "["
953
- pos = toml_skip_chars(src, pos, TOML_WS)
954
- pos, key = toml_parse_key(src, pos)
753
+ if (cp := os.fork()):
754
+ # Parent process
955
755
 
956
- if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
957
- raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
958
- out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
959
- try:
960
- out.data.get_or_create_nest(key)
961
- except KeyError:
962
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
756
+ # Dup original stdin to comm_fd for use as comm channel
757
+ os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
963
758
 
964
- if not src.startswith(']', pos):
965
- raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
966
- return pos + 1, key
759
+ # Overwrite stdin (fed to python repl) with first copy of src
760
+ os.dup2(r0, 0)
967
761
 
762
+ # Dup second copy of src to src_fd to recover after launch
763
+ os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
968
764
 
969
- def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
970
- pos += 2 # Skip "[["
971
- pos = toml_skip_chars(src, pos, TOML_WS)
972
- pos, key = toml_parse_key(src, pos)
973
-
974
- if out.flags.is_(key, TomlFlags.FROZEN):
975
- raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
976
- # Free the namespace now that it points to another empty list item...
977
- out.flags.unset_all(key)
978
- # ...but this key precisely is still prohibited from table declaration
979
- out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
980
- try:
981
- out.data.append_nest_to_list(key)
982
- except KeyError:
983
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
765
+ # Close remaining fd's
766
+ for f in [r0, w0, r1, w1]:
767
+ os.close(f)
984
768
 
985
- if not src.startswith(']]', pos):
986
- raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
987
- return pos + 2, key
769
+ # Save vars
770
+ env = os.environ
771
+ exe = sys.executable
772
+ env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
773
+ env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
774
+ env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
988
775
 
776
+ # Start repl reading stdin from r0
777
+ os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
989
778
 
990
- def toml_key_value_rule(
991
- src: str,
992
- pos: TomlPos,
993
- out: TomlOutput,
994
- header: TomlKey,
995
- parse_float: TomlParseFloat,
996
- ) -> TomlPos:
997
- pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
998
- key_parent, key_stem = key[:-1], key[-1]
999
- abs_key_parent = header + key_parent
779
+ else:
780
+ # Child process
1000
781
 
1001
- relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
1002
- for cont_key in relative_path_cont_keys:
1003
- # Check that dotted key syntax does not redefine an existing table
1004
- if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
1005
- raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
1006
- # Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
1007
- # table sections.
1008
- out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
782
+ # Write first ack
783
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
1009
784
 
1010
- if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
1011
- raise toml_suffixed_err(
1012
- src,
1013
- pos,
1014
- f'Cannot mutate immutable namespace {abs_key_parent}',
1015
- )
785
+ # Write pid
786
+ os.write(1, struct.pack('<Q', pid))
1016
787
 
1017
- try:
1018
- nest = out.data.get_or_create_nest(abs_key_parent)
1019
- except KeyError:
1020
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1021
- if key_stem in nest:
1022
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
1023
- # Mark inline table and array namespaces recursively immutable
1024
- if isinstance(value, (dict, list)):
1025
- out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
1026
- nest[key_stem] = value
1027
- return pos
788
+ # Read payload src from stdin
789
+ payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
790
+ if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
791
+ raise EOFError
792
+ payload_src = zlib.decompress(payload_z)
1028
793
 
794
+ # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
795
+ # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
796
+ for w in [w0, w1]:
797
+ fp = os.fdopen(w, 'wb', 0)
798
+ fp.write(payload_src)
799
+ fp.close()
1029
800
 
1030
- def toml_parse_key_value_pair(
1031
- src: str,
1032
- pos: TomlPos,
1033
- parse_float: TomlParseFloat,
1034
- ) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
1035
- pos, key = toml_parse_key(src, pos)
1036
- try:
1037
- char: ta.Optional[str] = src[pos]
1038
- except IndexError:
1039
- char = None
1040
- if char != '=':
1041
- raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
1042
- pos += 1
1043
- pos = toml_skip_chars(src, pos, TOML_WS)
1044
- pos, value = toml_parse_value(src, pos, parse_float)
1045
- return pos, key, value
801
+ # Write second ack
802
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
1046
803
 
804
+ # Exit child
805
+ sys.exit(0)
1047
806
 
1048
- def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
1049
- pos, key_part = toml_parse_key_part(src, pos)
1050
- key: TomlKey = (key_part,)
1051
- pos = toml_skip_chars(src, pos, TOML_WS)
1052
- while True:
1053
- try:
1054
- char: ta.Optional[str] = src[pos]
1055
- except IndexError:
1056
- char = None
1057
- if char != '.':
1058
- return pos, key
1059
- pos += 1
1060
- pos = toml_skip_chars(src, pos, TOML_WS)
1061
- pos, key_part = toml_parse_key_part(src, pos)
1062
- key += (key_part,)
1063
- pos = toml_skip_chars(src, pos, TOML_WS)
1064
807
 
808
+ ##
1065
809
 
1066
- def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1067
- try:
1068
- char: ta.Optional[str] = src[pos]
1069
- except IndexError:
1070
- char = None
1071
- if char in TOML_BARE_KEY_CHARS:
1072
- start_pos = pos
1073
- pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
1074
- return pos, src[start_pos:pos]
1075
- if char == "'":
1076
- return toml_parse_literal_str(src, pos)
1077
- if char == '"':
1078
- return toml_parse_one_line_basic_str(src, pos)
1079
- raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
1080
810
 
811
+ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
812
+ if any(c in context_name for c in '\'"'):
813
+ raise NameError(context_name)
1081
814
 
1082
- def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1083
- pos += 1
1084
- return toml_parse_basic_str(src, pos, multiline=False)
815
+ import inspect
816
+ import textwrap
817
+ bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
1085
818
 
819
+ for gl in [
820
+ '_PYREMOTE_BOOTSTRAP_INPUT_FD',
821
+ '_PYREMOTE_BOOTSTRAP_SRC_FD',
1086
822
 
1087
- def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
1088
- pos += 1
1089
- array: list = []
823
+ '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
824
+ '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
825
+ '_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
1090
826
 
1091
- pos = toml_skip_comments_and_array_ws(src, pos)
1092
- if src.startswith(']', pos):
1093
- return pos + 1, array
1094
- while True:
1095
- pos, val = toml_parse_value(src, pos, parse_float)
1096
- array.append(val)
1097
- pos = toml_skip_comments_and_array_ws(src, pos)
827
+ '_PYREMOTE_BOOTSTRAP_ACK0',
828
+ '_PYREMOTE_BOOTSTRAP_ACK1',
1098
829
 
1099
- c = src[pos:pos + 1]
1100
- if c == ']':
1101
- return pos + 1, array
1102
- if c != ',':
1103
- raise toml_suffixed_err(src, pos, 'Unclosed array')
1104
- pos += 1
830
+ '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
831
+ ]:
832
+ bs_src = bs_src.replace(gl, repr(globals()[gl]))
1105
833
 
1106
- pos = toml_skip_comments_and_array_ws(src, pos)
1107
- if src.startswith(']', pos):
1108
- return pos + 1, array
834
+ bs_src = '\n'.join(
835
+ cl
836
+ for l in bs_src.splitlines()
837
+ if (cl := (l.split('#')[0]).rstrip())
838
+ if cl.strip()
839
+ )
1109
840
 
841
+ bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
842
+ bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
843
+ if b'"' in bs_z85:
844
+ raise ValueError(bs_z85)
1110
845
 
1111
- def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
1112
- pos += 1
1113
- nested_dict = TomlNestedDict()
1114
- flags = TomlFlags()
846
+ stmts = [
847
+ f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
848
+ f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
849
+ f'_pyremote_bootstrap_main("{context_name}")',
850
+ ]
1115
851
 
1116
- pos = toml_skip_chars(src, pos, TOML_WS)
1117
- if src.startswith('}', pos):
1118
- return pos + 1, nested_dict.dict
1119
- while True:
1120
- pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1121
- key_parent, key_stem = key[:-1], key[-1]
1122
- if flags.is_(key, TomlFlags.FROZEN):
1123
- raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1124
- try:
1125
- nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
1126
- except KeyError:
1127
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1128
- if key_stem in nest:
1129
- raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
1130
- nest[key_stem] = value
1131
- pos = toml_skip_chars(src, pos, TOML_WS)
1132
- c = src[pos:pos + 1]
1133
- if c == '}':
1134
- return pos + 1, nested_dict.dict
1135
- if c != ',':
1136
- raise toml_suffixed_err(src, pos, 'Unclosed inline table')
1137
- if isinstance(value, (dict, list)):
1138
- flags.set(key, TomlFlags.FROZEN, recursive=True)
1139
- pos += 1
1140
- pos = toml_skip_chars(src, pos, TOML_WS)
852
+ cmd = '; '.join(stmts)
853
+ return cmd
1141
854
 
1142
855
 
1143
- def toml_parse_basic_str_escape(
1144
- src: str,
1145
- pos: TomlPos,
1146
- *,
1147
- multiline: bool = False,
1148
- ) -> ta.Tuple[TomlPos, str]:
1149
- escape_id = src[pos:pos + 2]
1150
- pos += 2
1151
- if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
1152
- # Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
1153
- # newline.
1154
- if escape_id != '\\\n':
1155
- pos = toml_skip_chars(src, pos, TOML_WS)
1156
- try:
1157
- char = src[pos]
1158
- except IndexError:
1159
- return pos, ''
1160
- if char != '\n':
1161
- raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
1162
- pos += 1
1163
- pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1164
- return pos, ''
1165
- if escape_id == '\\u':
1166
- return toml_parse_hex_char(src, pos, 4)
1167
- if escape_id == '\\U':
1168
- return toml_parse_hex_char(src, pos, 8)
1169
- try:
1170
- return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
1171
- except KeyError:
1172
- raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
856
+ ##
1173
857
 
1174
858
 
1175
- def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1176
- return toml_parse_basic_str_escape(src, pos, multiline=True)
859
+ @dc.dataclass(frozen=True)
860
+ class PyremotePayloadRuntime:
861
+ input: ta.BinaryIO
862
+ output: ta.BinaryIO
863
+ context_name: str
864
+ payload_src: str
865
+ options: PyremoteBootstrapOptions
866
+ env_info: PyremoteEnvInfo
1177
867
 
1178
868
 
1179
- def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
1180
- hex_str = src[pos:pos + hex_len]
1181
- if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
1182
- raise toml_suffixed_err(src, pos, 'Invalid hex value')
1183
- pos += hex_len
1184
- hex_int = int(hex_str, 16)
1185
- if not toml_is_unicode_scalar_value(hex_int):
1186
- raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
1187
- return pos, chr(hex_int)
869
+ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
870
+ # If src file var is not present we need to do initial finalization
871
+ if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
872
+ # Read second copy of payload src
873
+ r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
874
+ payload_src = r1.read().decode('utf-8')
875
+ r1.close()
876
+
877
+ # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
878
+ # a pipe at once.
879
+ os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
1188
880
 
881
+ # Read options
882
+ options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
883
+ if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
884
+ raise EOFError
885
+ options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
1189
886
 
1190
- def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1191
- pos += 1 # Skip starting apostrophe
1192
- start_pos = pos
1193
- pos = toml_skip_until(
1194
- src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
1195
- )
1196
- return pos + 1, src[start_pos:pos] # Skip ending apostrophe
887
+ # If debugging, re-exec as file
888
+ if options.debug:
889
+ # Write temp source file
890
+ import tempfile
891
+ tfd, tfn = tempfile.mkstemp('-pyremote.py')
892
+ os.write(tfd, payload_src.encode('utf-8'))
893
+ os.close(tfd)
1197
894
 
895
+ # Set vars
896
+ os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
897
+ os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
1198
898
 
1199
- def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
1200
- pos += 3
1201
- if src.startswith('\n', pos):
1202
- pos += 1
899
+ # Re-exec temp file
900
+ exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
901
+ context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
902
+ os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
1203
903
 
1204
- if literal:
1205
- delim = "'"
1206
- end_pos = toml_skip_until(
1207
- src,
1208
- pos,
1209
- "'''",
1210
- error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
1211
- error_on_eof=True,
1212
- )
1213
- result = src[pos:end_pos]
1214
- pos = end_pos + 3
1215
904
  else:
1216
- delim = '"'
1217
- pos, result = toml_parse_basic_str(src, pos, multiline=True)
905
+ # Load options json var
906
+ options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
907
+ options = PyremoteBootstrapOptions(**json.loads(options_json_str))
1218
908
 
1219
- # Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
1220
- if not src.startswith(delim, pos):
1221
- return pos, result
1222
- pos += 1
1223
- if not src.startswith(delim, pos):
1224
- return pos, result + delim
1225
- pos += 1
1226
- return pos, result + (delim * 2)
909
+ # Read temp source file
910
+ with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
911
+ payload_src = sf.read()
1227
912
 
913
+ # Restore vars
914
+ sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
915
+ context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
1228
916
 
1229
- def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
1230
- if multiline:
1231
- error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1232
- parse_escapes = toml_parse_basic_str_escape_multiline
1233
- else:
1234
- error_on = TOML_ILLEGAL_BASIC_STR_CHARS
1235
- parse_escapes = toml_parse_basic_str_escape
1236
- result = ''
1237
- start_pos = pos
1238
- while True:
1239
- try:
1240
- char = src[pos]
1241
- except IndexError:
1242
- raise toml_suffixed_err(src, pos, 'Unterminated string') from None
1243
- if char == '"':
1244
- if not multiline:
1245
- return pos + 1, result + src[start_pos:pos]
1246
- if src.startswith('"""', pos):
1247
- return pos + 3, result + src[start_pos:pos]
1248
- pos += 1
1249
- continue
1250
- if char == '\\':
1251
- result += src[start_pos:pos]
1252
- pos, parsed_escape = parse_escapes(src, pos)
1253
- result += parsed_escape
1254
- start_pos = pos
1255
- continue
1256
- if char in error_on:
1257
- raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
1258
- pos += 1
917
+ # Write third ack
918
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
1259
919
 
920
+ # Write env info
921
+ env_info = _get_pyremote_env_info()
922
+ env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
923
+ os.write(1, struct.pack('<I', len(env_info_json)))
924
+ os.write(1, env_info_json.encode('utf-8'))
1260
925
 
1261
- def toml_parse_value( # noqa: C901
1262
- src: str,
1263
- pos: TomlPos,
1264
- parse_float: TomlParseFloat,
1265
- ) -> ta.Tuple[TomlPos, ta.Any]:
1266
- try:
1267
- char: ta.Optional[str] = src[pos]
1268
- except IndexError:
1269
- char = None
926
+ # Setup IO
927
+ input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
928
+ output = os.fdopen(os.dup(1), 'wb', 0) # noqa
929
+ os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
930
+ os.close(nfd)
1270
931
 
1271
- # IMPORTANT: order conditions based on speed of checking and likelihood
932
+ if (mn := options.main_name_override) is not None:
933
+ # Inspections like typing.get_type_hints need an entry in sys.modules.
934
+ sys.modules[mn] = sys.modules['__main__']
1272
935
 
1273
- # Basic strings
1274
- if char == '"':
1275
- if src.startswith('"""', pos):
1276
- return toml_parse_multiline_str(src, pos, literal=False)
1277
- return toml_parse_one_line_basic_str(src, pos)
936
+ # Write fourth ack
937
+ output.write(_PYREMOTE_BOOTSTRAP_ACK3)
1278
938
 
1279
- # Literal strings
1280
- if char == "'":
1281
- if src.startswith("'''", pos):
1282
- return toml_parse_multiline_str(src, pos, literal=True)
1283
- return toml_parse_literal_str(src, pos)
939
+ # Return
940
+ return PyremotePayloadRuntime(
941
+ input=input,
942
+ output=output,
943
+ context_name=context_name,
944
+ payload_src=payload_src,
945
+ options=options,
946
+ env_info=env_info,
947
+ )
1284
948
 
1285
- # Booleans
1286
- if char == 't':
1287
- if src.startswith('true', pos):
1288
- return pos + 4, True
1289
- if char == 'f':
1290
- if src.startswith('false', pos):
1291
- return pos + 5, False
1292
949
 
1293
- # Arrays
1294
- if char == '[':
1295
- return toml_parse_array(src, pos, parse_float)
950
+ ##
1296
951
 
1297
- # Inline tables
1298
- if char == '{':
1299
- return toml_parse_inline_table(src, pos, parse_float)
1300
952
 
1301
- # Dates and times
1302
- datetime_match = TOML_RE_DATETIME.match(src, pos)
1303
- if datetime_match:
1304
- try:
1305
- datetime_obj = toml_match_to_datetime(datetime_match)
1306
- except ValueError as e:
1307
- raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
1308
- return datetime_match.end(), datetime_obj
1309
- localtime_match = TOML_RE_LOCALTIME.match(src, pos)
1310
- if localtime_match:
1311
- return localtime_match.end(), toml_match_to_localtime(localtime_match)
953
+ class PyremoteBootstrapDriver:
954
+ def __init__(
955
+ self,
956
+ payload_src: ta.Union[str, ta.Sequence[str]],
957
+ options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
958
+ ) -> None:
959
+ super().__init__()
1312
960
 
1313
- # Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
1314
- # located after handling of dates and times.
1315
- number_match = TOML_RE_NUMBER.match(src, pos)
1316
- if number_match:
1317
- return number_match.end(), toml_match_to_number(number_match, parse_float)
961
+ self._payload_src = payload_src
962
+ self._options = options
1318
963
 
1319
- # Special floats
1320
- first_three = src[pos:pos + 3]
1321
- if first_three in {'inf', 'nan'}:
1322
- return pos + 3, parse_float(first_three)
1323
- first_four = src[pos:pos + 4]
1324
- if first_four in {'-inf', '+inf', '-nan', '+nan'}:
1325
- return pos + 4, parse_float(first_four)
964
+ self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
965
+ self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
1326
966
 
1327
- raise toml_suffixed_err(src, pos, 'Invalid value')
967
+ self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
968
+ #
1328
969
 
970
+ @classmethod
971
+ def _prepare_payload_src(
972
+ cls,
973
+ payload_src: ta.Union[str, ta.Sequence[str]],
974
+ options: PyremoteBootstrapOptions,
975
+ ) -> str:
976
+ parts: ta.List[str]
977
+ if isinstance(payload_src, str):
978
+ parts = [payload_src]
979
+ else:
980
+ parts = list(payload_src)
1329
981
 
1330
- def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
1331
- """Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
982
+ if (mn := options.main_name_override) is not None:
983
+ parts.insert(0, f'__name__ = {mn!r}')
1332
984
 
1333
- def coord_repr(src: str, pos: TomlPos) -> str:
1334
- if pos >= len(src):
1335
- return 'end of document'
1336
- line = src.count('\n', 0, pos) + 1
1337
- if line == 1:
1338
- column = pos + 1
985
+ if len(parts) == 1:
986
+ return parts[0]
1339
987
  else:
1340
- column = pos - src.rindex('\n', 0, pos)
1341
- return f'line {line}, column {column}'
988
+ return '\n\n'.join(parts)
1342
989
 
1343
- return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
990
+ #
1344
991
 
992
+ @dc.dataclass(frozen=True)
993
+ class Read:
994
+ sz: int
1345
995
 
1346
- def toml_is_unicode_scalar_value(codepoint: int) -> bool:
1347
- return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
996
+ @dc.dataclass(frozen=True)
997
+ class Write:
998
+ d: bytes
1348
999
 
1000
+ class ProtocolError(Exception):
1001
+ pass
1349
1002
 
1350
- def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
1351
- """A decorator to make `parse_float` safe.
1003
+ @dc.dataclass(frozen=True)
1004
+ class Result:
1005
+ pid: int
1006
+ env_info: PyremoteEnvInfo
1352
1007
 
1353
- `parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
1354
- thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
1355
- """
1356
- # The default `float` callable never returns illegal types. Optimize it.
1357
- if parse_float is float:
1358
- return float
1008
+ def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
1009
+ # Read first ack (after fork)
1010
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
1359
1011
 
1360
- def safe_parse_float(float_str: str) -> ta.Any:
1361
- float_value = parse_float(float_str)
1362
- if isinstance(float_value, (dict, list)):
1363
- raise ValueError('parse_float must not return dicts or lists') # noqa
1364
- return float_value
1012
+ # Read pid
1013
+ d = yield from self._read(8)
1014
+ pid = struct.unpack('<Q', d)[0]
1365
1015
 
1366
- return safe_parse_float
1016
+ # Write payload src
1017
+ yield from self._write(struct.pack('<I', len(self._payload_z)))
1018
+ yield from self._write(self._payload_z)
1367
1019
 
1020
+ # Read second ack (after writing src copies)
1021
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
1368
1022
 
1369
- ########################################
1370
- # ../config.py
1023
+ # Write options
1024
+ yield from self._write(struct.pack('<I', len(self._options_json)))
1025
+ yield from self._write(self._options_json)
1371
1026
 
1027
+ # Read third ack (after reaping child process)
1028
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
1372
1029
 
1373
- @dc.dataclass(frozen=True)
1374
- class MainConfig:
1375
- log_level: ta.Optional[str] = 'INFO'
1030
+ # Read env info
1031
+ d = yield from self._read(4)
1032
+ env_info_json_len = struct.unpack('<I', d)[0]
1033
+ d = yield from self._read(env_info_json_len)
1034
+ env_info_json = d.decode('utf-8')
1035
+ env_info = PyremoteEnvInfo(**json.loads(env_info_json))
1376
1036
 
1377
- debug: bool = False
1037
+ # Read fourth ack (after finalization completed)
1038
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
1378
1039
 
1040
+ # Return
1041
+ return self.Result(
1042
+ pid=pid,
1043
+ env_info=env_info,
1044
+ )
1379
1045
 
1380
- ########################################
1381
- # ../deploy/config.py
1046
+ def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
1047
+ d = yield self.Read(sz)
1048
+ if not isinstance(d, bytes):
1049
+ raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
1050
+ if len(d) != sz:
1051
+ raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
1052
+ return d
1382
1053
 
1054
+ def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
1055
+ d = yield from self._read(len(e))
1056
+ if d != e:
1057
+ raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
1383
1058
 
1384
- ##
1059
+ def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
1060
+ i = yield self.Write(d)
1061
+ if i is not None:
1062
+ raise self.ProtocolError('Unexpected input after write')
1385
1063
 
1064
+ #
1386
1065
 
1387
- @dc.dataclass(frozen=True)
1388
- class DeployConfig:
1389
- pass
1066
+ def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
1067
+ gen = self.gen()
1390
1068
 
1069
+ gi: ta.Optional[bytes] = None
1070
+ while True:
1071
+ try:
1072
+ if gi is not None:
1073
+ go = gen.send(gi)
1074
+ else:
1075
+ go = next(gen)
1076
+ except StopIteration as e:
1077
+ return e.value
1391
1078
 
1392
- ########################################
1393
- # ../deploy/paths/types.py
1079
+ if isinstance(go, self.Read):
1080
+ if len(gi := input.read(go.sz)) != go.sz:
1081
+ raise EOFError
1082
+ elif isinstance(go, self.Write):
1083
+ gi = None
1084
+ output.write(go.d)
1085
+ output.flush()
1086
+ else:
1087
+ raise TypeError(go)
1394
1088
 
1089
+ async def async_run(
1090
+ self,
1091
+ input: ta.Any, # asyncio.StreamWriter # noqa
1092
+ output: ta.Any, # asyncio.StreamReader
1093
+ ) -> Result:
1094
+ gen = self.gen()
1395
1095
 
1396
- ##
1096
+ gi: ta.Optional[bytes] = None
1097
+ while True:
1098
+ try:
1099
+ if gi is not None:
1100
+ go = gen.send(gi)
1101
+ else:
1102
+ go = next(gen)
1103
+ except StopIteration as e:
1104
+ return e.value
1105
+
1106
+ if isinstance(go, self.Read):
1107
+ if len(gi := await input.read(go.sz)) != go.sz:
1108
+ raise EOFError
1109
+ elif isinstance(go, self.Write):
1110
+ gi = None
1111
+ output.write(go.d)
1112
+ await output.drain()
1113
+ else:
1114
+ raise TypeError(go)
1397
1115
 
1398
1116
 
1399
1117
  ########################################
1400
- # ../deploy/types.py
1118
+ # ../../../omlish/asyncs/asyncio/channels.py
1401
1119
 
1402
1120
 
1403
- ##
1121
+ class AsyncioBytesChannelTransport(asyncio.Transport):
1122
+ def __init__(self, reader: asyncio.StreamReader) -> None:
1123
+ super().__init__()
1404
1124
 
1125
+ self.reader = reader
1126
+ self.closed: asyncio.Future = asyncio.Future()
1405
1127
 
1406
- DeployHome = ta.NewType('DeployHome', str)
1128
+ # @ta.override
1129
+ def write(self, data: bytes) -> None:
1130
+ self.reader.feed_data(data)
1407
1131
 
1408
- DeployRev = ta.NewType('DeployRev', str)
1132
+ # @ta.override
1133
+ def close(self) -> None:
1134
+ self.reader.feed_eof()
1135
+ if not self.closed.done():
1136
+ self.closed.set_result(True)
1409
1137
 
1138
+ # @ta.override
1139
+ def is_closing(self) -> bool:
1140
+ return self.closed.done()
1410
1141
 
1411
- ########################################
1412
- # ../../pyremote.py
1413
- """
1414
- Basically this: https://mitogen.networkgenomics.com/howitworks.html
1415
1142
 
1416
- TODO:
1417
- - log: ta.Optional[logging.Logger] = None + log.debug's
1418
- """
1143
+ def asyncio_create_bytes_channel(
1144
+ loop: ta.Any = None,
1145
+ ) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
1146
+ if loop is None:
1147
+ loop = asyncio.get_running_loop()
1419
1148
 
1149
+ reader = asyncio.StreamReader()
1150
+ protocol = asyncio.StreamReaderProtocol(reader)
1151
+ transport = AsyncioBytesChannelTransport(reader)
1152
+ writer = asyncio.StreamWriter(transport, protocol, reader, loop)
1420
1153
 
1421
- ##
1154
+ return reader, writer
1422
1155
 
1423
1156
 
1424
- @dc.dataclass(frozen=True)
1425
- class PyremoteBootstrapOptions:
1426
- debug: bool = False
1157
+ ########################################
1158
+ # ../../../omlish/asyncs/asyncio/streams.py
1427
1159
 
1428
- DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
1429
- main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
1430
1160
 
1161
+ ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
1431
1162
 
1432
- ##
1433
1163
 
1164
+ async def asyncio_open_stream_reader(
1165
+ f: ta.IO,
1166
+ loop: ta.Any = None,
1167
+ *,
1168
+ limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1169
+ ) -> asyncio.StreamReader:
1170
+ if loop is None:
1171
+ loop = asyncio.get_running_loop()
1434
1172
 
1435
- @dc.dataclass(frozen=True)
1436
- class PyremoteEnvInfo:
1437
- sys_base_prefix: str
1438
- sys_byteorder: str
1439
- sys_defaultencoding: str
1440
- sys_exec_prefix: str
1441
- sys_executable: str
1442
- sys_implementation_name: str
1443
- sys_path: ta.List[str]
1444
- sys_platform: str
1445
- sys_prefix: str
1446
- sys_version: str
1447
- sys_version_info: ta.List[ta.Union[int, str]]
1173
+ reader = asyncio.StreamReader(limit=limit, loop=loop)
1174
+ await loop.connect_read_pipe(
1175
+ lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1176
+ f,
1177
+ )
1448
1178
 
1449
- platform_architecture: ta.List[str]
1450
- platform_machine: str
1451
- platform_platform: str
1452
- platform_processor: str
1453
- platform_system: str
1454
- platform_release: str
1455
- platform_version: str
1179
+ return reader
1456
1180
 
1457
- site_userbase: str
1458
1181
 
1459
- os_cwd: str
1460
- os_gid: int
1461
- os_loadavg: ta.List[float]
1462
- os_login: ta.Optional[str]
1463
- os_pgrp: int
1464
- os_pid: int
1465
- os_ppid: int
1466
- os_uid: int
1182
+ async def asyncio_open_stream_writer(
1183
+ f: ta.IO,
1184
+ loop: ta.Any = None,
1185
+ ) -> asyncio.StreamWriter:
1186
+ if loop is None:
1187
+ loop = asyncio.get_running_loop()
1467
1188
 
1468
- pw_name: str
1469
- pw_uid: int
1470
- pw_gid: int
1471
- pw_gecos: str
1472
- pw_dir: str
1473
- pw_shell: str
1189
+ writer_transport, writer_protocol = await loop.connect_write_pipe(
1190
+ lambda: asyncio.streams.FlowControlMixin(loop=loop),
1191
+ f,
1192
+ )
1474
1193
 
1475
- env_path: ta.Optional[str]
1194
+ return asyncio.streams.StreamWriter(
1195
+ writer_transport,
1196
+ writer_protocol,
1197
+ None,
1198
+ loop,
1199
+ )
1476
1200
 
1477
1201
 
1478
- def _get_pyremote_env_info() -> PyremoteEnvInfo:
1479
- os_uid = os.getuid()
1202
+ ########################################
1203
+ # ../../../omlish/asyncs/asyncio/timeouts.py
1480
1204
 
1481
- pw = pwd.getpwuid(os_uid)
1482
1205
 
1483
- os_login: ta.Optional[str]
1484
- try:
1485
- os_login = os.getlogin()
1486
- except OSError:
1487
- os_login = None
1206
+ def asyncio_maybe_timeout(
1207
+ fut: AwaitableT,
1208
+ timeout: ta.Optional[float] = None,
1209
+ ) -> AwaitableT:
1210
+ if timeout is not None:
1211
+ fut = asyncio.wait_for(fut, timeout) # type: ignore
1212
+ return fut
1488
1213
 
1489
- return PyremoteEnvInfo(
1490
- sys_base_prefix=sys.base_prefix,
1491
- sys_byteorder=sys.byteorder,
1492
- sys_defaultencoding=sys.getdefaultencoding(),
1493
- sys_exec_prefix=sys.exec_prefix,
1494
- sys_executable=sys.executable,
1495
- sys_implementation_name=sys.implementation.name,
1496
- sys_path=sys.path,
1497
- sys_platform=sys.platform,
1498
- sys_prefix=sys.prefix,
1499
- sys_version=sys.version,
1500
- sys_version_info=list(sys.version_info),
1501
1214
 
1502
- platform_architecture=list(platform.architecture()),
1503
- platform_machine=platform.machine(),
1504
- platform_platform=platform.platform(),
1505
- platform_processor=platform.processor(),
1506
- platform_system=platform.system(),
1507
- platform_release=platform.release(),
1508
- platform_version=platform.version(),
1215
+ ########################################
1216
+ # ../../../omlish/formats/ini/sections.py
1509
1217
 
1510
- site_userbase=site.getuserbase(),
1511
1218
 
1512
- os_cwd=os.getcwd(),
1513
- os_gid=os.getgid(),
1514
- os_loadavg=list(os.getloadavg()),
1515
- os_login=os_login,
1516
- os_pgrp=os.getpgrp(),
1517
- os_pid=os.getpid(),
1518
- os_ppid=os.getppid(),
1519
- os_uid=os_uid,
1219
+ ##
1520
1220
 
1521
- pw_name=pw.pw_name,
1522
- pw_uid=pw.pw_uid,
1523
- pw_gid=pw.pw_gid,
1524
- pw_gecos=pw.pw_gecos,
1525
- pw_dir=pw.pw_dir,
1526
- pw_shell=pw.pw_shell,
1527
1221
 
1528
- env_path=os.environ.get('PATH'),
1529
- )
1222
+ def extract_ini_sections(cp: configparser.ConfigParser) -> IniSectionSettingsMap:
1223
+ config_dct: ta.Dict[str, ta.Any] = {}
1224
+ for sec in cp.sections():
1225
+ cd = config_dct
1226
+ for k in sec.split('.'):
1227
+ cd = cd.setdefault(k, {})
1228
+ cd.update(cp.items(sec))
1229
+ return config_dct
1530
1230
 
1531
1231
 
1532
1232
  ##
1533
1233
 
1534
1234
 
1535
- _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
1536
- _PYREMOTE_BOOTSTRAP_SRC_FD = 101
1537
-
1538
- _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
1539
- _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
1540
- _PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
1541
- _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
1542
- _PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
1543
-
1544
- _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
1545
- _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
1546
- _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
1547
- _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
1548
-
1549
- _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
1235
+ def render_ini_sections(
1236
+ settings_by_section: IniSectionSettingsMap,
1237
+ ) -> str:
1238
+ out = io.StringIO()
1550
1239
 
1551
- _PYREMOTE_BOOTSTRAP_IMPORTS = [
1552
- 'base64',
1553
- 'os',
1554
- 'struct',
1555
- 'sys',
1556
- 'zlib',
1557
- ]
1240
+ for i, (section, settings) in enumerate(settings_by_section.items()):
1241
+ if i:
1242
+ out.write('\n')
1558
1243
 
1244
+ out.write(f'[{section}]\n')
1559
1245
 
1560
- def _pyremote_bootstrap_main(context_name: str) -> None:
1561
- # Get pid
1562
- pid = os.getpid()
1246
+ for k, v in settings.items():
1247
+ if isinstance(v, str):
1248
+ out.write(f'{k}={v}\n')
1249
+ else:
1250
+ for vv in v:
1251
+ out.write(f'{k}={vv}\n')
1563
1252
 
1564
- # Two copies of payload src to be sent to parent
1565
- r0, w0 = os.pipe()
1566
- r1, w1 = os.pipe()
1253
+ return out.getvalue()
1567
1254
 
1568
- if (cp := os.fork()):
1569
- # Parent process
1570
1255
 
1571
- # Dup original stdin to comm_fd for use as comm channel
1572
- os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
1256
+ ########################################
1257
+ # ../../../omlish/formats/toml/parser.py
1258
+ # SPDX-License-Identifier: MIT
1259
+ # SPDX-FileCopyrightText: 2021 Taneli Hukkinen
1260
+ # Licensed to PSF under a Contributor Agreement.
1261
+ #
1262
+ # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
1263
+ # --------------------------------------------
1264
+ #
1265
+ # 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
1266
+ # ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
1267
+ # documentation.
1268
+ #
1269
+ # 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
1270
+ # royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
1271
+ # works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
1272
+ # Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
1273
+ # 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
1274
+ # Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
1275
+ #
1276
+ # 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
1277
+ # wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
1278
+ # any such work a brief summary of the changes made to Python.
1279
+ #
1280
+ # 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
1281
+ # EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
1282
+ # OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
1283
+ # RIGHTS.
1284
+ #
1285
+ # 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
1286
+ # DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
1287
+ # ADVISED OF THE POSSIBILITY THEREOF.
1288
+ #
1289
+ # 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
1290
+ #
1291
+ # 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
1292
+ # venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
1293
+ # name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
1294
+ #
1295
+ # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
1296
+ # License Agreement.
1297
+ #
1298
+ # https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
1573
1299
 
1574
- # Overwrite stdin (fed to python repl) with first copy of src
1575
- os.dup2(r0, 0)
1576
1300
 
1577
- # Dup second copy of src to src_fd to recover after launch
1578
- os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
1301
+ ##
1579
1302
 
1580
- # Close remaining fd's
1581
- for f in [r0, w0, r1, w1]:
1582
- os.close(f)
1583
1303
 
1584
- # Save vars
1585
- env = os.environ
1586
- exe = sys.executable
1587
- env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
1588
- env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
1589
- env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
1304
+ _TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
1590
1305
 
1591
- # Start repl reading stdin from r0
1592
- os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
1306
+ TOML_RE_NUMBER = re.compile(
1307
+ r"""
1308
+ 0
1309
+ (?:
1310
+ x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
1311
+ |
1312
+ b[01](?:_?[01])* # bin
1313
+ |
1314
+ o[0-7](?:_?[0-7])* # oct
1315
+ )
1316
+ |
1317
+ [+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
1318
+ (?P<floatpart>
1319
+ (?:\.[0-9](?:_?[0-9])*)? # optional fractional part
1320
+ (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
1321
+ )
1322
+ """,
1323
+ flags=re.VERBOSE,
1324
+ )
1325
+ TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
1326
+ TOML_RE_DATETIME = re.compile(
1327
+ rf"""
1328
+ ([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
1329
+ (?:
1330
+ [Tt ]
1331
+ {_TOML_TIME_RE_STR}
1332
+ (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
1333
+ )?
1334
+ """,
1335
+ flags=re.VERBOSE,
1336
+ )
1593
1337
 
1594
- else:
1595
- # Child process
1596
1338
 
1597
- # Write first ack
1598
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
1339
+ def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
1340
+ """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
1599
1341
 
1600
- # Write pid
1601
- os.write(1, struct.pack('<Q', pid))
1342
+ Raises ValueError if the match does not correspond to a valid date or datetime.
1343
+ """
1344
+ (
1345
+ year_str,
1346
+ month_str,
1347
+ day_str,
1348
+ hour_str,
1349
+ minute_str,
1350
+ sec_str,
1351
+ micros_str,
1352
+ zulu_time,
1353
+ offset_sign_str,
1354
+ offset_hour_str,
1355
+ offset_minute_str,
1356
+ ) = match.groups()
1357
+ year, month, day = int(year_str), int(month_str), int(day_str)
1358
+ if hour_str is None:
1359
+ return datetime.date(year, month, day)
1360
+ hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
1361
+ micros = int(micros_str.ljust(6, '0')) if micros_str else 0
1362
+ if offset_sign_str:
1363
+ tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
1364
+ offset_hour_str, offset_minute_str, offset_sign_str,
1365
+ )
1366
+ elif zulu_time:
1367
+ tz = datetime.UTC
1368
+ else: # local date-time
1369
+ tz = None
1370
+ return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
1602
1371
 
1603
- # Read payload src from stdin
1604
- payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
1605
- if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
1606
- raise EOFError
1607
- payload_src = zlib.decompress(payload_z)
1608
1372
 
1609
- # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
1610
- # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
1611
- for w in [w0, w1]:
1612
- fp = os.fdopen(w, 'wb', 0)
1613
- fp.write(payload_src)
1614
- fp.close()
1373
+ @functools.lru_cache() # noqa
1374
+ def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
1375
+ sign = 1 if sign_str == '+' else -1
1376
+ return datetime.timezone(
1377
+ datetime.timedelta(
1378
+ hours=sign * int(hour_str),
1379
+ minutes=sign * int(minute_str),
1380
+ ),
1381
+ )
1615
1382
 
1616
- # Write second ack
1617
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
1618
1383
 
1619
- # Exit child
1620
- sys.exit(0)
1384
+ def toml_match_to_localtime(match: re.Match) -> datetime.time:
1385
+ hour_str, minute_str, sec_str, micros_str = match.groups()
1386
+ micros = int(micros_str.ljust(6, '0')) if micros_str else 0
1387
+ return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
1621
1388
 
1622
1389
 
1623
- ##
1390
+ def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
1391
+ if match.group('floatpart'):
1392
+ return parse_float(match.group())
1393
+ return int(match.group(), 0)
1624
1394
 
1625
1395
 
1626
- def pyremote_build_bootstrap_cmd(context_name: str) -> str:
1627
- if any(c in context_name for c in '\'"'):
1628
- raise NameError(context_name)
1396
+ TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
1629
1397
 
1630
- import inspect
1631
- import textwrap
1632
- bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
1398
+ # Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
1399
+ # functions.
1400
+ TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
1401
+ TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
1633
1402
 
1634
- for gl in [
1635
- '_PYREMOTE_BOOTSTRAP_INPUT_FD',
1636
- '_PYREMOTE_BOOTSTRAP_SRC_FD',
1403
+ TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
1404
+ TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1637
1405
 
1638
- '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
1639
- '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
1640
- '_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
1406
+ TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
1641
1407
 
1642
- '_PYREMOTE_BOOTSTRAP_ACK0',
1643
- '_PYREMOTE_BOOTSTRAP_ACK1',
1408
+ TOML_WS = frozenset(' \t')
1409
+ TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
1410
+ TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
1411
+ TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
1412
+ TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
1644
1413
 
1645
- '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
1646
- ]:
1647
- bs_src = bs_src.replace(gl, repr(globals()[gl]))
1414
+ TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
1415
+ {
1416
+ '\\b': '\u0008', # backspace
1417
+ '\\t': '\u0009', # tab
1418
+ '\\n': '\u000A', # linefeed
1419
+ '\\f': '\u000C', # form feed
1420
+ '\\r': '\u000D', # carriage return
1421
+ '\\"': '\u0022', # quote
1422
+ '\\\\': '\u005C', # backslash
1423
+ },
1424
+ )
1648
1425
 
1649
- bs_src = '\n'.join(
1650
- cl
1651
- for l in bs_src.splitlines()
1652
- if (cl := (l.split('#')[0]).rstrip())
1653
- if cl.strip()
1654
- )
1655
1426
 
1656
- bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
1657
- bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
1658
- if b'"' in bs_z85:
1659
- raise ValueError(bs_z85)
1427
+ class TomlDecodeError(ValueError):
1428
+ """An error raised if a document is not valid TOML."""
1660
1429
 
1661
- stmts = [
1662
- f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
1663
- f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
1664
- f'_pyremote_bootstrap_main("{context_name}")',
1665
- ]
1666
1430
 
1667
- cmd = '; '.join(stmts)
1668
- return cmd
1431
+ def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
1432
+ """Parse TOML from a binary file object."""
1433
+ b = fp.read()
1434
+ try:
1435
+ s = b.decode()
1436
+ except AttributeError:
1437
+ raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
1438
+ return toml_loads(s, parse_float=parse_float)
1669
1439
 
1670
1440
 
1671
- ##
1441
+ def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
1442
+ """Parse TOML from a string."""
1672
1443
 
1444
+ # The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
1445
+ try:
1446
+ src = s.replace('\r\n', '\n')
1447
+ except (AttributeError, TypeError):
1448
+ raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
1449
+ pos = 0
1450
+ out = TomlOutput(TomlNestedDict(), TomlFlags())
1451
+ header: TomlKey = ()
1452
+ parse_float = toml_make_safe_parse_float(parse_float)
1673
1453
 
1674
- @dc.dataclass(frozen=True)
1675
- class PyremotePayloadRuntime:
1676
- input: ta.BinaryIO
1677
- output: ta.BinaryIO
1678
- context_name: str
1679
- payload_src: str
1680
- options: PyremoteBootstrapOptions
1681
- env_info: PyremoteEnvInfo
1454
+ # Parse one statement at a time (typically means one line in TOML source)
1455
+ while True:
1456
+ # 1. Skip line leading whitespace
1457
+ pos = toml_skip_chars(src, pos, TOML_WS)
1682
1458
 
1459
+ # 2. Parse rules. Expect one of the following:
1460
+ # - end of file
1461
+ # - end of line
1462
+ # - comment
1463
+ # - key/value pair
1464
+ # - append dict to list (and move to its namespace)
1465
+ # - create dict (and move to its namespace)
1466
+ # Skip trailing whitespace when applicable.
1467
+ try:
1468
+ char = src[pos]
1469
+ except IndexError:
1470
+ break
1471
+ if char == '\n':
1472
+ pos += 1
1473
+ continue
1474
+ if char in TOML_KEY_INITIAL_CHARS:
1475
+ pos = toml_key_value_rule(src, pos, out, header, parse_float)
1476
+ pos = toml_skip_chars(src, pos, TOML_WS)
1477
+ elif char == '[':
1478
+ try:
1479
+ second_char: ta.Optional[str] = src[pos + 1]
1480
+ except IndexError:
1481
+ second_char = None
1482
+ out.flags.finalize_pending()
1483
+ if second_char == '[':
1484
+ pos, header = toml_create_list_rule(src, pos, out)
1485
+ else:
1486
+ pos, header = toml_create_dict_rule(src, pos, out)
1487
+ pos = toml_skip_chars(src, pos, TOML_WS)
1488
+ elif char != '#':
1489
+ raise toml_suffixed_err(src, pos, 'Invalid statement')
1683
1490
 
1684
- def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
1685
- # If src file var is not present we need to do initial finalization
1686
- if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
1687
- # Read second copy of payload src
1688
- r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
1689
- payload_src = r1.read().decode('utf-8')
1690
- r1.close()
1491
+ # 3. Skip comment
1492
+ pos = toml_skip_comment(src, pos)
1691
1493
 
1692
- # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
1693
- # a pipe at once.
1694
- os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
1494
+ # 4. Expect end of line or end of file
1495
+ try:
1496
+ char = src[pos]
1497
+ except IndexError:
1498
+ break
1499
+ if char != '\n':
1500
+ raise toml_suffixed_err(
1501
+ src, pos, 'Expected newline or end of document after a statement',
1502
+ )
1503
+ pos += 1
1695
1504
 
1696
- # Read options
1697
- options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
1698
- if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
1699
- raise EOFError
1700
- options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
1505
+ return out.data.dict
1701
1506
 
1702
- # If debugging, re-exec as file
1703
- if options.debug:
1704
- # Write temp source file
1705
- import tempfile
1706
- tfd, tfn = tempfile.mkstemp('-pyremote.py')
1707
- os.write(tfd, payload_src.encode('utf-8'))
1708
- os.close(tfd)
1709
1507
 
1710
- # Set vars
1711
- os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
1712
- os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
1508
+ class TomlFlags:
1509
+ """Flags that map to parsed keys/namespaces."""
1713
1510
 
1714
- # Re-exec temp file
1715
- exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
1716
- context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
1717
- os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
1511
+ # Marks an immutable namespace (inline array or inline table).
1512
+ FROZEN = 0
1513
+ # Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
1514
+ EXPLICIT_NEST = 1
1718
1515
 
1719
- else:
1720
- # Load options json var
1721
- options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
1722
- options = PyremoteBootstrapOptions(**json.loads(options_json_str))
1516
+ def __init__(self) -> None:
1517
+ self._flags: ta.Dict[str, dict] = {}
1518
+ self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
1723
1519
 
1724
- # Read temp source file
1725
- with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
1726
- payload_src = sf.read()
1520
+ def add_pending(self, key: TomlKey, flag: int) -> None:
1521
+ self._pending_flags.add((key, flag))
1727
1522
 
1728
- # Restore vars
1729
- sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
1730
- context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
1523
+ def finalize_pending(self) -> None:
1524
+ for key, flag in self._pending_flags:
1525
+ self.set(key, flag, recursive=False)
1526
+ self._pending_flags.clear()
1731
1527
 
1732
- # Write third ack
1733
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
1528
+ def unset_all(self, key: TomlKey) -> None:
1529
+ cont = self._flags
1530
+ for k in key[:-1]:
1531
+ if k not in cont:
1532
+ return
1533
+ cont = cont[k]['nested']
1534
+ cont.pop(key[-1], None)
1734
1535
 
1735
- # Write env info
1736
- env_info = _get_pyremote_env_info()
1737
- env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
1738
- os.write(1, struct.pack('<I', len(env_info_json)))
1739
- os.write(1, env_info_json.encode('utf-8'))
1536
+ def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
1537
+ cont = self._flags
1538
+ key_parent, key_stem = key[:-1], key[-1]
1539
+ for k in key_parent:
1540
+ if k not in cont:
1541
+ cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
1542
+ cont = cont[k]['nested']
1543
+ if key_stem not in cont:
1544
+ cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
1545
+ cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
1740
1546
 
1741
- # Setup IO
1742
- input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
1743
- output = os.fdopen(os.dup(1), 'wb', 0) # noqa
1744
- os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
1745
- os.close(nfd)
1547
+ def is_(self, key: TomlKey, flag: int) -> bool:
1548
+ if not key:
1549
+ return False # document root has no flags
1550
+ cont = self._flags
1551
+ for k in key[:-1]:
1552
+ if k not in cont:
1553
+ return False
1554
+ inner_cont = cont[k]
1555
+ if flag in inner_cont['recursive_flags']:
1556
+ return True
1557
+ cont = inner_cont['nested']
1558
+ key_stem = key[-1]
1559
+ if key_stem in cont:
1560
+ cont = cont[key_stem]
1561
+ return flag in cont['flags'] or flag in cont['recursive_flags']
1562
+ return False
1746
1563
 
1747
- if (mn := options.main_name_override) is not None:
1748
- # Inspections like typing.get_type_hints need an entry in sys.modules.
1749
- sys.modules[mn] = sys.modules['__main__']
1750
1564
 
1751
- # Write fourth ack
1752
- output.write(_PYREMOTE_BOOTSTRAP_ACK3)
1565
+ class TomlNestedDict:
1566
+ def __init__(self) -> None:
1567
+ # The parsed content of the TOML document
1568
+ self.dict: ta.Dict[str, ta.Any] = {}
1753
1569
 
1754
- # Return
1755
- return PyremotePayloadRuntime(
1756
- input=input,
1757
- output=output,
1758
- context_name=context_name,
1759
- payload_src=payload_src,
1760
- options=options,
1761
- env_info=env_info,
1762
- )
1570
+ def get_or_create_nest(
1571
+ self,
1572
+ key: TomlKey,
1573
+ *,
1574
+ access_lists: bool = True,
1575
+ ) -> dict:
1576
+ cont: ta.Any = self.dict
1577
+ for k in key:
1578
+ if k not in cont:
1579
+ cont[k] = {}
1580
+ cont = cont[k]
1581
+ if access_lists and isinstance(cont, list):
1582
+ cont = cont[-1]
1583
+ if not isinstance(cont, dict):
1584
+ raise KeyError('There is no nest behind this key')
1585
+ return cont
1763
1586
 
1587
+ def append_nest_to_list(self, key: TomlKey) -> None:
1588
+ cont = self.get_or_create_nest(key[:-1])
1589
+ last_key = key[-1]
1590
+ if last_key in cont:
1591
+ list_ = cont[last_key]
1592
+ if not isinstance(list_, list):
1593
+ raise KeyError('An object other than list found behind this key')
1594
+ list_.append({})
1595
+ else:
1596
+ cont[last_key] = [{}]
1764
1597
 
1765
- ##
1766
1598
 
1599
+ class TomlOutput(ta.NamedTuple):
1600
+ data: TomlNestedDict
1601
+ flags: TomlFlags
1767
1602
 
1768
- class PyremoteBootstrapDriver:
1769
- def __init__(
1770
- self,
1771
- payload_src: ta.Union[str, ta.Sequence[str]],
1772
- options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
1773
- ) -> None:
1774
- super().__init__()
1775
1603
 
1776
- self._payload_src = payload_src
1777
- self._options = options
1604
+ def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
1605
+ try:
1606
+ while src[pos] in chars:
1607
+ pos += 1
1608
+ except IndexError:
1609
+ pass
1610
+ return pos
1778
1611
 
1779
- self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
1780
- self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
1781
1612
 
1782
- self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
1783
- #
1613
+ def toml_skip_until(
1614
+ src: str,
1615
+ pos: TomlPos,
1616
+ expect: str,
1617
+ *,
1618
+ error_on: ta.FrozenSet[str],
1619
+ error_on_eof: bool,
1620
+ ) -> TomlPos:
1621
+ try:
1622
+ new_pos = src.index(expect, pos)
1623
+ except ValueError:
1624
+ new_pos = len(src)
1625
+ if error_on_eof:
1626
+ raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
1784
1627
 
1785
- @classmethod
1786
- def _prepare_payload_src(
1787
- cls,
1788
- payload_src: ta.Union[str, ta.Sequence[str]],
1789
- options: PyremoteBootstrapOptions,
1790
- ) -> str:
1791
- parts: ta.List[str]
1792
- if isinstance(payload_src, str):
1793
- parts = [payload_src]
1794
- else:
1795
- parts = list(payload_src)
1628
+ if not error_on.isdisjoint(src[pos:new_pos]):
1629
+ while src[pos] not in error_on:
1630
+ pos += 1
1631
+ raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
1632
+ return new_pos
1796
1633
 
1797
- if (mn := options.main_name_override) is not None:
1798
- parts.insert(0, f'__name__ = {mn!r}')
1799
1634
 
1800
- if len(parts) == 1:
1801
- return parts[0]
1802
- else:
1803
- return '\n\n'.join(parts)
1635
+ def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
1636
+ try:
1637
+ char: ta.Optional[str] = src[pos]
1638
+ except IndexError:
1639
+ char = None
1640
+ if char == '#':
1641
+ return toml_skip_until(
1642
+ src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
1643
+ )
1644
+ return pos
1804
1645
 
1805
- #
1806
1646
 
1807
- @dc.dataclass(frozen=True)
1808
- class Read:
1809
- sz: int
1647
+ def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
1648
+ while True:
1649
+ pos_before_skip = pos
1650
+ pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1651
+ pos = toml_skip_comment(src, pos)
1652
+ if pos == pos_before_skip:
1653
+ return pos
1810
1654
 
1811
- @dc.dataclass(frozen=True)
1812
- class Write:
1813
- d: bytes
1814
1655
 
1815
- class ProtocolError(Exception):
1816
- pass
1656
+ def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
1657
+ pos += 1 # Skip "["
1658
+ pos = toml_skip_chars(src, pos, TOML_WS)
1659
+ pos, key = toml_parse_key(src, pos)
1817
1660
 
1818
- @dc.dataclass(frozen=True)
1819
- class Result:
1820
- pid: int
1821
- env_info: PyremoteEnvInfo
1661
+ if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
1662
+ raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
1663
+ out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
1664
+ try:
1665
+ out.data.get_or_create_nest(key)
1666
+ except KeyError:
1667
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1822
1668
 
1823
- def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
1824
- # Read first ack (after fork)
1825
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
1669
+ if not src.startswith(']', pos):
1670
+ raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
1671
+ return pos + 1, key
1826
1672
 
1827
- # Read pid
1828
- d = yield from self._read(8)
1829
- pid = struct.unpack('<Q', d)[0]
1830
1673
 
1831
- # Write payload src
1832
- yield from self._write(struct.pack('<I', len(self._payload_z)))
1833
- yield from self._write(self._payload_z)
1674
+ def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
1675
+ pos += 2 # Skip "[["
1676
+ pos = toml_skip_chars(src, pos, TOML_WS)
1677
+ pos, key = toml_parse_key(src, pos)
1834
1678
 
1835
- # Read second ack (after writing src copies)
1836
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
1679
+ if out.flags.is_(key, TomlFlags.FROZEN):
1680
+ raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1681
+ # Free the namespace now that it points to another empty list item...
1682
+ out.flags.unset_all(key)
1683
+ # ...but this key precisely is still prohibited from table declaration
1684
+ out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
1685
+ try:
1686
+ out.data.append_nest_to_list(key)
1687
+ except KeyError:
1688
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1837
1689
 
1838
- # Write options
1839
- yield from self._write(struct.pack('<I', len(self._options_json)))
1840
- yield from self._write(self._options_json)
1690
+ if not src.startswith(']]', pos):
1691
+ raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
1692
+ return pos + 2, key
1841
1693
 
1842
- # Read third ack (after reaping child process)
1843
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
1844
1694
 
1845
- # Read env info
1846
- d = yield from self._read(4)
1847
- env_info_json_len = struct.unpack('<I', d)[0]
1848
- d = yield from self._read(env_info_json_len)
1849
- env_info_json = d.decode('utf-8')
1850
- env_info = PyremoteEnvInfo(**json.loads(env_info_json))
1695
+ def toml_key_value_rule(
1696
+ src: str,
1697
+ pos: TomlPos,
1698
+ out: TomlOutput,
1699
+ header: TomlKey,
1700
+ parse_float: TomlParseFloat,
1701
+ ) -> TomlPos:
1702
+ pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1703
+ key_parent, key_stem = key[:-1], key[-1]
1704
+ abs_key_parent = header + key_parent
1851
1705
 
1852
- # Read fourth ack (after finalization completed)
1853
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
1706
+ relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
1707
+ for cont_key in relative_path_cont_keys:
1708
+ # Check that dotted key syntax does not redefine an existing table
1709
+ if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
1710
+ raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
1711
+ # Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
1712
+ # table sections.
1713
+ out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
1854
1714
 
1855
- # Return
1856
- return self.Result(
1857
- pid=pid,
1858
- env_info=env_info,
1715
+ if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
1716
+ raise toml_suffixed_err(
1717
+ src,
1718
+ pos,
1719
+ f'Cannot mutate immutable namespace {abs_key_parent}',
1859
1720
  )
1860
1721
 
1861
- def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
1862
- d = yield self.Read(sz)
1863
- if not isinstance(d, bytes):
1864
- raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
1865
- if len(d) != sz:
1866
- raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
1867
- return d
1722
+ try:
1723
+ nest = out.data.get_or_create_nest(abs_key_parent)
1724
+ except KeyError:
1725
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1726
+ if key_stem in nest:
1727
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
1728
+ # Mark inline table and array namespaces recursively immutable
1729
+ if isinstance(value, (dict, list)):
1730
+ out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
1731
+ nest[key_stem] = value
1732
+ return pos
1868
1733
 
1869
- def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
1870
- d = yield from self._read(len(e))
1871
- if d != e:
1872
- raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
1873
1734
 
1874
- def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
1875
- i = yield self.Write(d)
1876
- if i is not None:
1877
- raise self.ProtocolError('Unexpected input after write')
1735
+ def toml_parse_key_value_pair(
1736
+ src: str,
1737
+ pos: TomlPos,
1738
+ parse_float: TomlParseFloat,
1739
+ ) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
1740
+ pos, key = toml_parse_key(src, pos)
1741
+ try:
1742
+ char: ta.Optional[str] = src[pos]
1743
+ except IndexError:
1744
+ char = None
1745
+ if char != '=':
1746
+ raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
1747
+ pos += 1
1748
+ pos = toml_skip_chars(src, pos, TOML_WS)
1749
+ pos, value = toml_parse_value(src, pos, parse_float)
1750
+ return pos, key, value
1878
1751
 
1879
- #
1880
1752
 
1881
- def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
1882
- gen = self.gen()
1753
+ def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
1754
+ pos, key_part = toml_parse_key_part(src, pos)
1755
+ key: TomlKey = (key_part,)
1756
+ pos = toml_skip_chars(src, pos, TOML_WS)
1757
+ while True:
1758
+ try:
1759
+ char: ta.Optional[str] = src[pos]
1760
+ except IndexError:
1761
+ char = None
1762
+ if char != '.':
1763
+ return pos, key
1764
+ pos += 1
1765
+ pos = toml_skip_chars(src, pos, TOML_WS)
1766
+ pos, key_part = toml_parse_key_part(src, pos)
1767
+ key += (key_part,)
1768
+ pos = toml_skip_chars(src, pos, TOML_WS)
1769
+
1770
+
1771
+ def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1772
+ try:
1773
+ char: ta.Optional[str] = src[pos]
1774
+ except IndexError:
1775
+ char = None
1776
+ if char in TOML_BARE_KEY_CHARS:
1777
+ start_pos = pos
1778
+ pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
1779
+ return pos, src[start_pos:pos]
1780
+ if char == "'":
1781
+ return toml_parse_literal_str(src, pos)
1782
+ if char == '"':
1783
+ return toml_parse_one_line_basic_str(src, pos)
1784
+ raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
1785
+
1786
+
1787
+ def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1788
+ pos += 1
1789
+ return toml_parse_basic_str(src, pos, multiline=False)
1790
+
1791
+
1792
+ def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
1793
+ pos += 1
1794
+ array: list = []
1795
+
1796
+ pos = toml_skip_comments_and_array_ws(src, pos)
1797
+ if src.startswith(']', pos):
1798
+ return pos + 1, array
1799
+ while True:
1800
+ pos, val = toml_parse_value(src, pos, parse_float)
1801
+ array.append(val)
1802
+ pos = toml_skip_comments_and_array_ws(src, pos)
1803
+
1804
+ c = src[pos:pos + 1]
1805
+ if c == ']':
1806
+ return pos + 1, array
1807
+ if c != ',':
1808
+ raise toml_suffixed_err(src, pos, 'Unclosed array')
1809
+ pos += 1
1810
+
1811
+ pos = toml_skip_comments_and_array_ws(src, pos)
1812
+ if src.startswith(']', pos):
1813
+ return pos + 1, array
1814
+
1815
+
1816
+ def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
1817
+ pos += 1
1818
+ nested_dict = TomlNestedDict()
1819
+ flags = TomlFlags()
1820
+
1821
+ pos = toml_skip_chars(src, pos, TOML_WS)
1822
+ if src.startswith('}', pos):
1823
+ return pos + 1, nested_dict.dict
1824
+ while True:
1825
+ pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1826
+ key_parent, key_stem = key[:-1], key[-1]
1827
+ if flags.is_(key, TomlFlags.FROZEN):
1828
+ raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1829
+ try:
1830
+ nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
1831
+ except KeyError:
1832
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1833
+ if key_stem in nest:
1834
+ raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
1835
+ nest[key_stem] = value
1836
+ pos = toml_skip_chars(src, pos, TOML_WS)
1837
+ c = src[pos:pos + 1]
1838
+ if c == '}':
1839
+ return pos + 1, nested_dict.dict
1840
+ if c != ',':
1841
+ raise toml_suffixed_err(src, pos, 'Unclosed inline table')
1842
+ if isinstance(value, (dict, list)):
1843
+ flags.set(key, TomlFlags.FROZEN, recursive=True)
1844
+ pos += 1
1845
+ pos = toml_skip_chars(src, pos, TOML_WS)
1846
+
1847
+
1848
+ def toml_parse_basic_str_escape(
1849
+ src: str,
1850
+ pos: TomlPos,
1851
+ *,
1852
+ multiline: bool = False,
1853
+ ) -> ta.Tuple[TomlPos, str]:
1854
+ escape_id = src[pos:pos + 2]
1855
+ pos += 2
1856
+ if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
1857
+ # Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
1858
+ # newline.
1859
+ if escape_id != '\\\n':
1860
+ pos = toml_skip_chars(src, pos, TOML_WS)
1861
+ try:
1862
+ char = src[pos]
1863
+ except IndexError:
1864
+ return pos, ''
1865
+ if char != '\n':
1866
+ raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
1867
+ pos += 1
1868
+ pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1869
+ return pos, ''
1870
+ if escape_id == '\\u':
1871
+ return toml_parse_hex_char(src, pos, 4)
1872
+ if escape_id == '\\U':
1873
+ return toml_parse_hex_char(src, pos, 8)
1874
+ try:
1875
+ return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
1876
+ except KeyError:
1877
+ raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
1878
+
1883
1879
 
1884
- gi: ta.Optional[bytes] = None
1885
- while True:
1886
- try:
1887
- if gi is not None:
1888
- go = gen.send(gi)
1889
- else:
1890
- go = next(gen)
1891
- except StopIteration as e:
1892
- return e.value
1880
+ def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1881
+ return toml_parse_basic_str_escape(src, pos, multiline=True)
1893
1882
 
1894
- if isinstance(go, self.Read):
1895
- if len(gi := input.read(go.sz)) != go.sz:
1896
- raise EOFError
1897
- elif isinstance(go, self.Write):
1898
- gi = None
1899
- output.write(go.d)
1900
- output.flush()
1901
- else:
1902
- raise TypeError(go)
1903
1883
 
1904
- async def async_run(
1905
- self,
1906
- input: ta.Any, # asyncio.StreamWriter # noqa
1907
- output: ta.Any, # asyncio.StreamReader
1908
- ) -> Result:
1909
- gen = self.gen()
1884
+ def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
1885
+ hex_str = src[pos:pos + hex_len]
1886
+ if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
1887
+ raise toml_suffixed_err(src, pos, 'Invalid hex value')
1888
+ pos += hex_len
1889
+ hex_int = int(hex_str, 16)
1890
+ if not toml_is_unicode_scalar_value(hex_int):
1891
+ raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
1892
+ return pos, chr(hex_int)
1910
1893
 
1911
- gi: ta.Optional[bytes] = None
1912
- while True:
1913
- try:
1914
- if gi is not None:
1915
- go = gen.send(gi)
1916
- else:
1917
- go = next(gen)
1918
- except StopIteration as e:
1919
- return e.value
1920
1894
 
1921
- if isinstance(go, self.Read):
1922
- if len(gi := await input.read(go.sz)) != go.sz:
1923
- raise EOFError
1924
- elif isinstance(go, self.Write):
1925
- gi = None
1926
- output.write(go.d)
1927
- await output.drain()
1928
- else:
1929
- raise TypeError(go)
1895
+ def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1896
+ pos += 1 # Skip starting apostrophe
1897
+ start_pos = pos
1898
+ pos = toml_skip_until(
1899
+ src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
1900
+ )
1901
+ return pos + 1, src[start_pos:pos] # Skip ending apostrophe
1930
1902
 
1931
1903
 
1932
- ########################################
1933
- # ../../../omlish/asyncs/asyncio/channels.py
1904
+ def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
1905
+ pos += 3
1906
+ if src.startswith('\n', pos):
1907
+ pos += 1
1934
1908
 
1909
+ if literal:
1910
+ delim = "'"
1911
+ end_pos = toml_skip_until(
1912
+ src,
1913
+ pos,
1914
+ "'''",
1915
+ error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
1916
+ error_on_eof=True,
1917
+ )
1918
+ result = src[pos:end_pos]
1919
+ pos = end_pos + 3
1920
+ else:
1921
+ delim = '"'
1922
+ pos, result = toml_parse_basic_str(src, pos, multiline=True)
1935
1923
 
1936
- class AsyncioBytesChannelTransport(asyncio.Transport):
1937
- def __init__(self, reader: asyncio.StreamReader) -> None:
1938
- super().__init__()
1924
+ # Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
1925
+ if not src.startswith(delim, pos):
1926
+ return pos, result
1927
+ pos += 1
1928
+ if not src.startswith(delim, pos):
1929
+ return pos, result + delim
1930
+ pos += 1
1931
+ return pos, result + (delim * 2)
1939
1932
 
1940
- self.reader = reader
1941
- self.closed: asyncio.Future = asyncio.Future()
1942
1933
 
1943
- # @ta.override
1944
- def write(self, data: bytes) -> None:
1945
- self.reader.feed_data(data)
1934
+ def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
1935
+ if multiline:
1936
+ error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1937
+ parse_escapes = toml_parse_basic_str_escape_multiline
1938
+ else:
1939
+ error_on = TOML_ILLEGAL_BASIC_STR_CHARS
1940
+ parse_escapes = toml_parse_basic_str_escape
1941
+ result = ''
1942
+ start_pos = pos
1943
+ while True:
1944
+ try:
1945
+ char = src[pos]
1946
+ except IndexError:
1947
+ raise toml_suffixed_err(src, pos, 'Unterminated string') from None
1948
+ if char == '"':
1949
+ if not multiline:
1950
+ return pos + 1, result + src[start_pos:pos]
1951
+ if src.startswith('"""', pos):
1952
+ return pos + 3, result + src[start_pos:pos]
1953
+ pos += 1
1954
+ continue
1955
+ if char == '\\':
1956
+ result += src[start_pos:pos]
1957
+ pos, parsed_escape = parse_escapes(src, pos)
1958
+ result += parsed_escape
1959
+ start_pos = pos
1960
+ continue
1961
+ if char in error_on:
1962
+ raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
1963
+ pos += 1
1946
1964
 
1947
- # @ta.override
1948
- def close(self) -> None:
1949
- self.reader.feed_eof()
1950
- if not self.closed.done():
1951
- self.closed.set_result(True)
1952
1965
 
1953
- # @ta.override
1954
- def is_closing(self) -> bool:
1955
- return self.closed.done()
1966
+ def toml_parse_value( # noqa: C901
1967
+ src: str,
1968
+ pos: TomlPos,
1969
+ parse_float: TomlParseFloat,
1970
+ ) -> ta.Tuple[TomlPos, ta.Any]:
1971
+ try:
1972
+ char: ta.Optional[str] = src[pos]
1973
+ except IndexError:
1974
+ char = None
1956
1975
 
1976
+ # IMPORTANT: order conditions based on speed of checking and likelihood
1957
1977
 
1958
- def asyncio_create_bytes_channel(
1959
- loop: ta.Any = None,
1960
- ) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
1961
- if loop is None:
1962
- loop = asyncio.get_running_loop()
1978
+ # Basic strings
1979
+ if char == '"':
1980
+ if src.startswith('"""', pos):
1981
+ return toml_parse_multiline_str(src, pos, literal=False)
1982
+ return toml_parse_one_line_basic_str(src, pos)
1963
1983
 
1964
- reader = asyncio.StreamReader()
1965
- protocol = asyncio.StreamReaderProtocol(reader)
1966
- transport = AsyncioBytesChannelTransport(reader)
1967
- writer = asyncio.StreamWriter(transport, protocol, reader, loop)
1984
+ # Literal strings
1985
+ if char == "'":
1986
+ if src.startswith("'''", pos):
1987
+ return toml_parse_multiline_str(src, pos, literal=True)
1988
+ return toml_parse_literal_str(src, pos)
1968
1989
 
1969
- return reader, writer
1990
+ # Booleans
1991
+ if char == 't':
1992
+ if src.startswith('true', pos):
1993
+ return pos + 4, True
1994
+ if char == 'f':
1995
+ if src.startswith('false', pos):
1996
+ return pos + 5, False
1970
1997
 
1998
+ # Arrays
1999
+ if char == '[':
2000
+ return toml_parse_array(src, pos, parse_float)
1971
2001
 
1972
- ########################################
1973
- # ../../../omlish/asyncs/asyncio/streams.py
2002
+ # Inline tables
2003
+ if char == '{':
2004
+ return toml_parse_inline_table(src, pos, parse_float)
1974
2005
 
2006
+ # Dates and times
2007
+ datetime_match = TOML_RE_DATETIME.match(src, pos)
2008
+ if datetime_match:
2009
+ try:
2010
+ datetime_obj = toml_match_to_datetime(datetime_match)
2011
+ except ValueError as e:
2012
+ raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
2013
+ return datetime_match.end(), datetime_obj
2014
+ localtime_match = TOML_RE_LOCALTIME.match(src, pos)
2015
+ if localtime_match:
2016
+ return localtime_match.end(), toml_match_to_localtime(localtime_match)
1975
2017
 
1976
- ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
2018
+ # Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
2019
+ # located after handling of dates and times.
2020
+ number_match = TOML_RE_NUMBER.match(src, pos)
2021
+ if number_match:
2022
+ return number_match.end(), toml_match_to_number(number_match, parse_float)
1977
2023
 
2024
+ # Special floats
2025
+ first_three = src[pos:pos + 3]
2026
+ if first_three in {'inf', 'nan'}:
2027
+ return pos + 3, parse_float(first_three)
2028
+ first_four = src[pos:pos + 4]
2029
+ if first_four in {'-inf', '+inf', '-nan', '+nan'}:
2030
+ return pos + 4, parse_float(first_four)
1978
2031
 
1979
- async def asyncio_open_stream_reader(
1980
- f: ta.IO,
1981
- loop: ta.Any = None,
1982
- *,
1983
- limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1984
- ) -> asyncio.StreamReader:
1985
- if loop is None:
1986
- loop = asyncio.get_running_loop()
2032
+ raise toml_suffixed_err(src, pos, 'Invalid value')
1987
2033
 
1988
- reader = asyncio.StreamReader(limit=limit, loop=loop)
1989
- await loop.connect_read_pipe(
1990
- lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1991
- f,
1992
- )
1993
2034
 
1994
- return reader
2035
+ def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
2036
+ """Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
1995
2037
 
2038
+ def coord_repr(src: str, pos: TomlPos) -> str:
2039
+ if pos >= len(src):
2040
+ return 'end of document'
2041
+ line = src.count('\n', 0, pos) + 1
2042
+ if line == 1:
2043
+ column = pos + 1
2044
+ else:
2045
+ column = pos - src.rindex('\n', 0, pos)
2046
+ return f'line {line}, column {column}'
1996
2047
 
1997
- async def asyncio_open_stream_writer(
1998
- f: ta.IO,
1999
- loop: ta.Any = None,
2000
- ) -> asyncio.StreamWriter:
2001
- if loop is None:
2002
- loop = asyncio.get_running_loop()
2048
+ return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
2003
2049
 
2004
- writer_transport, writer_protocol = await loop.connect_write_pipe(
2005
- lambda: asyncio.streams.FlowControlMixin(loop=loop),
2006
- f,
2007
- )
2008
2050
 
2009
- return asyncio.streams.StreamWriter(
2010
- writer_transport,
2011
- writer_protocol,
2012
- None,
2013
- loop,
2014
- )
2051
+ def toml_is_unicode_scalar_value(codepoint: int) -> bool:
2052
+ return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
2015
2053
 
2016
2054
 
2017
- ########################################
2018
- # ../../../omlish/asyncs/asyncio/timeouts.py
2055
+ def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
2056
+ """A decorator to make `parse_float` safe.
2057
+
2058
+ `parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
2059
+ thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
2060
+ """
2061
+ # The default `float` callable never returns illegal types. Optimize it.
2062
+ if parse_float is float:
2063
+ return float
2019
2064
 
2065
+ def safe_parse_float(float_str: str) -> ta.Any:
2066
+ float_value = parse_float(float_str)
2067
+ if isinstance(float_value, (dict, list)):
2068
+ raise ValueError('parse_float must not return dicts or lists') # noqa
2069
+ return float_value
2020
2070
 
2021
- def asyncio_maybe_timeout(
2022
- fut: AwaitableT,
2023
- timeout: ta.Optional[float] = None,
2024
- ) -> AwaitableT:
2025
- if timeout is not None:
2026
- fut = asyncio.wait_for(fut, timeout) # type: ignore
2027
- return fut
2071
+ return safe_parse_float
2028
2072
 
2029
2073
 
2030
2074
  ########################################
@@ -6986,30 +7030,6 @@ def build_config_named_children(
6986
7030
  return lst
6987
7031
 
6988
7032
 
6989
- ##
6990
-
6991
-
6992
- def render_ini_config(
6993
- settings_by_section: IniConfigSectionSettingsMap,
6994
- ) -> str:
6995
- out = io.StringIO()
6996
-
6997
- for i, (section, settings) in enumerate(settings_by_section.items()):
6998
- if i:
6999
- out.write('\n')
7000
-
7001
- out.write(f'[{section}]\n')
7002
-
7003
- for k, v in settings.items():
7004
- if isinstance(v, str):
7005
- out.write(f'{k}={v}\n')
7006
- else:
7007
- for vv in v:
7008
- out.write(f'{k}={vv}\n')
7009
-
7010
- return out.getvalue()
7011
-
7012
-
7013
7033
  ########################################
7014
7034
  # ../commands/marshal.py
7015
7035
 
@@ -7063,6 +7083,104 @@ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
7063
7083
  CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
7064
7084
 
7065
7085
 
7086
+ ########################################
7087
+ # ../deploy/conf/specs.py
7088
+
7089
+
7090
+ ##
7091
+
7092
+
7093
+ class DeployAppConfContent(abc.ABC): # noqa
7094
+ pass
7095
+
7096
+
7097
+ #
7098
+
7099
+
7100
+ @register_single_field_type_obj_marshaler('body')
7101
+ @dc.dataclass(frozen=True)
7102
+ class RawDeployAppConfContent(DeployAppConfContent):
7103
+ body: str
7104
+
7105
+
7106
+ #
7107
+
7108
+
7109
+ @register_single_field_type_obj_marshaler('obj')
7110
+ @dc.dataclass(frozen=True)
7111
+ class JsonDeployAppConfContent(DeployAppConfContent):
7112
+ obj: ta.Any
7113
+
7114
+
7115
+ #
7116
+
7117
+
7118
+ @register_single_field_type_obj_marshaler('sections')
7119
+ @dc.dataclass(frozen=True)
7120
+ class IniDeployAppConfContent(DeployAppConfContent):
7121
+ sections: IniSectionSettingsMap
7122
+
7123
+
7124
+ #
7125
+
7126
+
7127
+ @register_single_field_type_obj_marshaler('items')
7128
+ @dc.dataclass(frozen=True)
7129
+ class NginxDeployAppConfContent(DeployAppConfContent):
7130
+ items: ta.Any
7131
+
7132
+
7133
+ ##
7134
+
7135
+
7136
+ @dc.dataclass(frozen=True)
7137
+ class DeployAppConfFile:
7138
+ path: str
7139
+ content: DeployAppConfContent
7140
+
7141
+ def __post_init__(self) -> None:
7142
+ check_valid_deploy_spec_path(self.path)
7143
+
7144
+
7145
+ ##
7146
+
7147
+
7148
+ @dc.dataclass(frozen=True)
7149
+ class DeployAppConfLink: # noqa
7150
+ """
7151
+ May be either:
7152
+ - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
7153
+ - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
7154
+ - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
7155
+ """
7156
+
7157
+ src: str
7158
+
7159
+ kind: ta.Literal['current_only', 'all_active'] = 'current_only'
7160
+
7161
+ def __post_init__(self) -> None:
7162
+ check_valid_deploy_spec_path(self.src)
7163
+ if '/' in self.src:
7164
+ check.equal(self.src.count('/'), 1)
7165
+
7166
+
7167
+ ##
7168
+
7169
+
7170
+ @dc.dataclass(frozen=True)
7171
+ class DeployAppConfSpec:
7172
+ files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
7173
+
7174
+ links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
7175
+
7176
+ def __post_init__(self) -> None:
7177
+ if self.files:
7178
+ seen: ta.Set[str] = set()
7179
+ for f in self.files:
7180
+ check.not_in(f.path, seen)
7181
+ seen.add(f.path)
7182
+
7183
+
7066
7184
  ########################################
7067
7185
  # ../deploy/tags.py
7068
7186
 
@@ -7978,112 +8096,14 @@ class LocalCommandExecutor(CommandExecutor):
7978
8096
  self,
7979
8097
  *,
7980
8098
  command_executors: CommandExecutorMap,
7981
- ) -> None:
7982
- super().__init__()
7983
-
7984
- self._command_executors = command_executors
7985
-
7986
- async def execute(self, cmd: Command) -> Command.Output:
7987
- ce: CommandExecutor = self._command_executors[type(cmd)]
7988
- return await ce.execute(cmd)
7989
-
7990
-
7991
- ########################################
7992
- # ../deploy/conf/specs.py
7993
-
7994
-
7995
- ##
7996
-
7997
-
7998
- class DeployAppConfContent(abc.ABC): # noqa
7999
- pass
8000
-
8001
-
8002
- #
8003
-
8004
-
8005
- @register_single_field_type_obj_marshaler('body')
8006
- @dc.dataclass(frozen=True)
8007
- class RawDeployAppConfContent(DeployAppConfContent):
8008
- body: str
8009
-
8010
-
8011
- #
8012
-
8013
-
8014
- @register_single_field_type_obj_marshaler('obj')
8015
- @dc.dataclass(frozen=True)
8016
- class JsonDeployAppConfContent(DeployAppConfContent):
8017
- obj: ta.Any
8018
-
8019
-
8020
- #
8021
-
8022
-
8023
- @register_single_field_type_obj_marshaler('sections')
8024
- @dc.dataclass(frozen=True)
8025
- class IniDeployAppConfContent(DeployAppConfContent):
8026
- sections: IniConfigSectionSettingsMap
8027
-
8028
-
8029
- #
8030
-
8031
-
8032
- @register_single_field_type_obj_marshaler('items')
8033
- @dc.dataclass(frozen=True)
8034
- class NginxDeployAppConfContent(DeployAppConfContent):
8035
- items: ta.Any
8036
-
8037
-
8038
- ##
8039
-
8040
-
8041
- @dc.dataclass(frozen=True)
8042
- class DeployAppConfFile:
8043
- path: str
8044
- content: DeployAppConfContent
8045
-
8046
- def __post_init__(self) -> None:
8047
- check_valid_deploy_spec_path(self.path)
8048
-
8049
-
8050
- ##
8051
-
8052
-
8053
- @dc.dataclass(frozen=True)
8054
- class DeployAppConfLink: # noqa
8055
- """
8056
- May be either:
8057
- - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
8058
- - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
8059
- - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
8060
- """
8061
-
8062
- src: str
8063
-
8064
- kind: ta.Literal['current_only', 'all_active'] = 'current_only'
8065
-
8066
- def __post_init__(self) -> None:
8067
- check_valid_deploy_spec_path(self.src)
8068
- if '/' in self.src:
8069
- check.equal(self.src.count('/'), 1)
8070
-
8071
-
8072
- ##
8073
-
8074
-
8075
- @dc.dataclass(frozen=True)
8076
- class DeployAppConfSpec:
8077
- files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
8078
-
8079
- links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
8080
-
8081
- def __post_init__(self) -> None:
8082
- if self.files:
8083
- seen: ta.Set[str] = set()
8084
- for f in self.files:
8085
- check.not_in(f.path, seen)
8086
- seen.add(f.path)
8099
+ ) -> None:
8100
+ super().__init__()
8101
+
8102
+ self._command_executors = command_executors
8103
+
8104
+ async def execute(self, cmd: Command) -> Command.Output:
8105
+ ce: CommandExecutor = self._command_executors[type(cmd)]
8106
+ return await ce.execute(cmd)
8087
8107
 
8088
8108
 
8089
8109
  ########################################
@@ -8288,6 +8308,127 @@ class DeployPath(DeployPathRenderable):
8288
8308
  return cls(tuple(DeployPathPart.parse(p) for p in ps))
8289
8309
 
8290
8310
 
8311
+ ########################################
8312
+ # ../deploy/specs.py
8313
+
8314
+
8315
+ ##
8316
+
8317
+
8318
+ class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
8319
+ @cached_nullary
8320
+ def _key_str(self) -> str:
8321
+ return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
8322
+
8323
+ @abc.abstractmethod
8324
+ def key(self) -> KeyDeployTagT:
8325
+ raise NotImplementedError
8326
+
8327
+
8328
+ ##
8329
+
8330
+
8331
+ @dc.dataclass(frozen=True)
8332
+ class DeployGitRepo:
8333
+ host: ta.Optional[str] = None
8334
+ username: ta.Optional[str] = None
8335
+ path: ta.Optional[str] = None
8336
+
8337
+ def __post_init__(self) -> None:
8338
+ check.not_in('..', check.non_empty_str(self.host))
8339
+ check.not_in('.', check.non_empty_str(self.path))
8340
+
8341
+
8342
+ @dc.dataclass(frozen=True)
8343
+ class DeployGitSpec:
8344
+ repo: DeployGitRepo
8345
+ rev: DeployRev
8346
+
8347
+ subtrees: ta.Optional[ta.Sequence[str]] = None
8348
+
8349
+ def __post_init__(self) -> None:
8350
+ check.non_empty_str(self.rev)
8351
+ if self.subtrees is not None:
8352
+ for st in self.subtrees:
8353
+ check.non_empty_str(st)
8354
+
8355
+
8356
+ ##
8357
+
8358
+
8359
+ @dc.dataclass(frozen=True)
8360
+ class DeployVenvSpec:
8361
+ interp: ta.Optional[str] = None
8362
+
8363
+ requirements_files: ta.Optional[ta.Sequence[str]] = None
8364
+ extra_dependencies: ta.Optional[ta.Sequence[str]] = None
8365
+
8366
+ use_uv: bool = False
8367
+
8368
+
8369
+ ##
8370
+
8371
+
8372
+ @dc.dataclass(frozen=True)
8373
+ class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
8374
+ app: DeployApp
8375
+
8376
+ git: DeployGitSpec
8377
+
8378
+ venv: ta.Optional[DeployVenvSpec] = None
8379
+
8380
+ conf: ta.Optional[DeployAppConfSpec] = None
8381
+
8382
+ # @ta.override
8383
+ def key(self) -> DeployAppKey:
8384
+ return DeployAppKey(self._key_str())
8385
+
8386
+
8387
+ @dc.dataclass(frozen=True)
8388
+ class DeployAppLinksSpec:
8389
+ apps: ta.Sequence[DeployApp] = ()
8390
+
8391
+ removed_apps: ta.Sequence[DeployApp] = ()
8392
+
8393
+ exclude_unspecified: bool = False
8394
+
8395
+
8396
+ ##
8397
+
8398
+
8399
+ @dc.dataclass(frozen=True)
8400
+ class DeploySystemdSpec:
8401
+ # ~/.config/systemd/user/
8402
+ unit_dir: ta.Optional[str] = None
8403
+
8404
+
8405
+ ##
8406
+
8407
+
8408
+ @dc.dataclass(frozen=True)
8409
+ class DeploySpec(DeploySpecKeyed[DeployKey]):
8410
+ home: DeployHome
8411
+
8412
+ apps: ta.Sequence[DeployAppSpec] = ()
8413
+
8414
+ app_links: DeployAppLinksSpec = DeployAppLinksSpec()
8415
+
8416
+ systemd: ta.Optional[DeploySystemdSpec] = None
8417
+
8418
+ def __post_init__(self) -> None:
8419
+ check.non_empty_str(self.home)
8420
+
8421
+ seen: ta.Set[DeployApp] = set()
8422
+ for a in self.apps:
8423
+ if a.app in seen:
8424
+ raise KeyError(a.app)
8425
+ seen.add(a.app)
8426
+
8427
+ # @ta.override
8428
+ def key(self) -> DeployKey:
8429
+ return DeployKey(self._key_str())
8430
+
8431
+
8291
8432
  ########################################
8292
8433
  # ../remote/execution.py
8293
8434
  """
@@ -9320,7 +9461,7 @@ class DeployConfManager:
9320
9461
 
9321
9462
  elif isinstance(ac, IniDeployAppConfContent):
9322
9463
  ini_sections = pcc(ac.sections)
9323
- return strip_with_newline(render_ini_config(ini_sections))
9464
+ return strip_with_newline(render_ini_sections(ini_sections))
9324
9465
 
9325
9466
  elif isinstance(ac, NginxDeployAppConfContent):
9326
9467
  nginx_items = NginxConfigItems.of(pcc(ac.items))
@@ -9353,9 +9494,15 @@ class DeployConfManager:
9353
9494
  self,
9354
9495
  spec: DeployAppConfSpec,
9355
9496
  app_conf_dir: str,
9497
+ *,
9498
+ string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
9356
9499
  ) -> None:
9357
- def process_str(s: str) -> str:
9358
- return s
9500
+ process_str: ta.Any
9501
+ if string_ns is not None:
9502
+ def process_str(s: str) -> str:
9503
+ return s.format(**string_ns)
9504
+ else:
9505
+ process_str = None
9359
9506
 
9360
9507
  for acf in spec.files or []:
9361
9508
  await self._write_app_conf_file(
@@ -9528,125 +9675,6 @@ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
9528
9675
  return self._owned_deploy_paths
9529
9676
 
9530
9677
 
9531
- ########################################
9532
- # ../deploy/specs.py
9533
-
9534
-
9535
- ##
9536
-
9537
-
9538
- class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
9539
- @cached_nullary
9540
- def _key_str(self) -> str:
9541
- return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
9542
-
9543
- @abc.abstractmethod
9544
- def key(self) -> KeyDeployTagT:
9545
- raise NotImplementedError
9546
-
9547
-
9548
- ##
9549
-
9550
-
9551
- @dc.dataclass(frozen=True)
9552
- class DeployGitRepo:
9553
- host: ta.Optional[str] = None
9554
- username: ta.Optional[str] = None
9555
- path: ta.Optional[str] = None
9556
-
9557
- def __post_init__(self) -> None:
9558
- check.not_in('..', check.non_empty_str(self.host))
9559
- check.not_in('.', check.non_empty_str(self.path))
9560
-
9561
-
9562
- @dc.dataclass(frozen=True)
9563
- class DeployGitSpec:
9564
- repo: DeployGitRepo
9565
- rev: DeployRev
9566
-
9567
- subtrees: ta.Optional[ta.Sequence[str]] = None
9568
-
9569
- def __post_init__(self) -> None:
9570
- check.non_empty_str(self.rev)
9571
- if self.subtrees is not None:
9572
- for st in self.subtrees:
9573
- check.non_empty_str(st)
9574
-
9575
-
9576
- ##
9577
-
9578
-
9579
- @dc.dataclass(frozen=True)
9580
- class DeployVenvSpec:
9581
- interp: ta.Optional[str] = None
9582
-
9583
- requirements_files: ta.Optional[ta.Sequence[str]] = None
9584
- extra_dependencies: ta.Optional[ta.Sequence[str]] = None
9585
-
9586
- use_uv: bool = False
9587
-
9588
-
9589
- ##
9590
-
9591
-
9592
- @dc.dataclass(frozen=True)
9593
- class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
9594
- app: DeployApp
9595
-
9596
- git: DeployGitSpec
9597
-
9598
- venv: ta.Optional[DeployVenvSpec] = None
9599
-
9600
- conf: ta.Optional[DeployAppConfSpec] = None
9601
-
9602
- # @ta.override
9603
- def key(self) -> DeployAppKey:
9604
- return DeployAppKey(self._key_str())
9605
-
9606
-
9607
- @dc.dataclass(frozen=True)
9608
- class DeployAppLinksSpec:
9609
- apps: ta.Sequence[DeployApp] = ()
9610
-
9611
- exclude_unspecified: bool = False
9612
-
9613
-
9614
- ##
9615
-
9616
-
9617
- @dc.dataclass(frozen=True)
9618
- class DeploySystemdSpec:
9619
- # ~/.config/systemd/user/
9620
- unit_dir: ta.Optional[str] = None
9621
-
9622
-
9623
- ##
9624
-
9625
-
9626
- @dc.dataclass(frozen=True)
9627
- class DeploySpec(DeploySpecKeyed[DeployKey]):
9628
- home: DeployHome
9629
-
9630
- apps: ta.Sequence[DeployAppSpec] = ()
9631
-
9632
- app_links: DeployAppLinksSpec = DeployAppLinksSpec()
9633
-
9634
- systemd: ta.Optional[DeploySystemdSpec] = None
9635
-
9636
- def __post_init__(self) -> None:
9637
- check.non_empty_str(self.home)
9638
-
9639
- seen: ta.Set[DeployApp] = set()
9640
- for a in self.apps:
9641
- if a.app in seen:
9642
- raise KeyError(a.app)
9643
- seen.add(a.app)
9644
-
9645
- # @ta.override
9646
- def key(self) -> DeployKey:
9647
- return DeployKey(self._key_str())
9648
-
9649
-
9650
9678
  ########################################
9651
9679
  # ../remote/_main.py
9652
9680
 
@@ -10967,7 +10995,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
10967
10995
 
10968
10996
  async def fetch(self, rev: DeployRev) -> None:
10969
10997
  await self.init()
10970
- await self._call('git', 'fetch', '--depth=1', 'origin', rev)
10998
+
10999
+ # This fetch shouldn't be depth=1 - git doesn't reuse local data with shallow fetches.
11000
+ await self._call('git', 'fetch', 'origin', rev)
10971
11001
 
10972
11002
  #
10973
11003
 
@@ -11125,7 +11155,7 @@ class DeploySystemdManager:
11125
11155
 
11126
11156
  #
11127
11157
 
11128
- if sys.platform == 'linux':
11158
+ if sys.platform == 'linux' and shutil.which('systemctl') is not None:
11129
11159
  async def reload() -> None:
11130
11160
  await asyncio_subprocesses.check_call('systemctl', '--user', 'daemon-reload')
11131
11161
 
@@ -11614,6 +11644,8 @@ class DeployAppManager(DeployPathOwner):
11614
11644
  spec: DeployAppSpec,
11615
11645
  home: DeployHome,
11616
11646
  tags: DeployTagMap,
11647
+ *,
11648
+ conf_string_ns: ta.Optional[ta.Mapping[str, ta.Any]] = None,
11617
11649
  ) -> PreparedApp:
11618
11650
  spec_json = json_dumps_pretty(self._msh.marshal_obj(spec))
11619
11651
 
@@ -11670,9 +11702,17 @@ class DeployAppManager(DeployPathOwner):
11670
11702
  if spec.conf is not None:
11671
11703
  conf_dir = os.path.join(app_dir, 'conf')
11672
11704
  rkw.update(conf_dir=conf_dir)
11705
+
11706
+ conf_ns: ta.Dict[str, ta.Any] = dict(
11707
+ **(conf_string_ns or {}),
11708
+ app=spec.app.s,
11709
+ app_dir=app_dir.rstrip('/'),
11710
+ )
11711
+
11673
11712
  await self._conf.write_app_conf(
11674
11713
  spec.conf,
11675
11714
  conf_dir,
11715
+ string_ns=conf_ns,
11676
11716
  )
11677
11717
 
11678
11718
  #
@@ -11735,7 +11775,7 @@ class DeployAppManager(DeployPathOwner):
11735
11775
  ##
11736
11776
 
11737
11777
 
11738
- DEPLOY_TAG_DATETIME_FMT = '%Y%m%dT%H%M%SZ'
11778
+ DEPLOY_TAG_DATETIME_FMT = '%Y-%m-%d-T-%H-%M-%S-%f-Z'
11739
11779
 
11740
11780
 
11741
11781
  DeployManagerUtcClock = ta.NewType('DeployManagerUtcClock', Func0[datetime.datetime])
@@ -11888,8 +11928,9 @@ class DeployDriver:
11888
11928
 
11889
11929
  das: ta.Set[DeployApp] = {a.app for a in self._spec.apps}
11890
11930
  las: ta.Set[DeployApp] = set(self._spec.app_links.apps)
11891
- if (ras := das & las):
11892
- raise RuntimeError(f'Must not specify apps as both deploy and link: {sorted(a.s for a in ras)}')
11931
+ ras: ta.Set[DeployApp] = set(self._spec.app_links.removed_apps)
11932
+ check.empty(das & (las | ras))
11933
+ check.empty(las & ras)
11893
11934
 
11894
11935
  #
11895
11936
 
@@ -11935,10 +11976,10 @@ class DeployDriver:
11935
11976
  cad = abs_real_path(os.path.join(current_link, 'apps'))
11936
11977
  if os.path.exists(cad):
11937
11978
  for d in os.listdir(cad):
11938
- if (da := DeployApp(d)) not in das:
11979
+ if (da := DeployApp(d)) not in das and da not in ras:
11939
11980
  las.add(da)
11940
11981
 
11941
- for la in self._spec.app_links.apps:
11982
+ for la in las:
11942
11983
  await self._drive_app_link(
11943
11984
  la,
11944
11985
  current_link,
@@ -11968,10 +12009,16 @@ class DeployDriver:
11968
12009
  #
11969
12010
 
11970
12011
  async def _drive_app_deploy(self, app: DeployAppSpec) -> None:
12012
+ current_deploy_link = os.path.join(self._home, self._deploys.CURRENT_DEPLOY_LINK.render())
12013
+
11971
12014
  pa = await self._apps.prepare_app(
11972
12015
  app,
11973
12016
  self._home,
11974
12017
  self.tags,
12018
+ conf_string_ns=dict(
12019
+ deploy_home=self._home,
12020
+ current_deploy_link=current_deploy_link,
12021
+ ),
11975
12022
  )
11976
12023
 
11977
12024
  #