ominfra 0.0.0.dev192__py3-none-any.whl → 0.0.0.dev194__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
ominfra/scripts/manage.py CHANGED
@@ -16,6 +16,7 @@ import asyncio.subprocess
16
16
  import base64
17
17
  import collections
18
18
  import collections.abc
19
+ import configparser
19
20
  import contextlib
20
21
  import contextvars
21
22
  import ctypes as ct
@@ -73,17 +74,23 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
73
74
  VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
74
75
  VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
75
76
 
76
- # ../../omdev/toml/parser.py
77
- TomlParseFloat = ta.Callable[[str], ta.Any]
78
- TomlKey = ta.Tuple[str, ...]
79
- TomlPos = int # ta.TypeAlias
80
-
81
77
  # deploy/paths/types.py
82
78
  DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
83
79
 
84
80
  # ../../omlish/asyncs/asyncio/timeouts.py
85
81
  AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
86
82
 
83
+ # ../../omlish/configs/types.py
84
+ ConfigMap = ta.Mapping[str, ta.Any]
85
+
86
+ # ../../omlish/formats/ini/sections.py
87
+ IniSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
88
+
89
+ # ../../omlish/formats/toml/parser.py
90
+ TomlParseFloat = ta.Callable[[str], ta.Any]
91
+ TomlKey = ta.Tuple[str, ...]
92
+ TomlPos = int # ta.TypeAlias
93
+
87
94
  # ../../omlish/lite/cached.py
88
95
  T = ta.TypeVar('T')
89
96
  CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
@@ -113,6 +120,9 @@ CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
113
120
  # ../../omlish/argparse/cli.py
114
121
  ArgparseCmdFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
115
122
 
123
+ # ../../omlish/configs/formats.py
124
+ ConfigDataT = ta.TypeVar('ConfigDataT', bound='ConfigData')
125
+
116
126
  # ../../omlish/lite/contextmanagers.py
117
127
  ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
118
128
 
@@ -127,10 +137,6 @@ InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
127
137
  AtomicPathSwapKind = ta.Literal['dir', 'file']
128
138
  AtomicPathSwapState = ta.Literal['open', 'committed', 'aborted'] # ta.TypeAlias
129
139
 
130
- # ../configs.py
131
- ConfigMapping = ta.Mapping[str, ta.Any]
132
- IniConfigSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
133
-
134
140
  # ../../omlish/subprocesses.py
135
141
  SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
136
142
 
@@ -549,1482 +555,1653 @@ def canonicalize_version(
549
555
 
550
556
 
551
557
  ########################################
552
- # ../../../omdev/toml/parser.py
553
- # SPDX-License-Identifier: MIT
554
- # SPDX-FileCopyrightText: 2021 Taneli Hukkinen
555
- # Licensed to PSF under a Contributor Agreement.
556
- #
557
- # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
558
- # --------------------------------------------
559
- #
560
- # 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
561
- # ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
562
- # documentation.
563
- #
564
- # 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
565
- # royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
566
- # works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
567
- # Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
568
- # 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
569
- # Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
570
- #
571
- # 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
572
- # wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
573
- # any such work a brief summary of the changes made to Python.
574
- #
575
- # 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
576
- # EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
577
- # OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
578
- # RIGHTS.
579
- #
580
- # 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
581
- # DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
582
- # ADVISED OF THE POSSIBILITY THEREOF.
583
- #
584
- # 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
585
- #
586
- # 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
587
- # venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
588
- # name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
589
- #
590
- # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
591
- # License Agreement.
592
- #
593
- # https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
558
+ # ../config.py
594
559
 
595
560
 
596
- ##
561
+ @dc.dataclass(frozen=True)
562
+ class MainConfig:
563
+ log_level: ta.Optional[str] = 'INFO'
597
564
 
565
+ debug: bool = False
598
566
 
599
- _TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
600
567
 
601
- TOML_RE_NUMBER = re.compile(
602
- r"""
603
- 0
604
- (?:
605
- x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
606
- |
607
- b[01](?:_?[01])* # bin
608
- |
609
- o[0-7](?:_?[0-7])* # oct
610
- )
611
- |
612
- [+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
613
- (?P<floatpart>
614
- (?:\.[0-9](?:_?[0-9])*)? # optional fractional part
615
- (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
616
- )
617
- """,
618
- flags=re.VERBOSE,
619
- )
620
- TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
621
- TOML_RE_DATETIME = re.compile(
622
- rf"""
623
- ([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
624
- (?:
625
- [Tt ]
626
- {_TOML_TIME_RE_STR}
627
- (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
628
- )?
629
- """,
630
- flags=re.VERBOSE,
631
- )
568
+ ########################################
569
+ # ../deploy/config.py
632
570
 
633
571
 
634
- def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
635
- """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
572
+ ##
636
573
 
637
- Raises ValueError if the match does not correspond to a valid date or datetime.
638
- """
639
- (
640
- year_str,
641
- month_str,
642
- day_str,
643
- hour_str,
644
- minute_str,
645
- sec_str,
646
- micros_str,
647
- zulu_time,
648
- offset_sign_str,
649
- offset_hour_str,
650
- offset_minute_str,
651
- ) = match.groups()
652
- year, month, day = int(year_str), int(month_str), int(day_str)
653
- if hour_str is None:
654
- return datetime.date(year, month, day)
655
- hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
656
- micros = int(micros_str.ljust(6, '0')) if micros_str else 0
657
- if offset_sign_str:
658
- tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
659
- offset_hour_str, offset_minute_str, offset_sign_str,
660
- )
661
- elif zulu_time:
662
- tz = datetime.UTC
663
- else: # local date-time
664
- tz = None
665
- return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
666
574
 
575
+ @dc.dataclass(frozen=True)
576
+ class DeployConfig:
577
+ pass
667
578
 
668
- @functools.lru_cache() # noqa
669
- def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
670
- sign = 1 if sign_str == '+' else -1
671
- return datetime.timezone(
672
- datetime.timedelta(
673
- hours=sign * int(hour_str),
674
- minutes=sign * int(minute_str),
675
- ),
676
- )
677
579
 
580
+ ########################################
581
+ # ../deploy/paths/types.py
678
582
 
679
- def toml_match_to_localtime(match: re.Match) -> datetime.time:
680
- hour_str, minute_str, sec_str, micros_str = match.groups()
681
- micros = int(micros_str.ljust(6, '0')) if micros_str else 0
682
- return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
683
583
 
584
+ ##
684
585
 
685
- def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
686
- if match.group('floatpart'):
687
- return parse_float(match.group())
688
- return int(match.group(), 0)
689
586
 
587
+ ########################################
588
+ # ../deploy/types.py
690
589
 
691
- TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
692
590
 
693
- # Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
694
- # functions.
695
- TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
696
- TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
591
+ ##
697
592
 
698
- TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
699
- TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
700
593
 
701
- TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
594
+ DeployHome = ta.NewType('DeployHome', str)
702
595
 
703
- TOML_WS = frozenset(' \t')
704
- TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
705
- TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
706
- TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
707
- TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
596
+ DeployRev = ta.NewType('DeployRev', str)
708
597
 
709
- TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
710
- {
711
- '\\b': '\u0008', # backspace
712
- '\\t': '\u0009', # tab
713
- '\\n': '\u000A', # linefeed
714
- '\\f': '\u000C', # form feed
715
- '\\r': '\u000D', # carriage return
716
- '\\"': '\u0022', # quote
717
- '\\\\': '\u005C', # backslash
718
- },
719
- )
720
598
 
599
+ ########################################
600
+ # ../../pyremote.py
601
+ """
602
+ Basically this: https://mitogen.networkgenomics.com/howitworks.html
721
603
 
722
- class TomlDecodeError(ValueError):
723
- """An error raised if a document is not valid TOML."""
604
+ TODO:
605
+ - log: ta.Optional[logging.Logger] = None + log.debug's
606
+ """
724
607
 
725
608
 
726
- def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
727
- """Parse TOML from a binary file object."""
728
- b = fp.read()
729
- try:
730
- s = b.decode()
731
- except AttributeError:
732
- raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
733
- return toml_loads(s, parse_float=parse_float)
609
+ ##
734
610
 
735
611
 
736
- def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
737
- """Parse TOML from a string."""
612
+ @dc.dataclass(frozen=True)
613
+ class PyremoteBootstrapOptions:
614
+ debug: bool = False
738
615
 
739
- # The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
740
- try:
741
- src = s.replace('\r\n', '\n')
742
- except (AttributeError, TypeError):
743
- raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
744
- pos = 0
745
- out = TomlOutput(TomlNestedDict(), TomlFlags())
746
- header: TomlKey = ()
747
- parse_float = toml_make_safe_parse_float(parse_float)
616
+ DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
617
+ main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
748
618
 
749
- # Parse one statement at a time (typically means one line in TOML source)
750
- while True:
751
- # 1. Skip line leading whitespace
752
- pos = toml_skip_chars(src, pos, TOML_WS)
753
619
 
754
- # 2. Parse rules. Expect one of the following:
755
- # - end of file
756
- # - end of line
757
- # - comment
758
- # - key/value pair
759
- # - append dict to list (and move to its namespace)
760
- # - create dict (and move to its namespace)
761
- # Skip trailing whitespace when applicable.
762
- try:
763
- char = src[pos]
764
- except IndexError:
765
- break
766
- if char == '\n':
767
- pos += 1
768
- continue
769
- if char in TOML_KEY_INITIAL_CHARS:
770
- pos = toml_key_value_rule(src, pos, out, header, parse_float)
771
- pos = toml_skip_chars(src, pos, TOML_WS)
772
- elif char == '[':
773
- try:
774
- second_char: ta.Optional[str] = src[pos + 1]
775
- except IndexError:
776
- second_char = None
777
- out.flags.finalize_pending()
778
- if second_char == '[':
779
- pos, header = toml_create_list_rule(src, pos, out)
780
- else:
781
- pos, header = toml_create_dict_rule(src, pos, out)
782
- pos = toml_skip_chars(src, pos, TOML_WS)
783
- elif char != '#':
784
- raise toml_suffixed_err(src, pos, 'Invalid statement')
620
+ ##
785
621
 
786
- # 3. Skip comment
787
- pos = toml_skip_comment(src, pos)
788
622
 
789
- # 4. Expect end of line or end of file
790
- try:
791
- char = src[pos]
792
- except IndexError:
793
- break
794
- if char != '\n':
795
- raise toml_suffixed_err(
796
- src, pos, 'Expected newline or end of document after a statement',
797
- )
798
- pos += 1
623
+ @dc.dataclass(frozen=True)
624
+ class PyremoteEnvInfo:
625
+ sys_base_prefix: str
626
+ sys_byteorder: str
627
+ sys_defaultencoding: str
628
+ sys_exec_prefix: str
629
+ sys_executable: str
630
+ sys_implementation_name: str
631
+ sys_path: ta.List[str]
632
+ sys_platform: str
633
+ sys_prefix: str
634
+ sys_version: str
635
+ sys_version_info: ta.List[ta.Union[int, str]]
799
636
 
800
- return out.data.dict
637
+ platform_architecture: ta.List[str]
638
+ platform_machine: str
639
+ platform_platform: str
640
+ platform_processor: str
641
+ platform_system: str
642
+ platform_release: str
643
+ platform_version: str
801
644
 
645
+ site_userbase: str
802
646
 
803
- class TomlFlags:
804
- """Flags that map to parsed keys/namespaces."""
647
+ os_cwd: str
648
+ os_gid: int
649
+ os_loadavg: ta.List[float]
650
+ os_login: ta.Optional[str]
651
+ os_pgrp: int
652
+ os_pid: int
653
+ os_ppid: int
654
+ os_uid: int
805
655
 
806
- # Marks an immutable namespace (inline array or inline table).
807
- FROZEN = 0
808
- # Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
809
- EXPLICIT_NEST = 1
656
+ pw_name: str
657
+ pw_uid: int
658
+ pw_gid: int
659
+ pw_gecos: str
660
+ pw_dir: str
661
+ pw_shell: str
810
662
 
811
- def __init__(self) -> None:
812
- self._flags: ta.Dict[str, dict] = {}
813
- self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
663
+ env_path: ta.Optional[str]
814
664
 
815
- def add_pending(self, key: TomlKey, flag: int) -> None:
816
- self._pending_flags.add((key, flag))
817
665
 
818
- def finalize_pending(self) -> None:
819
- for key, flag in self._pending_flags:
820
- self.set(key, flag, recursive=False)
821
- self._pending_flags.clear()
666
+ def _get_pyremote_env_info() -> PyremoteEnvInfo:
667
+ os_uid = os.getuid()
822
668
 
823
- def unset_all(self, key: TomlKey) -> None:
824
- cont = self._flags
825
- for k in key[:-1]:
826
- if k not in cont:
827
- return
828
- cont = cont[k]['nested']
829
- cont.pop(key[-1], None)
669
+ pw = pwd.getpwuid(os_uid)
830
670
 
831
- def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
832
- cont = self._flags
833
- key_parent, key_stem = key[:-1], key[-1]
834
- for k in key_parent:
835
- if k not in cont:
836
- cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
837
- cont = cont[k]['nested']
838
- if key_stem not in cont:
839
- cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
840
- cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
671
+ os_login: ta.Optional[str]
672
+ try:
673
+ os_login = os.getlogin()
674
+ except OSError:
675
+ os_login = None
841
676
 
842
- def is_(self, key: TomlKey, flag: int) -> bool:
843
- if not key:
844
- return False # document root has no flags
845
- cont = self._flags
846
- for k in key[:-1]:
847
- if k not in cont:
848
- return False
849
- inner_cont = cont[k]
850
- if flag in inner_cont['recursive_flags']:
851
- return True
852
- cont = inner_cont['nested']
853
- key_stem = key[-1]
854
- if key_stem in cont:
855
- cont = cont[key_stem]
856
- return flag in cont['flags'] or flag in cont['recursive_flags']
857
- return False
677
+ return PyremoteEnvInfo(
678
+ sys_base_prefix=sys.base_prefix,
679
+ sys_byteorder=sys.byteorder,
680
+ sys_defaultencoding=sys.getdefaultencoding(),
681
+ sys_exec_prefix=sys.exec_prefix,
682
+ sys_executable=sys.executable,
683
+ sys_implementation_name=sys.implementation.name,
684
+ sys_path=sys.path,
685
+ sys_platform=sys.platform,
686
+ sys_prefix=sys.prefix,
687
+ sys_version=sys.version,
688
+ sys_version_info=list(sys.version_info),
858
689
 
690
+ platform_architecture=list(platform.architecture()),
691
+ platform_machine=platform.machine(),
692
+ platform_platform=platform.platform(),
693
+ platform_processor=platform.processor(),
694
+ platform_system=platform.system(),
695
+ platform_release=platform.release(),
696
+ platform_version=platform.version(),
859
697
 
860
- class TomlNestedDict:
861
- def __init__(self) -> None:
862
- # The parsed content of the TOML document
863
- self.dict: ta.Dict[str, ta.Any] = {}
698
+ site_userbase=site.getuserbase(),
864
699
 
865
- def get_or_create_nest(
866
- self,
867
- key: TomlKey,
868
- *,
869
- access_lists: bool = True,
870
- ) -> dict:
871
- cont: ta.Any = self.dict
872
- for k in key:
873
- if k not in cont:
874
- cont[k] = {}
875
- cont = cont[k]
876
- if access_lists and isinstance(cont, list):
877
- cont = cont[-1]
878
- if not isinstance(cont, dict):
879
- raise KeyError('There is no nest behind this key')
880
- return cont
700
+ os_cwd=os.getcwd(),
701
+ os_gid=os.getgid(),
702
+ os_loadavg=list(os.getloadavg()),
703
+ os_login=os_login,
704
+ os_pgrp=os.getpgrp(),
705
+ os_pid=os.getpid(),
706
+ os_ppid=os.getppid(),
707
+ os_uid=os_uid,
881
708
 
882
- def append_nest_to_list(self, key: TomlKey) -> None:
883
- cont = self.get_or_create_nest(key[:-1])
884
- last_key = key[-1]
885
- if last_key in cont:
886
- list_ = cont[last_key]
887
- if not isinstance(list_, list):
888
- raise KeyError('An object other than list found behind this key')
889
- list_.append({})
890
- else:
891
- cont[last_key] = [{}]
709
+ pw_name=pw.pw_name,
710
+ pw_uid=pw.pw_uid,
711
+ pw_gid=pw.pw_gid,
712
+ pw_gecos=pw.pw_gecos,
713
+ pw_dir=pw.pw_dir,
714
+ pw_shell=pw.pw_shell,
892
715
 
716
+ env_path=os.environ.get('PATH'),
717
+ )
893
718
 
894
- class TomlOutput(ta.NamedTuple):
895
- data: TomlNestedDict
896
- flags: TomlFlags
897
719
 
720
+ ##
898
721
 
899
- def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
900
- try:
901
- while src[pos] in chars:
902
- pos += 1
903
- except IndexError:
904
- pass
905
- return pos
906
722
 
723
+ _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
724
+ _PYREMOTE_BOOTSTRAP_SRC_FD = 101
907
725
 
908
- def toml_skip_until(
909
- src: str,
910
- pos: TomlPos,
911
- expect: str,
912
- *,
913
- error_on: ta.FrozenSet[str],
914
- error_on_eof: bool,
915
- ) -> TomlPos:
916
- try:
917
- new_pos = src.index(expect, pos)
918
- except ValueError:
919
- new_pos = len(src)
920
- if error_on_eof:
921
- raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
726
+ _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
727
+ _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
728
+ _PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
729
+ _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
730
+ _PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
922
731
 
923
- if not error_on.isdisjoint(src[pos:new_pos]):
924
- while src[pos] not in error_on:
925
- pos += 1
926
- raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
927
- return new_pos
732
+ _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
733
+ _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
734
+ _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
735
+ _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
928
736
 
737
+ _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
929
738
 
930
- def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
931
- try:
932
- char: ta.Optional[str] = src[pos]
933
- except IndexError:
934
- char = None
935
- if char == '#':
936
- return toml_skip_until(
937
- src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
938
- )
939
- return pos
739
+ _PYREMOTE_BOOTSTRAP_IMPORTS = [
740
+ 'base64',
741
+ 'os',
742
+ 'struct',
743
+ 'sys',
744
+ 'zlib',
745
+ ]
940
746
 
941
747
 
942
- def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
943
- while True:
944
- pos_before_skip = pos
945
- pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
946
- pos = toml_skip_comment(src, pos)
947
- if pos == pos_before_skip:
948
- return pos
748
+ def _pyremote_bootstrap_main(context_name: str) -> None:
749
+ # Get pid
750
+ pid = os.getpid()
949
751
 
752
+ # Two copies of payload src to be sent to parent
753
+ r0, w0 = os.pipe()
754
+ r1, w1 = os.pipe()
950
755
 
951
- def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
952
- pos += 1 # Skip "["
953
- pos = toml_skip_chars(src, pos, TOML_WS)
954
- pos, key = toml_parse_key(src, pos)
756
+ if (cp := os.fork()):
757
+ # Parent process
955
758
 
956
- if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
957
- raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
958
- out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
959
- try:
960
- out.data.get_or_create_nest(key)
961
- except KeyError:
962
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
759
+ # Dup original stdin to comm_fd for use as comm channel
760
+ os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
963
761
 
964
- if not src.startswith(']', pos):
965
- raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
966
- return pos + 1, key
762
+ # Overwrite stdin (fed to python repl) with first copy of src
763
+ os.dup2(r0, 0)
967
764
 
765
+ # Dup second copy of src to src_fd to recover after launch
766
+ os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
968
767
 
969
- def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
970
- pos += 2 # Skip "[["
971
- pos = toml_skip_chars(src, pos, TOML_WS)
972
- pos, key = toml_parse_key(src, pos)
768
+ # Close remaining fd's
769
+ for f in [r0, w0, r1, w1]:
770
+ os.close(f)
973
771
 
974
- if out.flags.is_(key, TomlFlags.FROZEN):
975
- raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
976
- # Free the namespace now that it points to another empty list item...
977
- out.flags.unset_all(key)
978
- # ...but this key precisely is still prohibited from table declaration
979
- out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
980
- try:
981
- out.data.append_nest_to_list(key)
982
- except KeyError:
983
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
772
+ # Save vars
773
+ env = os.environ
774
+ exe = sys.executable
775
+ env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
776
+ env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
777
+ env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
984
778
 
985
- if not src.startswith(']]', pos):
986
- raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
987
- return pos + 2, key
779
+ # Start repl reading stdin from r0
780
+ os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
988
781
 
782
+ else:
783
+ # Child process
989
784
 
990
- def toml_key_value_rule(
991
- src: str,
992
- pos: TomlPos,
993
- out: TomlOutput,
994
- header: TomlKey,
995
- parse_float: TomlParseFloat,
996
- ) -> TomlPos:
997
- pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
998
- key_parent, key_stem = key[:-1], key[-1]
999
- abs_key_parent = header + key_parent
785
+ # Write first ack
786
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
1000
787
 
1001
- relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
1002
- for cont_key in relative_path_cont_keys:
1003
- # Check that dotted key syntax does not redefine an existing table
1004
- if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
1005
- raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
1006
- # Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
1007
- # table sections.
1008
- out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
788
+ # Write pid
789
+ os.write(1, struct.pack('<Q', pid))
1009
790
 
1010
- if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
1011
- raise toml_suffixed_err(
1012
- src,
1013
- pos,
1014
- f'Cannot mutate immutable namespace {abs_key_parent}',
1015
- )
791
+ # Read payload src from stdin
792
+ payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
793
+ if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
794
+ raise EOFError
795
+ payload_src = zlib.decompress(payload_z)
1016
796
 
1017
- try:
1018
- nest = out.data.get_or_create_nest(abs_key_parent)
1019
- except KeyError:
1020
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1021
- if key_stem in nest:
1022
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
1023
- # Mark inline table and array namespaces recursively immutable
1024
- if isinstance(value, (dict, list)):
1025
- out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
1026
- nest[key_stem] = value
1027
- return pos
797
+ # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
798
+ # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
799
+ for w in [w0, w1]:
800
+ fp = os.fdopen(w, 'wb', 0)
801
+ fp.write(payload_src)
802
+ fp.close()
1028
803
 
804
+ # Write second ack
805
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
1029
806
 
1030
- def toml_parse_key_value_pair(
1031
- src: str,
1032
- pos: TomlPos,
1033
- parse_float: TomlParseFloat,
1034
- ) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
1035
- pos, key = toml_parse_key(src, pos)
1036
- try:
1037
- char: ta.Optional[str] = src[pos]
1038
- except IndexError:
1039
- char = None
1040
- if char != '=':
1041
- raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
1042
- pos += 1
1043
- pos = toml_skip_chars(src, pos, TOML_WS)
1044
- pos, value = toml_parse_value(src, pos, parse_float)
1045
- return pos, key, value
807
+ # Exit child
808
+ sys.exit(0)
1046
809
 
1047
810
 
1048
- def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
1049
- pos, key_part = toml_parse_key_part(src, pos)
1050
- key: TomlKey = (key_part,)
1051
- pos = toml_skip_chars(src, pos, TOML_WS)
1052
- while True:
1053
- try:
1054
- char: ta.Optional[str] = src[pos]
1055
- except IndexError:
1056
- char = None
1057
- if char != '.':
1058
- return pos, key
1059
- pos += 1
1060
- pos = toml_skip_chars(src, pos, TOML_WS)
1061
- pos, key_part = toml_parse_key_part(src, pos)
1062
- key += (key_part,)
1063
- pos = toml_skip_chars(src, pos, TOML_WS)
811
+ ##
1064
812
 
1065
813
 
1066
- def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1067
- try:
1068
- char: ta.Optional[str] = src[pos]
1069
- except IndexError:
1070
- char = None
1071
- if char in TOML_BARE_KEY_CHARS:
1072
- start_pos = pos
1073
- pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
1074
- return pos, src[start_pos:pos]
1075
- if char == "'":
1076
- return toml_parse_literal_str(src, pos)
1077
- if char == '"':
1078
- return toml_parse_one_line_basic_str(src, pos)
1079
- raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
814
+ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
815
+ if any(c in context_name for c in '\'"'):
816
+ raise NameError(context_name)
1080
817
 
818
+ import inspect
819
+ import textwrap
820
+ bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
1081
821
 
1082
- def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1083
- pos += 1
1084
- return toml_parse_basic_str(src, pos, multiline=False)
822
+ for gl in [
823
+ '_PYREMOTE_BOOTSTRAP_INPUT_FD',
824
+ '_PYREMOTE_BOOTSTRAP_SRC_FD',
1085
825
 
826
+ '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
827
+ '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
828
+ '_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
1086
829
 
1087
- def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
1088
- pos += 1
1089
- array: list = []
830
+ '_PYREMOTE_BOOTSTRAP_ACK0',
831
+ '_PYREMOTE_BOOTSTRAP_ACK1',
1090
832
 
1091
- pos = toml_skip_comments_and_array_ws(src, pos)
1092
- if src.startswith(']', pos):
1093
- return pos + 1, array
1094
- while True:
1095
- pos, val = toml_parse_value(src, pos, parse_float)
1096
- array.append(val)
1097
- pos = toml_skip_comments_and_array_ws(src, pos)
833
+ '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
834
+ ]:
835
+ bs_src = bs_src.replace(gl, repr(globals()[gl]))
1098
836
 
1099
- c = src[pos:pos + 1]
1100
- if c == ']':
1101
- return pos + 1, array
1102
- if c != ',':
1103
- raise toml_suffixed_err(src, pos, 'Unclosed array')
1104
- pos += 1
837
+ bs_src = '\n'.join(
838
+ cl
839
+ for l in bs_src.splitlines()
840
+ if (cl := (l.split('#')[0]).rstrip())
841
+ if cl.strip()
842
+ )
1105
843
 
1106
- pos = toml_skip_comments_and_array_ws(src, pos)
1107
- if src.startswith(']', pos):
1108
- return pos + 1, array
844
+ bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
845
+ bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
846
+ if b'"' in bs_z85:
847
+ raise ValueError(bs_z85)
1109
848
 
849
+ stmts = [
850
+ f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
851
+ f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
852
+ f'_pyremote_bootstrap_main("{context_name}")',
853
+ ]
1110
854
 
1111
- def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
1112
- pos += 1
1113
- nested_dict = TomlNestedDict()
1114
- flags = TomlFlags()
855
+ cmd = '; '.join(stmts)
856
+ return cmd
1115
857
 
1116
- pos = toml_skip_chars(src, pos, TOML_WS)
1117
- if src.startswith('}', pos):
1118
- return pos + 1, nested_dict.dict
1119
- while True:
1120
- pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1121
- key_parent, key_stem = key[:-1], key[-1]
1122
- if flags.is_(key, TomlFlags.FROZEN):
1123
- raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1124
- try:
1125
- nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
1126
- except KeyError:
1127
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1128
- if key_stem in nest:
1129
- raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
1130
- nest[key_stem] = value
1131
- pos = toml_skip_chars(src, pos, TOML_WS)
1132
- c = src[pos:pos + 1]
1133
- if c == '}':
1134
- return pos + 1, nested_dict.dict
1135
- if c != ',':
1136
- raise toml_suffixed_err(src, pos, 'Unclosed inline table')
1137
- if isinstance(value, (dict, list)):
1138
- flags.set(key, TomlFlags.FROZEN, recursive=True)
1139
- pos += 1
1140
- pos = toml_skip_chars(src, pos, TOML_WS)
1141
858
 
859
+ ##
1142
860
 
1143
- def toml_parse_basic_str_escape(
1144
- src: str,
1145
- pos: TomlPos,
1146
- *,
1147
- multiline: bool = False,
1148
- ) -> ta.Tuple[TomlPos, str]:
1149
- escape_id = src[pos:pos + 2]
1150
- pos += 2
1151
- if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
1152
- # Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
1153
- # newline.
1154
- if escape_id != '\\\n':
1155
- pos = toml_skip_chars(src, pos, TOML_WS)
1156
- try:
1157
- char = src[pos]
1158
- except IndexError:
1159
- return pos, ''
1160
- if char != '\n':
1161
- raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
1162
- pos += 1
1163
- pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1164
- return pos, ''
1165
- if escape_id == '\\u':
1166
- return toml_parse_hex_char(src, pos, 4)
1167
- if escape_id == '\\U':
1168
- return toml_parse_hex_char(src, pos, 8)
1169
- try:
1170
- return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
1171
- except KeyError:
1172
- raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
1173
861
 
862
+ @dc.dataclass(frozen=True)
863
+ class PyremotePayloadRuntime:
864
+ input: ta.BinaryIO
865
+ output: ta.BinaryIO
866
+ context_name: str
867
+ payload_src: str
868
+ options: PyremoteBootstrapOptions
869
+ env_info: PyremoteEnvInfo
1174
870
 
1175
- def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1176
- return toml_parse_basic_str_escape(src, pos, multiline=True)
1177
871
 
872
+ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
873
+ # If src file var is not present we need to do initial finalization
874
+ if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
875
+ # Read second copy of payload src
876
+ r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
877
+ payload_src = r1.read().decode('utf-8')
878
+ r1.close()
1178
879
 
1179
- def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
1180
- hex_str = src[pos:pos + hex_len]
1181
- if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
1182
- raise toml_suffixed_err(src, pos, 'Invalid hex value')
1183
- pos += hex_len
1184
- hex_int = int(hex_str, 16)
1185
- if not toml_is_unicode_scalar_value(hex_int):
1186
- raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
1187
- return pos, chr(hex_int)
880
+ # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
881
+ # a pipe at once.
882
+ os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
1188
883
 
884
+ # Read options
885
+ options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
886
+ if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
887
+ raise EOFError
888
+ options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
1189
889
 
1190
- def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1191
- pos += 1 # Skip starting apostrophe
1192
- start_pos = pos
1193
- pos = toml_skip_until(
1194
- src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
1195
- )
1196
- return pos + 1, src[start_pos:pos] # Skip ending apostrophe
890
+ # If debugging, re-exec as file
891
+ if options.debug:
892
+ # Write temp source file
893
+ import tempfile
894
+ tfd, tfn = tempfile.mkstemp('-pyremote.py')
895
+ os.write(tfd, payload_src.encode('utf-8'))
896
+ os.close(tfd)
1197
897
 
898
+ # Set vars
899
+ os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
900
+ os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
1198
901
 
1199
- def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
1200
- pos += 3
1201
- if src.startswith('\n', pos):
1202
- pos += 1
902
+ # Re-exec temp file
903
+ exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
904
+ context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
905
+ os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
1203
906
 
1204
- if literal:
1205
- delim = "'"
1206
- end_pos = toml_skip_until(
1207
- src,
1208
- pos,
1209
- "'''",
1210
- error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
1211
- error_on_eof=True,
1212
- )
1213
- result = src[pos:end_pos]
1214
- pos = end_pos + 3
1215
907
  else:
1216
- delim = '"'
1217
- pos, result = toml_parse_basic_str(src, pos, multiline=True)
908
+ # Load options json var
909
+ options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
910
+ options = PyremoteBootstrapOptions(**json.loads(options_json_str))
1218
911
 
1219
- # Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
1220
- if not src.startswith(delim, pos):
1221
- return pos, result
1222
- pos += 1
1223
- if not src.startswith(delim, pos):
1224
- return pos, result + delim
1225
- pos += 1
1226
- return pos, result + (delim * 2)
912
+ # Read temp source file
913
+ with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
914
+ payload_src = sf.read()
1227
915
 
916
+ # Restore vars
917
+ sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
918
+ context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
1228
919
 
1229
- def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
1230
- if multiline:
1231
- error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1232
- parse_escapes = toml_parse_basic_str_escape_multiline
1233
- else:
1234
- error_on = TOML_ILLEGAL_BASIC_STR_CHARS
1235
- parse_escapes = toml_parse_basic_str_escape
1236
- result = ''
1237
- start_pos = pos
1238
- while True:
1239
- try:
1240
- char = src[pos]
1241
- except IndexError:
1242
- raise toml_suffixed_err(src, pos, 'Unterminated string') from None
1243
- if char == '"':
1244
- if not multiline:
1245
- return pos + 1, result + src[start_pos:pos]
1246
- if src.startswith('"""', pos):
1247
- return pos + 3, result + src[start_pos:pos]
1248
- pos += 1
1249
- continue
1250
- if char == '\\':
1251
- result += src[start_pos:pos]
1252
- pos, parsed_escape = parse_escapes(src, pos)
1253
- result += parsed_escape
1254
- start_pos = pos
1255
- continue
1256
- if char in error_on:
1257
- raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
1258
- pos += 1
920
+ # Write third ack
921
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
1259
922
 
923
+ # Write env info
924
+ env_info = _get_pyremote_env_info()
925
+ env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
926
+ os.write(1, struct.pack('<I', len(env_info_json)))
927
+ os.write(1, env_info_json.encode('utf-8'))
1260
928
 
1261
- def toml_parse_value( # noqa: C901
1262
- src: str,
1263
- pos: TomlPos,
1264
- parse_float: TomlParseFloat,
1265
- ) -> ta.Tuple[TomlPos, ta.Any]:
1266
- try:
1267
- char: ta.Optional[str] = src[pos]
1268
- except IndexError:
1269
- char = None
929
+ # Setup IO
930
+ input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
931
+ output = os.fdopen(os.dup(1), 'wb', 0) # noqa
932
+ os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
933
+ os.close(nfd)
1270
934
 
1271
- # IMPORTANT: order conditions based on speed of checking and likelihood
935
+ if (mn := options.main_name_override) is not None:
936
+ # Inspections like typing.get_type_hints need an entry in sys.modules.
937
+ sys.modules[mn] = sys.modules['__main__']
1272
938
 
1273
- # Basic strings
1274
- if char == '"':
1275
- if src.startswith('"""', pos):
1276
- return toml_parse_multiline_str(src, pos, literal=False)
1277
- return toml_parse_one_line_basic_str(src, pos)
939
+ # Write fourth ack
940
+ output.write(_PYREMOTE_BOOTSTRAP_ACK3)
1278
941
 
1279
- # Literal strings
1280
- if char == "'":
1281
- if src.startswith("'''", pos):
1282
- return toml_parse_multiline_str(src, pos, literal=True)
1283
- return toml_parse_literal_str(src, pos)
942
+ # Return
943
+ return PyremotePayloadRuntime(
944
+ input=input,
945
+ output=output,
946
+ context_name=context_name,
947
+ payload_src=payload_src,
948
+ options=options,
949
+ env_info=env_info,
950
+ )
1284
951
 
1285
- # Booleans
1286
- if char == 't':
1287
- if src.startswith('true', pos):
1288
- return pos + 4, True
1289
- if char == 'f':
1290
- if src.startswith('false', pos):
1291
- return pos + 5, False
1292
952
 
1293
- # Arrays
1294
- if char == '[':
1295
- return toml_parse_array(src, pos, parse_float)
953
+ ##
1296
954
 
1297
- # Inline tables
1298
- if char == '{':
1299
- return toml_parse_inline_table(src, pos, parse_float)
1300
955
 
1301
- # Dates and times
1302
- datetime_match = TOML_RE_DATETIME.match(src, pos)
1303
- if datetime_match:
1304
- try:
1305
- datetime_obj = toml_match_to_datetime(datetime_match)
1306
- except ValueError as e:
1307
- raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
1308
- return datetime_match.end(), datetime_obj
1309
- localtime_match = TOML_RE_LOCALTIME.match(src, pos)
1310
- if localtime_match:
1311
- return localtime_match.end(), toml_match_to_localtime(localtime_match)
956
+ class PyremoteBootstrapDriver:
957
+ def __init__(
958
+ self,
959
+ payload_src: ta.Union[str, ta.Sequence[str]],
960
+ options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
961
+ ) -> None:
962
+ super().__init__()
1312
963
 
1313
- # Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
1314
- # located after handling of dates and times.
1315
- number_match = TOML_RE_NUMBER.match(src, pos)
1316
- if number_match:
1317
- return number_match.end(), toml_match_to_number(number_match, parse_float)
964
+ self._payload_src = payload_src
965
+ self._options = options
1318
966
 
1319
- # Special floats
1320
- first_three = src[pos:pos + 3]
1321
- if first_three in {'inf', 'nan'}:
1322
- return pos + 3, parse_float(first_three)
1323
- first_four = src[pos:pos + 4]
1324
- if first_four in {'-inf', '+inf', '-nan', '+nan'}:
1325
- return pos + 4, parse_float(first_four)
967
+ self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
968
+ self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
1326
969
 
1327
- raise toml_suffixed_err(src, pos, 'Invalid value')
970
+ self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
971
+ #
1328
972
 
973
+ @classmethod
974
+ def _prepare_payload_src(
975
+ cls,
976
+ payload_src: ta.Union[str, ta.Sequence[str]],
977
+ options: PyremoteBootstrapOptions,
978
+ ) -> str:
979
+ parts: ta.List[str]
980
+ if isinstance(payload_src, str):
981
+ parts = [payload_src]
982
+ else:
983
+ parts = list(payload_src)
1329
984
 
1330
- def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
1331
- """Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
985
+ if (mn := options.main_name_override) is not None:
986
+ parts.insert(0, f'__name__ = {mn!r}')
1332
987
 
1333
- def coord_repr(src: str, pos: TomlPos) -> str:
1334
- if pos >= len(src):
1335
- return 'end of document'
1336
- line = src.count('\n', 0, pos) + 1
1337
- if line == 1:
1338
- column = pos + 1
988
+ if len(parts) == 1:
989
+ return parts[0]
1339
990
  else:
1340
- column = pos - src.rindex('\n', 0, pos)
1341
- return f'line {line}, column {column}'
991
+ return '\n\n'.join(parts)
1342
992
 
1343
- return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
993
+ #
1344
994
 
995
+ @dc.dataclass(frozen=True)
996
+ class Read:
997
+ sz: int
1345
998
 
1346
- def toml_is_unicode_scalar_value(codepoint: int) -> bool:
1347
- return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
999
+ @dc.dataclass(frozen=True)
1000
+ class Write:
1001
+ d: bytes
1348
1002
 
1003
+ class ProtocolError(Exception):
1004
+ pass
1349
1005
 
1350
- def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
1351
- """A decorator to make `parse_float` safe.
1006
+ @dc.dataclass(frozen=True)
1007
+ class Result:
1008
+ pid: int
1009
+ env_info: PyremoteEnvInfo
1352
1010
 
1353
- `parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
1354
- thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
1355
- """
1356
- # The default `float` callable never returns illegal types. Optimize it.
1357
- if parse_float is float:
1358
- return float
1011
+ def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
1012
+ # Read first ack (after fork)
1013
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
1359
1014
 
1360
- def safe_parse_float(float_str: str) -> ta.Any:
1361
- float_value = parse_float(float_str)
1362
- if isinstance(float_value, (dict, list)):
1363
- raise ValueError('parse_float must not return dicts or lists') # noqa
1364
- return float_value
1015
+ # Read pid
1016
+ d = yield from self._read(8)
1017
+ pid = struct.unpack('<Q', d)[0]
1365
1018
 
1366
- return safe_parse_float
1019
+ # Write payload src
1020
+ yield from self._write(struct.pack('<I', len(self._payload_z)))
1021
+ yield from self._write(self._payload_z)
1367
1022
 
1023
+ # Read second ack (after writing src copies)
1024
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
1368
1025
 
1369
- ########################################
1370
- # ../config.py
1026
+ # Write options
1027
+ yield from self._write(struct.pack('<I', len(self._options_json)))
1028
+ yield from self._write(self._options_json)
1371
1029
 
1030
+ # Read third ack (after reaping child process)
1031
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
1372
1032
 
1373
- @dc.dataclass(frozen=True)
1374
- class MainConfig:
1375
- log_level: ta.Optional[str] = 'INFO'
1033
+ # Read env info
1034
+ d = yield from self._read(4)
1035
+ env_info_json_len = struct.unpack('<I', d)[0]
1036
+ d = yield from self._read(env_info_json_len)
1037
+ env_info_json = d.decode('utf-8')
1038
+ env_info = PyremoteEnvInfo(**json.loads(env_info_json))
1376
1039
 
1377
- debug: bool = False
1040
+ # Read fourth ack (after finalization completed)
1041
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
1378
1042
 
1043
+ # Return
1044
+ return self.Result(
1045
+ pid=pid,
1046
+ env_info=env_info,
1047
+ )
1379
1048
 
1380
- ########################################
1381
- # ../deploy/config.py
1049
+ def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
1050
+ d = yield self.Read(sz)
1051
+ if not isinstance(d, bytes):
1052
+ raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
1053
+ if len(d) != sz:
1054
+ raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
1055
+ return d
1382
1056
 
1057
+ def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
1058
+ d = yield from self._read(len(e))
1059
+ if d != e:
1060
+ raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
1383
1061
 
1384
- ##
1062
+ def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
1063
+ i = yield self.Write(d)
1064
+ if i is not None:
1065
+ raise self.ProtocolError('Unexpected input after write')
1385
1066
 
1067
+ #
1386
1068
 
1387
- @dc.dataclass(frozen=True)
1388
- class DeployConfig:
1389
- pass
1069
+ def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
1070
+ gen = self.gen()
1390
1071
 
1072
+ gi: ta.Optional[bytes] = None
1073
+ while True:
1074
+ try:
1075
+ if gi is not None:
1076
+ go = gen.send(gi)
1077
+ else:
1078
+ go = next(gen)
1079
+ except StopIteration as e:
1080
+ return e.value
1391
1081
 
1392
- ########################################
1393
- # ../deploy/paths/types.py
1082
+ if isinstance(go, self.Read):
1083
+ if len(gi := input.read(go.sz)) != go.sz:
1084
+ raise EOFError
1085
+ elif isinstance(go, self.Write):
1086
+ gi = None
1087
+ output.write(go.d)
1088
+ output.flush()
1089
+ else:
1090
+ raise TypeError(go)
1394
1091
 
1092
+ async def async_run(
1093
+ self,
1094
+ input: ta.Any, # asyncio.StreamWriter # noqa
1095
+ output: ta.Any, # asyncio.StreamReader
1096
+ ) -> Result:
1097
+ gen = self.gen()
1395
1098
 
1396
- ##
1099
+ gi: ta.Optional[bytes] = None
1100
+ while True:
1101
+ try:
1102
+ if gi is not None:
1103
+ go = gen.send(gi)
1104
+ else:
1105
+ go = next(gen)
1106
+ except StopIteration as e:
1107
+ return e.value
1108
+
1109
+ if isinstance(go, self.Read):
1110
+ if len(gi := await input.read(go.sz)) != go.sz:
1111
+ raise EOFError
1112
+ elif isinstance(go, self.Write):
1113
+ gi = None
1114
+ output.write(go.d)
1115
+ await output.drain()
1116
+ else:
1117
+ raise TypeError(go)
1397
1118
 
1398
1119
 
1399
1120
  ########################################
1400
- # ../deploy/types.py
1121
+ # ../../../omlish/asyncs/asyncio/channels.py
1401
1122
 
1402
1123
 
1403
- ##
1124
+ class AsyncioBytesChannelTransport(asyncio.Transport):
1125
+ def __init__(self, reader: asyncio.StreamReader) -> None:
1126
+ super().__init__()
1404
1127
 
1128
+ self.reader = reader
1129
+ self.closed: asyncio.Future = asyncio.Future()
1405
1130
 
1406
- DeployHome = ta.NewType('DeployHome', str)
1131
+ # @ta.override
1132
+ def write(self, data: bytes) -> None:
1133
+ self.reader.feed_data(data)
1407
1134
 
1408
- DeployRev = ta.NewType('DeployRev', str)
1135
+ # @ta.override
1136
+ def close(self) -> None:
1137
+ self.reader.feed_eof()
1138
+ if not self.closed.done():
1139
+ self.closed.set_result(True)
1409
1140
 
1141
+ # @ta.override
1142
+ def is_closing(self) -> bool:
1143
+ return self.closed.done()
1410
1144
 
1411
- ########################################
1412
- # ../../pyremote.py
1413
- """
1414
- Basically this: https://mitogen.networkgenomics.com/howitworks.html
1415
1145
 
1416
- TODO:
1417
- - log: ta.Optional[logging.Logger] = None + log.debug's
1418
- """
1146
+ def asyncio_create_bytes_channel(
1147
+ loop: ta.Any = None,
1148
+ ) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
1149
+ if loop is None:
1150
+ loop = asyncio.get_running_loop()
1419
1151
 
1152
+ reader = asyncio.StreamReader()
1153
+ protocol = asyncio.StreamReaderProtocol(reader)
1154
+ transport = AsyncioBytesChannelTransport(reader)
1155
+ writer = asyncio.StreamWriter(transport, protocol, reader, loop)
1420
1156
 
1421
- ##
1157
+ return reader, writer
1422
1158
 
1423
1159
 
1424
- @dc.dataclass(frozen=True)
1425
- class PyremoteBootstrapOptions:
1426
- debug: bool = False
1160
+ ########################################
1161
+ # ../../../omlish/asyncs/asyncio/streams.py
1427
1162
 
1428
- DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
1429
- main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
1430
1163
 
1164
+ ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
1431
1165
 
1432
- ##
1433
1166
 
1167
+ async def asyncio_open_stream_reader(
1168
+ f: ta.IO,
1169
+ loop: ta.Any = None,
1170
+ *,
1171
+ limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1172
+ ) -> asyncio.StreamReader:
1173
+ if loop is None:
1174
+ loop = asyncio.get_running_loop()
1434
1175
 
1435
- @dc.dataclass(frozen=True)
1436
- class PyremoteEnvInfo:
1437
- sys_base_prefix: str
1438
- sys_byteorder: str
1439
- sys_defaultencoding: str
1440
- sys_exec_prefix: str
1441
- sys_executable: str
1442
- sys_implementation_name: str
1443
- sys_path: ta.List[str]
1444
- sys_platform: str
1445
- sys_prefix: str
1446
- sys_version: str
1447
- sys_version_info: ta.List[ta.Union[int, str]]
1176
+ reader = asyncio.StreamReader(limit=limit, loop=loop)
1177
+ await loop.connect_read_pipe(
1178
+ lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1179
+ f,
1180
+ )
1448
1181
 
1449
- platform_architecture: ta.List[str]
1450
- platform_machine: str
1451
- platform_platform: str
1452
- platform_processor: str
1453
- platform_system: str
1454
- platform_release: str
1455
- platform_version: str
1182
+ return reader
1456
1183
 
1457
- site_userbase: str
1458
1184
 
1459
- os_cwd: str
1460
- os_gid: int
1461
- os_loadavg: ta.List[float]
1462
- os_login: ta.Optional[str]
1463
- os_pgrp: int
1464
- os_pid: int
1465
- os_ppid: int
1466
- os_uid: int
1185
+ async def asyncio_open_stream_writer(
1186
+ f: ta.IO,
1187
+ loop: ta.Any = None,
1188
+ ) -> asyncio.StreamWriter:
1189
+ if loop is None:
1190
+ loop = asyncio.get_running_loop()
1467
1191
 
1468
- pw_name: str
1469
- pw_uid: int
1470
- pw_gid: int
1471
- pw_gecos: str
1472
- pw_dir: str
1473
- pw_shell: str
1192
+ writer_transport, writer_protocol = await loop.connect_write_pipe(
1193
+ lambda: asyncio.streams.FlowControlMixin(loop=loop),
1194
+ f,
1195
+ )
1474
1196
 
1475
- env_path: ta.Optional[str]
1197
+ return asyncio.streams.StreamWriter(
1198
+ writer_transport,
1199
+ writer_protocol,
1200
+ None,
1201
+ loop,
1202
+ )
1476
1203
 
1477
1204
 
1478
- def _get_pyremote_env_info() -> PyremoteEnvInfo:
1479
- os_uid = os.getuid()
1205
+ ########################################
1206
+ # ../../../omlish/asyncs/asyncio/timeouts.py
1480
1207
 
1481
- pw = pwd.getpwuid(os_uid)
1482
1208
 
1483
- os_login: ta.Optional[str]
1484
- try:
1485
- os_login = os.getlogin()
1486
- except OSError:
1487
- os_login = None
1209
+ def asyncio_maybe_timeout(
1210
+ fut: AwaitableT,
1211
+ timeout: ta.Optional[float] = None,
1212
+ ) -> AwaitableT:
1213
+ if timeout is not None:
1214
+ fut = asyncio.wait_for(fut, timeout) # type: ignore
1215
+ return fut
1488
1216
 
1489
- return PyremoteEnvInfo(
1490
- sys_base_prefix=sys.base_prefix,
1491
- sys_byteorder=sys.byteorder,
1492
- sys_defaultencoding=sys.getdefaultencoding(),
1493
- sys_exec_prefix=sys.exec_prefix,
1494
- sys_executable=sys.executable,
1495
- sys_implementation_name=sys.implementation.name,
1496
- sys_path=sys.path,
1497
- sys_platform=sys.platform,
1498
- sys_prefix=sys.prefix,
1499
- sys_version=sys.version,
1500
- sys_version_info=list(sys.version_info),
1501
1217
 
1502
- platform_architecture=list(platform.architecture()),
1503
- platform_machine=platform.machine(),
1504
- platform_platform=platform.platform(),
1505
- platform_processor=platform.processor(),
1506
- platform_system=platform.system(),
1507
- platform_release=platform.release(),
1508
- platform_version=platform.version(),
1218
+ ########################################
1219
+ # ../../../omlish/configs/types.py
1509
1220
 
1510
- site_userbase=site.getuserbase(),
1511
1221
 
1512
- os_cwd=os.getcwd(),
1513
- os_gid=os.getgid(),
1514
- os_loadavg=list(os.getloadavg()),
1515
- os_login=os_login,
1516
- os_pgrp=os.getpgrp(),
1517
- os_pid=os.getpid(),
1518
- os_ppid=os.getppid(),
1519
- os_uid=os_uid,
1222
+ #
1520
1223
 
1521
- pw_name=pw.pw_name,
1522
- pw_uid=pw.pw_uid,
1523
- pw_gid=pw.pw_gid,
1524
- pw_gecos=pw.pw_gecos,
1525
- pw_dir=pw.pw_dir,
1526
- pw_shell=pw.pw_shell,
1527
1224
 
1528
- env_path=os.environ.get('PATH'),
1529
- )
1225
+ ########################################
1226
+ # ../../../omlish/formats/ini/sections.py
1530
1227
 
1531
1228
 
1532
1229
  ##
1533
1230
 
1534
1231
 
1535
- _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
1536
- _PYREMOTE_BOOTSTRAP_SRC_FD = 101
1537
-
1538
- _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
1539
- _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
1540
- _PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
1541
- _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
1542
- _PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
1232
+ def extract_ini_sections(cp: configparser.ConfigParser) -> IniSectionSettingsMap:
1233
+ config_dct: ta.Dict[str, ta.Any] = {}
1234
+ for sec in cp.sections():
1235
+ cd = config_dct
1236
+ for k in sec.split('.'):
1237
+ cd = cd.setdefault(k, {})
1238
+ cd.update(cp.items(sec))
1239
+ return config_dct
1543
1240
 
1544
- _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
1545
- _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
1546
- _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
1547
- _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
1548
1241
 
1549
- _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
1242
+ ##
1550
1243
 
1551
- _PYREMOTE_BOOTSTRAP_IMPORTS = [
1552
- 'base64',
1553
- 'os',
1554
- 'struct',
1555
- 'sys',
1556
- 'zlib',
1557
- ]
1558
1244
 
1245
+ def render_ini_sections(
1246
+ settings_by_section: IniSectionSettingsMap,
1247
+ ) -> str:
1248
+ out = io.StringIO()
1559
1249
 
1560
- def _pyremote_bootstrap_main(context_name: str) -> None:
1561
- # Get pid
1562
- pid = os.getpid()
1250
+ for i, (section, settings) in enumerate(settings_by_section.items()):
1251
+ if i:
1252
+ out.write('\n')
1563
1253
 
1564
- # Two copies of payload src to be sent to parent
1565
- r0, w0 = os.pipe()
1566
- r1, w1 = os.pipe()
1254
+ out.write(f'[{section}]\n')
1567
1255
 
1568
- if (cp := os.fork()):
1569
- # Parent process
1256
+ for k, v in settings.items():
1257
+ if isinstance(v, str):
1258
+ out.write(f'{k}={v}\n')
1259
+ else:
1260
+ for vv in v:
1261
+ out.write(f'{k}={vv}\n')
1570
1262
 
1571
- # Dup original stdin to comm_fd for use as comm channel
1572
- os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
1263
+ return out.getvalue()
1573
1264
 
1574
- # Overwrite stdin (fed to python repl) with first copy of src
1575
- os.dup2(r0, 0)
1576
1265
 
1577
- # Dup second copy of src to src_fd to recover after launch
1578
- os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
1266
+ ########################################
1267
+ # ../../../omlish/formats/toml/parser.py
1268
+ # SPDX-License-Identifier: MIT
1269
+ # SPDX-FileCopyrightText: 2021 Taneli Hukkinen
1270
+ # Licensed to PSF under a Contributor Agreement.
1271
+ #
1272
+ # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
1273
+ # --------------------------------------------
1274
+ #
1275
+ # 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
1276
+ # ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
1277
+ # documentation.
1278
+ #
1279
+ # 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
1280
+ # royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
1281
+ # works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
1282
+ # Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
1283
+ # 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
1284
+ # Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
1285
+ #
1286
+ # 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
1287
+ # wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
1288
+ # any such work a brief summary of the changes made to Python.
1289
+ #
1290
+ # 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
1291
+ # EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
1292
+ # OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
1293
+ # RIGHTS.
1294
+ #
1295
+ # 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
1296
+ # DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
1297
+ # ADVISED OF THE POSSIBILITY THEREOF.
1298
+ #
1299
+ # 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
1300
+ #
1301
+ # 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
1302
+ # venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
1303
+ # name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
1304
+ #
1305
+ # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
1306
+ # License Agreement.
1307
+ #
1308
+ # https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
1579
1309
 
1580
- # Close remaining fd's
1581
- for f in [r0, w0, r1, w1]:
1582
- os.close(f)
1583
1310
 
1584
- # Save vars
1585
- env = os.environ
1586
- exe = sys.executable
1587
- env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
1588
- env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
1589
- env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
1311
+ ##
1590
1312
 
1591
- # Start repl reading stdin from r0
1592
- os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
1593
1313
 
1594
- else:
1595
- # Child process
1314
+ _TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
1596
1315
 
1597
- # Write first ack
1598
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
1316
+ TOML_RE_NUMBER = re.compile(
1317
+ r"""
1318
+ 0
1319
+ (?:
1320
+ x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
1321
+ |
1322
+ b[01](?:_?[01])* # bin
1323
+ |
1324
+ o[0-7](?:_?[0-7])* # oct
1325
+ )
1326
+ |
1327
+ [+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
1328
+ (?P<floatpart>
1329
+ (?:\.[0-9](?:_?[0-9])*)? # optional fractional part
1330
+ (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
1331
+ )
1332
+ """,
1333
+ flags=re.VERBOSE,
1334
+ )
1335
+ TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
1336
+ TOML_RE_DATETIME = re.compile(
1337
+ rf"""
1338
+ ([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
1339
+ (?:
1340
+ [Tt ]
1341
+ {_TOML_TIME_RE_STR}
1342
+ (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
1343
+ )?
1344
+ """,
1345
+ flags=re.VERBOSE,
1346
+ )
1599
1347
 
1600
- # Write pid
1601
- os.write(1, struct.pack('<Q', pid))
1602
1348
 
1603
- # Read payload src from stdin
1604
- payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
1605
- if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
1606
- raise EOFError
1607
- payload_src = zlib.decompress(payload_z)
1349
+ def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
1350
+ """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
1608
1351
 
1609
- # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
1610
- # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
1611
- for w in [w0, w1]:
1612
- fp = os.fdopen(w, 'wb', 0)
1613
- fp.write(payload_src)
1614
- fp.close()
1352
+ Raises ValueError if the match does not correspond to a valid date or datetime.
1353
+ """
1354
+ (
1355
+ year_str,
1356
+ month_str,
1357
+ day_str,
1358
+ hour_str,
1359
+ minute_str,
1360
+ sec_str,
1361
+ micros_str,
1362
+ zulu_time,
1363
+ offset_sign_str,
1364
+ offset_hour_str,
1365
+ offset_minute_str,
1366
+ ) = match.groups()
1367
+ year, month, day = int(year_str), int(month_str), int(day_str)
1368
+ if hour_str is None:
1369
+ return datetime.date(year, month, day)
1370
+ hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
1371
+ micros = int(micros_str.ljust(6, '0')) if micros_str else 0
1372
+ if offset_sign_str:
1373
+ tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
1374
+ offset_hour_str, offset_minute_str, offset_sign_str,
1375
+ )
1376
+ elif zulu_time:
1377
+ tz = datetime.UTC
1378
+ else: # local date-time
1379
+ tz = None
1380
+ return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
1615
1381
 
1616
- # Write second ack
1617
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
1618
1382
 
1619
- # Exit child
1620
- sys.exit(0)
1383
+ @functools.lru_cache() # noqa
1384
+ def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
1385
+ sign = 1 if sign_str == '+' else -1
1386
+ return datetime.timezone(
1387
+ datetime.timedelta(
1388
+ hours=sign * int(hour_str),
1389
+ minutes=sign * int(minute_str),
1390
+ ),
1391
+ )
1621
1392
 
1622
1393
 
1623
- ##
1394
+ def toml_match_to_localtime(match: re.Match) -> datetime.time:
1395
+ hour_str, minute_str, sec_str, micros_str = match.groups()
1396
+ micros = int(micros_str.ljust(6, '0')) if micros_str else 0
1397
+ return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
1624
1398
 
1625
1399
 
1626
- def pyremote_build_bootstrap_cmd(context_name: str) -> str:
1627
- if any(c in context_name for c in '\'"'):
1628
- raise NameError(context_name)
1400
+ def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
1401
+ if match.group('floatpart'):
1402
+ return parse_float(match.group())
1403
+ return int(match.group(), 0)
1629
1404
 
1630
- import inspect
1631
- import textwrap
1632
- bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
1633
1405
 
1634
- for gl in [
1635
- '_PYREMOTE_BOOTSTRAP_INPUT_FD',
1636
- '_PYREMOTE_BOOTSTRAP_SRC_FD',
1406
+ TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
1637
1407
 
1638
- '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
1639
- '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
1640
- '_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
1408
+ # Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
1409
+ # functions.
1410
+ TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
1411
+ TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
1641
1412
 
1642
- '_PYREMOTE_BOOTSTRAP_ACK0',
1643
- '_PYREMOTE_BOOTSTRAP_ACK1',
1413
+ TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
1414
+ TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1644
1415
 
1645
- '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
1646
- ]:
1647
- bs_src = bs_src.replace(gl, repr(globals()[gl]))
1416
+ TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
1648
1417
 
1649
- bs_src = '\n'.join(
1650
- cl
1651
- for l in bs_src.splitlines()
1652
- if (cl := (l.split('#')[0]).rstrip())
1653
- if cl.strip()
1654
- )
1655
-
1656
- bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
1657
- bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
1658
- if b'"' in bs_z85:
1659
- raise ValueError(bs_z85)
1660
-
1661
- stmts = [
1662
- f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
1663
- f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
1664
- f'_pyremote_bootstrap_main("{context_name}")',
1665
- ]
1418
+ TOML_WS = frozenset(' \t')
1419
+ TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
1420
+ TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
1421
+ TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
1422
+ TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
1666
1423
 
1667
- cmd = '; '.join(stmts)
1668
- return cmd
1424
+ TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
1425
+ {
1426
+ '\\b': '\u0008', # backspace
1427
+ '\\t': '\u0009', # tab
1428
+ '\\n': '\u000A', # linefeed
1429
+ '\\f': '\u000C', # form feed
1430
+ '\\r': '\u000D', # carriage return
1431
+ '\\"': '\u0022', # quote
1432
+ '\\\\': '\u005C', # backslash
1433
+ },
1434
+ )
1669
1435
 
1670
1436
 
1671
- ##
1437
+ class TomlDecodeError(ValueError):
1438
+ """An error raised if a document is not valid TOML."""
1672
1439
 
1673
1440
 
1674
- @dc.dataclass(frozen=True)
1675
- class PyremotePayloadRuntime:
1676
- input: ta.BinaryIO
1677
- output: ta.BinaryIO
1678
- context_name: str
1679
- payload_src: str
1680
- options: PyremoteBootstrapOptions
1681
- env_info: PyremoteEnvInfo
1441
+ def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
1442
+ """Parse TOML from a binary file object."""
1443
+ b = fp.read()
1444
+ try:
1445
+ s = b.decode()
1446
+ except AttributeError:
1447
+ raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
1448
+ return toml_loads(s, parse_float=parse_float)
1682
1449
 
1683
1450
 
1684
- def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
1685
- # If src file var is not present we need to do initial finalization
1686
- if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
1687
- # Read second copy of payload src
1688
- r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
1689
- payload_src = r1.read().decode('utf-8')
1690
- r1.close()
1451
+ def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
1452
+ """Parse TOML from a string."""
1691
1453
 
1692
- # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
1693
- # a pipe at once.
1694
- os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
1454
+ # The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
1455
+ try:
1456
+ src = s.replace('\r\n', '\n')
1457
+ except (AttributeError, TypeError):
1458
+ raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
1459
+ pos = 0
1460
+ out = TomlOutput(TomlNestedDict(), TomlFlags())
1461
+ header: TomlKey = ()
1462
+ parse_float = toml_make_safe_parse_float(parse_float)
1695
1463
 
1696
- # Read options
1697
- options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
1698
- if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
1699
- raise EOFError
1700
- options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
1464
+ # Parse one statement at a time (typically means one line in TOML source)
1465
+ while True:
1466
+ # 1. Skip line leading whitespace
1467
+ pos = toml_skip_chars(src, pos, TOML_WS)
1701
1468
 
1702
- # If debugging, re-exec as file
1703
- if options.debug:
1704
- # Write temp source file
1705
- import tempfile
1706
- tfd, tfn = tempfile.mkstemp('-pyremote.py')
1707
- os.write(tfd, payload_src.encode('utf-8'))
1708
- os.close(tfd)
1469
+ # 2. Parse rules. Expect one of the following:
1470
+ # - end of file
1471
+ # - end of line
1472
+ # - comment
1473
+ # - key/value pair
1474
+ # - append dict to list (and move to its namespace)
1475
+ # - create dict (and move to its namespace)
1476
+ # Skip trailing whitespace when applicable.
1477
+ try:
1478
+ char = src[pos]
1479
+ except IndexError:
1480
+ break
1481
+ if char == '\n':
1482
+ pos += 1
1483
+ continue
1484
+ if char in TOML_KEY_INITIAL_CHARS:
1485
+ pos = toml_key_value_rule(src, pos, out, header, parse_float)
1486
+ pos = toml_skip_chars(src, pos, TOML_WS)
1487
+ elif char == '[':
1488
+ try:
1489
+ second_char: ta.Optional[str] = src[pos + 1]
1490
+ except IndexError:
1491
+ second_char = None
1492
+ out.flags.finalize_pending()
1493
+ if second_char == '[':
1494
+ pos, header = toml_create_list_rule(src, pos, out)
1495
+ else:
1496
+ pos, header = toml_create_dict_rule(src, pos, out)
1497
+ pos = toml_skip_chars(src, pos, TOML_WS)
1498
+ elif char != '#':
1499
+ raise toml_suffixed_err(src, pos, 'Invalid statement')
1709
1500
 
1710
- # Set vars
1711
- os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
1712
- os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
1501
+ # 3. Skip comment
1502
+ pos = toml_skip_comment(src, pos)
1713
1503
 
1714
- # Re-exec temp file
1715
- exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
1716
- context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
1717
- os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
1504
+ # 4. Expect end of line or end of file
1505
+ try:
1506
+ char = src[pos]
1507
+ except IndexError:
1508
+ break
1509
+ if char != '\n':
1510
+ raise toml_suffixed_err(
1511
+ src, pos, 'Expected newline or end of document after a statement',
1512
+ )
1513
+ pos += 1
1718
1514
 
1719
- else:
1720
- # Load options json var
1721
- options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
1722
- options = PyremoteBootstrapOptions(**json.loads(options_json_str))
1515
+ return out.data.dict
1723
1516
 
1724
- # Read temp source file
1725
- with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
1726
- payload_src = sf.read()
1727
1517
 
1728
- # Restore vars
1729
- sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
1730
- context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
1518
+ class TomlFlags:
1519
+ """Flags that map to parsed keys/namespaces."""
1731
1520
 
1732
- # Write third ack
1733
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
1521
+ # Marks an immutable namespace (inline array or inline table).
1522
+ FROZEN = 0
1523
+ # Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
1524
+ EXPLICIT_NEST = 1
1734
1525
 
1735
- # Write env info
1736
- env_info = _get_pyremote_env_info()
1737
- env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
1738
- os.write(1, struct.pack('<I', len(env_info_json)))
1739
- os.write(1, env_info_json.encode('utf-8'))
1526
+ def __init__(self) -> None:
1527
+ self._flags: ta.Dict[str, dict] = {}
1528
+ self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
1740
1529
 
1741
- # Setup IO
1742
- input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
1743
- output = os.fdopen(os.dup(1), 'wb', 0) # noqa
1744
- os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
1745
- os.close(nfd)
1530
+ def add_pending(self, key: TomlKey, flag: int) -> None:
1531
+ self._pending_flags.add((key, flag))
1746
1532
 
1747
- if (mn := options.main_name_override) is not None:
1748
- # Inspections like typing.get_type_hints need an entry in sys.modules.
1749
- sys.modules[mn] = sys.modules['__main__']
1533
+ def finalize_pending(self) -> None:
1534
+ for key, flag in self._pending_flags:
1535
+ self.set(key, flag, recursive=False)
1536
+ self._pending_flags.clear()
1750
1537
 
1751
- # Write fourth ack
1752
- output.write(_PYREMOTE_BOOTSTRAP_ACK3)
1538
+ def unset_all(self, key: TomlKey) -> None:
1539
+ cont = self._flags
1540
+ for k in key[:-1]:
1541
+ if k not in cont:
1542
+ return
1543
+ cont = cont[k]['nested']
1544
+ cont.pop(key[-1], None)
1753
1545
 
1754
- # Return
1755
- return PyremotePayloadRuntime(
1756
- input=input,
1757
- output=output,
1758
- context_name=context_name,
1759
- payload_src=payload_src,
1760
- options=options,
1761
- env_info=env_info,
1762
- )
1546
+ def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
1547
+ cont = self._flags
1548
+ key_parent, key_stem = key[:-1], key[-1]
1549
+ for k in key_parent:
1550
+ if k not in cont:
1551
+ cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
1552
+ cont = cont[k]['nested']
1553
+ if key_stem not in cont:
1554
+ cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
1555
+ cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
1763
1556
 
1557
+ def is_(self, key: TomlKey, flag: int) -> bool:
1558
+ if not key:
1559
+ return False # document root has no flags
1560
+ cont = self._flags
1561
+ for k in key[:-1]:
1562
+ if k not in cont:
1563
+ return False
1564
+ inner_cont = cont[k]
1565
+ if flag in inner_cont['recursive_flags']:
1566
+ return True
1567
+ cont = inner_cont['nested']
1568
+ key_stem = key[-1]
1569
+ if key_stem in cont:
1570
+ cont = cont[key_stem]
1571
+ return flag in cont['flags'] or flag in cont['recursive_flags']
1572
+ return False
1764
1573
 
1765
- ##
1766
1574
 
1575
+ class TomlNestedDict:
1576
+ def __init__(self) -> None:
1577
+ # The parsed content of the TOML document
1578
+ self.dict: ta.Dict[str, ta.Any] = {}
1767
1579
 
1768
- class PyremoteBootstrapDriver:
1769
- def __init__(
1580
+ def get_or_create_nest(
1770
1581
  self,
1771
- payload_src: ta.Union[str, ta.Sequence[str]],
1772
- options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
1773
- ) -> None:
1774
- super().__init__()
1775
-
1776
- self._payload_src = payload_src
1777
- self._options = options
1582
+ key: TomlKey,
1583
+ *,
1584
+ access_lists: bool = True,
1585
+ ) -> dict:
1586
+ cont: ta.Any = self.dict
1587
+ for k in key:
1588
+ if k not in cont:
1589
+ cont[k] = {}
1590
+ cont = cont[k]
1591
+ if access_lists and isinstance(cont, list):
1592
+ cont = cont[-1]
1593
+ if not isinstance(cont, dict):
1594
+ raise KeyError('There is no nest behind this key')
1595
+ return cont
1778
1596
 
1779
- self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
1780
- self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
1597
+ def append_nest_to_list(self, key: TomlKey) -> None:
1598
+ cont = self.get_or_create_nest(key[:-1])
1599
+ last_key = key[-1]
1600
+ if last_key in cont:
1601
+ list_ = cont[last_key]
1602
+ if not isinstance(list_, list):
1603
+ raise KeyError('An object other than list found behind this key')
1604
+ list_.append({})
1605
+ else:
1606
+ cont[last_key] = [{}]
1781
1607
 
1782
- self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
1783
- #
1784
1608
 
1785
- @classmethod
1786
- def _prepare_payload_src(
1787
- cls,
1788
- payload_src: ta.Union[str, ta.Sequence[str]],
1789
- options: PyremoteBootstrapOptions,
1790
- ) -> str:
1791
- parts: ta.List[str]
1792
- if isinstance(payload_src, str):
1793
- parts = [payload_src]
1794
- else:
1795
- parts = list(payload_src)
1609
+ class TomlOutput(ta.NamedTuple):
1610
+ data: TomlNestedDict
1611
+ flags: TomlFlags
1796
1612
 
1797
- if (mn := options.main_name_override) is not None:
1798
- parts.insert(0, f'__name__ = {mn!r}')
1799
1613
 
1800
- if len(parts) == 1:
1801
- return parts[0]
1802
- else:
1803
- return '\n\n'.join(parts)
1614
+ def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
1615
+ try:
1616
+ while src[pos] in chars:
1617
+ pos += 1
1618
+ except IndexError:
1619
+ pass
1620
+ return pos
1804
1621
 
1805
- #
1806
1622
 
1807
- @dc.dataclass(frozen=True)
1808
- class Read:
1809
- sz: int
1623
+ def toml_skip_until(
1624
+ src: str,
1625
+ pos: TomlPos,
1626
+ expect: str,
1627
+ *,
1628
+ error_on: ta.FrozenSet[str],
1629
+ error_on_eof: bool,
1630
+ ) -> TomlPos:
1631
+ try:
1632
+ new_pos = src.index(expect, pos)
1633
+ except ValueError:
1634
+ new_pos = len(src)
1635
+ if error_on_eof:
1636
+ raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
1810
1637
 
1811
- @dc.dataclass(frozen=True)
1812
- class Write:
1813
- d: bytes
1638
+ if not error_on.isdisjoint(src[pos:new_pos]):
1639
+ while src[pos] not in error_on:
1640
+ pos += 1
1641
+ raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
1642
+ return new_pos
1814
1643
 
1815
- class ProtocolError(Exception):
1816
- pass
1817
1644
 
1818
- @dc.dataclass(frozen=True)
1819
- class Result:
1820
- pid: int
1821
- env_info: PyremoteEnvInfo
1645
+ def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
1646
+ try:
1647
+ char: ta.Optional[str] = src[pos]
1648
+ except IndexError:
1649
+ char = None
1650
+ if char == '#':
1651
+ return toml_skip_until(
1652
+ src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
1653
+ )
1654
+ return pos
1822
1655
 
1823
- def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
1824
- # Read first ack (after fork)
1825
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
1826
1656
 
1827
- # Read pid
1828
- d = yield from self._read(8)
1829
- pid = struct.unpack('<Q', d)[0]
1657
+ def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
1658
+ while True:
1659
+ pos_before_skip = pos
1660
+ pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1661
+ pos = toml_skip_comment(src, pos)
1662
+ if pos == pos_before_skip:
1663
+ return pos
1664
+
1665
+
1666
+ def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
1667
+ pos += 1 # Skip "["
1668
+ pos = toml_skip_chars(src, pos, TOML_WS)
1669
+ pos, key = toml_parse_key(src, pos)
1670
+
1671
+ if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
1672
+ raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
1673
+ out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
1674
+ try:
1675
+ out.data.get_or_create_nest(key)
1676
+ except KeyError:
1677
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1678
+
1679
+ if not src.startswith(']', pos):
1680
+ raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
1681
+ return pos + 1, key
1682
+
1683
+
1684
+ def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
1685
+ pos += 2 # Skip "[["
1686
+ pos = toml_skip_chars(src, pos, TOML_WS)
1687
+ pos, key = toml_parse_key(src, pos)
1688
+
1689
+ if out.flags.is_(key, TomlFlags.FROZEN):
1690
+ raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1691
+ # Free the namespace now that it points to another empty list item...
1692
+ out.flags.unset_all(key)
1693
+ # ...but this key precisely is still prohibited from table declaration
1694
+ out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
1695
+ try:
1696
+ out.data.append_nest_to_list(key)
1697
+ except KeyError:
1698
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1699
+
1700
+ if not src.startswith(']]', pos):
1701
+ raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
1702
+ return pos + 2, key
1703
+
1704
+
1705
+ def toml_key_value_rule(
1706
+ src: str,
1707
+ pos: TomlPos,
1708
+ out: TomlOutput,
1709
+ header: TomlKey,
1710
+ parse_float: TomlParseFloat,
1711
+ ) -> TomlPos:
1712
+ pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1713
+ key_parent, key_stem = key[:-1], key[-1]
1714
+ abs_key_parent = header + key_parent
1715
+
1716
+ relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
1717
+ for cont_key in relative_path_cont_keys:
1718
+ # Check that dotted key syntax does not redefine an existing table
1719
+ if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
1720
+ raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
1721
+ # Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
1722
+ # table sections.
1723
+ out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
1724
+
1725
+ if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
1726
+ raise toml_suffixed_err(
1727
+ src,
1728
+ pos,
1729
+ f'Cannot mutate immutable namespace {abs_key_parent}',
1730
+ )
1731
+
1732
+ try:
1733
+ nest = out.data.get_or_create_nest(abs_key_parent)
1734
+ except KeyError:
1735
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1736
+ if key_stem in nest:
1737
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
1738
+ # Mark inline table and array namespaces recursively immutable
1739
+ if isinstance(value, (dict, list)):
1740
+ out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
1741
+ nest[key_stem] = value
1742
+ return pos
1743
+
1744
+
1745
+ def toml_parse_key_value_pair(
1746
+ src: str,
1747
+ pos: TomlPos,
1748
+ parse_float: TomlParseFloat,
1749
+ ) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
1750
+ pos, key = toml_parse_key(src, pos)
1751
+ try:
1752
+ char: ta.Optional[str] = src[pos]
1753
+ except IndexError:
1754
+ char = None
1755
+ if char != '=':
1756
+ raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
1757
+ pos += 1
1758
+ pos = toml_skip_chars(src, pos, TOML_WS)
1759
+ pos, value = toml_parse_value(src, pos, parse_float)
1760
+ return pos, key, value
1761
+
1762
+
1763
+ def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
1764
+ pos, key_part = toml_parse_key_part(src, pos)
1765
+ key: TomlKey = (key_part,)
1766
+ pos = toml_skip_chars(src, pos, TOML_WS)
1767
+ while True:
1768
+ try:
1769
+ char: ta.Optional[str] = src[pos]
1770
+ except IndexError:
1771
+ char = None
1772
+ if char != '.':
1773
+ return pos, key
1774
+ pos += 1
1775
+ pos = toml_skip_chars(src, pos, TOML_WS)
1776
+ pos, key_part = toml_parse_key_part(src, pos)
1777
+ key += (key_part,)
1778
+ pos = toml_skip_chars(src, pos, TOML_WS)
1779
+
1780
+
1781
+ def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1782
+ try:
1783
+ char: ta.Optional[str] = src[pos]
1784
+ except IndexError:
1785
+ char = None
1786
+ if char in TOML_BARE_KEY_CHARS:
1787
+ start_pos = pos
1788
+ pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
1789
+ return pos, src[start_pos:pos]
1790
+ if char == "'":
1791
+ return toml_parse_literal_str(src, pos)
1792
+ if char == '"':
1793
+ return toml_parse_one_line_basic_str(src, pos)
1794
+ raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
1795
+
1796
+
1797
+ def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1798
+ pos += 1
1799
+ return toml_parse_basic_str(src, pos, multiline=False)
1800
+
1801
+
1802
+ def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
1803
+ pos += 1
1804
+ array: list = []
1805
+
1806
+ pos = toml_skip_comments_and_array_ws(src, pos)
1807
+ if src.startswith(']', pos):
1808
+ return pos + 1, array
1809
+ while True:
1810
+ pos, val = toml_parse_value(src, pos, parse_float)
1811
+ array.append(val)
1812
+ pos = toml_skip_comments_and_array_ws(src, pos)
1813
+
1814
+ c = src[pos:pos + 1]
1815
+ if c == ']':
1816
+ return pos + 1, array
1817
+ if c != ',':
1818
+ raise toml_suffixed_err(src, pos, 'Unclosed array')
1819
+ pos += 1
1820
+
1821
+ pos = toml_skip_comments_and_array_ws(src, pos)
1822
+ if src.startswith(']', pos):
1823
+ return pos + 1, array
1824
+
1825
+
1826
+ def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
1827
+ pos += 1
1828
+ nested_dict = TomlNestedDict()
1829
+ flags = TomlFlags()
1830
+
1831
+ pos = toml_skip_chars(src, pos, TOML_WS)
1832
+ if src.startswith('}', pos):
1833
+ return pos + 1, nested_dict.dict
1834
+ while True:
1835
+ pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1836
+ key_parent, key_stem = key[:-1], key[-1]
1837
+ if flags.is_(key, TomlFlags.FROZEN):
1838
+ raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1839
+ try:
1840
+ nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
1841
+ except KeyError:
1842
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1843
+ if key_stem in nest:
1844
+ raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
1845
+ nest[key_stem] = value
1846
+ pos = toml_skip_chars(src, pos, TOML_WS)
1847
+ c = src[pos:pos + 1]
1848
+ if c == '}':
1849
+ return pos + 1, nested_dict.dict
1850
+ if c != ',':
1851
+ raise toml_suffixed_err(src, pos, 'Unclosed inline table')
1852
+ if isinstance(value, (dict, list)):
1853
+ flags.set(key, TomlFlags.FROZEN, recursive=True)
1854
+ pos += 1
1855
+ pos = toml_skip_chars(src, pos, TOML_WS)
1856
+
1857
+
1858
+ def toml_parse_basic_str_escape(
1859
+ src: str,
1860
+ pos: TomlPos,
1861
+ *,
1862
+ multiline: bool = False,
1863
+ ) -> ta.Tuple[TomlPos, str]:
1864
+ escape_id = src[pos:pos + 2]
1865
+ pos += 2
1866
+ if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
1867
+ # Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
1868
+ # newline.
1869
+ if escape_id != '\\\n':
1870
+ pos = toml_skip_chars(src, pos, TOML_WS)
1871
+ try:
1872
+ char = src[pos]
1873
+ except IndexError:
1874
+ return pos, ''
1875
+ if char != '\n':
1876
+ raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
1877
+ pos += 1
1878
+ pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1879
+ return pos, ''
1880
+ if escape_id == '\\u':
1881
+ return toml_parse_hex_char(src, pos, 4)
1882
+ if escape_id == '\\U':
1883
+ return toml_parse_hex_char(src, pos, 8)
1884
+ try:
1885
+ return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
1886
+ except KeyError:
1887
+ raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
1888
+
1889
+
1890
+ def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1891
+ return toml_parse_basic_str_escape(src, pos, multiline=True)
1892
+
1893
+
1894
+ def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
1895
+ hex_str = src[pos:pos + hex_len]
1896
+ if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
1897
+ raise toml_suffixed_err(src, pos, 'Invalid hex value')
1898
+ pos += hex_len
1899
+ hex_int = int(hex_str, 16)
1900
+ if not toml_is_unicode_scalar_value(hex_int):
1901
+ raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
1902
+ return pos, chr(hex_int)
1903
+
1904
+
1905
+ def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1906
+ pos += 1 # Skip starting apostrophe
1907
+ start_pos = pos
1908
+ pos = toml_skip_until(
1909
+ src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
1910
+ )
1911
+ return pos + 1, src[start_pos:pos] # Skip ending apostrophe
1830
1912
 
1831
- # Write payload src
1832
- yield from self._write(struct.pack('<I', len(self._payload_z)))
1833
- yield from self._write(self._payload_z)
1834
1913
 
1835
- # Read second ack (after writing src copies)
1836
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
1914
+ def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
1915
+ pos += 3
1916
+ if src.startswith('\n', pos):
1917
+ pos += 1
1837
1918
 
1838
- # Write options
1839
- yield from self._write(struct.pack('<I', len(self._options_json)))
1840
- yield from self._write(self._options_json)
1919
+ if literal:
1920
+ delim = "'"
1921
+ end_pos = toml_skip_until(
1922
+ src,
1923
+ pos,
1924
+ "'''",
1925
+ error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
1926
+ error_on_eof=True,
1927
+ )
1928
+ result = src[pos:end_pos]
1929
+ pos = end_pos + 3
1930
+ else:
1931
+ delim = '"'
1932
+ pos, result = toml_parse_basic_str(src, pos, multiline=True)
1841
1933
 
1842
- # Read third ack (after reaping child process)
1843
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
1934
+ # Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
1935
+ if not src.startswith(delim, pos):
1936
+ return pos, result
1937
+ pos += 1
1938
+ if not src.startswith(delim, pos):
1939
+ return pos, result + delim
1940
+ pos += 1
1941
+ return pos, result + (delim * 2)
1844
1942
 
1845
- # Read env info
1846
- d = yield from self._read(4)
1847
- env_info_json_len = struct.unpack('<I', d)[0]
1848
- d = yield from self._read(env_info_json_len)
1849
- env_info_json = d.decode('utf-8')
1850
- env_info = PyremoteEnvInfo(**json.loads(env_info_json))
1851
1943
 
1852
- # Read fourth ack (after finalization completed)
1853
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
1944
+ def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
1945
+ if multiline:
1946
+ error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1947
+ parse_escapes = toml_parse_basic_str_escape_multiline
1948
+ else:
1949
+ error_on = TOML_ILLEGAL_BASIC_STR_CHARS
1950
+ parse_escapes = toml_parse_basic_str_escape
1951
+ result = ''
1952
+ start_pos = pos
1953
+ while True:
1954
+ try:
1955
+ char = src[pos]
1956
+ except IndexError:
1957
+ raise toml_suffixed_err(src, pos, 'Unterminated string') from None
1958
+ if char == '"':
1959
+ if not multiline:
1960
+ return pos + 1, result + src[start_pos:pos]
1961
+ if src.startswith('"""', pos):
1962
+ return pos + 3, result + src[start_pos:pos]
1963
+ pos += 1
1964
+ continue
1965
+ if char == '\\':
1966
+ result += src[start_pos:pos]
1967
+ pos, parsed_escape = parse_escapes(src, pos)
1968
+ result += parsed_escape
1969
+ start_pos = pos
1970
+ continue
1971
+ if char in error_on:
1972
+ raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
1973
+ pos += 1
1854
1974
 
1855
- # Return
1856
- return self.Result(
1857
- pid=pid,
1858
- env_info=env_info,
1859
- )
1860
1975
 
1861
- def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
1862
- d = yield self.Read(sz)
1863
- if not isinstance(d, bytes):
1864
- raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
1865
- if len(d) != sz:
1866
- raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
1867
- return d
1976
+ def toml_parse_value( # noqa: C901
1977
+ src: str,
1978
+ pos: TomlPos,
1979
+ parse_float: TomlParseFloat,
1980
+ ) -> ta.Tuple[TomlPos, ta.Any]:
1981
+ try:
1982
+ char: ta.Optional[str] = src[pos]
1983
+ except IndexError:
1984
+ char = None
1868
1985
 
1869
- def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
1870
- d = yield from self._read(len(e))
1871
- if d != e:
1872
- raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
1986
+ # IMPORTANT: order conditions based on speed of checking and likelihood
1873
1987
 
1874
- def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
1875
- i = yield self.Write(d)
1876
- if i is not None:
1877
- raise self.ProtocolError('Unexpected input after write')
1988
+ # Basic strings
1989
+ if char == '"':
1990
+ if src.startswith('"""', pos):
1991
+ return toml_parse_multiline_str(src, pos, literal=False)
1992
+ return toml_parse_one_line_basic_str(src, pos)
1878
1993
 
1879
- #
1994
+ # Literal strings
1995
+ if char == "'":
1996
+ if src.startswith("'''", pos):
1997
+ return toml_parse_multiline_str(src, pos, literal=True)
1998
+ return toml_parse_literal_str(src, pos)
1880
1999
 
1881
- def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
1882
- gen = self.gen()
2000
+ # Booleans
2001
+ if char == 't':
2002
+ if src.startswith('true', pos):
2003
+ return pos + 4, True
2004
+ if char == 'f':
2005
+ if src.startswith('false', pos):
2006
+ return pos + 5, False
1883
2007
 
1884
- gi: ta.Optional[bytes] = None
1885
- while True:
1886
- try:
1887
- if gi is not None:
1888
- go = gen.send(gi)
1889
- else:
1890
- go = next(gen)
1891
- except StopIteration as e:
1892
- return e.value
2008
+ # Arrays
2009
+ if char == '[':
2010
+ return toml_parse_array(src, pos, parse_float)
1893
2011
 
1894
- if isinstance(go, self.Read):
1895
- if len(gi := input.read(go.sz)) != go.sz:
1896
- raise EOFError
1897
- elif isinstance(go, self.Write):
1898
- gi = None
1899
- output.write(go.d)
1900
- output.flush()
1901
- else:
1902
- raise TypeError(go)
2012
+ # Inline tables
2013
+ if char == '{':
2014
+ return toml_parse_inline_table(src, pos, parse_float)
1903
2015
 
1904
- async def async_run(
1905
- self,
1906
- input: ta.Any, # asyncio.StreamWriter # noqa
1907
- output: ta.Any, # asyncio.StreamReader
1908
- ) -> Result:
1909
- gen = self.gen()
2016
+ # Dates and times
2017
+ datetime_match = TOML_RE_DATETIME.match(src, pos)
2018
+ if datetime_match:
2019
+ try:
2020
+ datetime_obj = toml_match_to_datetime(datetime_match)
2021
+ except ValueError as e:
2022
+ raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
2023
+ return datetime_match.end(), datetime_obj
2024
+ localtime_match = TOML_RE_LOCALTIME.match(src, pos)
2025
+ if localtime_match:
2026
+ return localtime_match.end(), toml_match_to_localtime(localtime_match)
1910
2027
 
1911
- gi: ta.Optional[bytes] = None
1912
- while True:
1913
- try:
1914
- if gi is not None:
1915
- go = gen.send(gi)
1916
- else:
1917
- go = next(gen)
1918
- except StopIteration as e:
1919
- return e.value
2028
+ # Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
2029
+ # located after handling of dates and times.
2030
+ number_match = TOML_RE_NUMBER.match(src, pos)
2031
+ if number_match:
2032
+ return number_match.end(), toml_match_to_number(number_match, parse_float)
1920
2033
 
1921
- if isinstance(go, self.Read):
1922
- if len(gi := await input.read(go.sz)) != go.sz:
1923
- raise EOFError
1924
- elif isinstance(go, self.Write):
1925
- gi = None
1926
- output.write(go.d)
1927
- await output.drain()
1928
- else:
1929
- raise TypeError(go)
2034
+ # Special floats
2035
+ first_three = src[pos:pos + 3]
2036
+ if first_three in {'inf', 'nan'}:
2037
+ return pos + 3, parse_float(first_three)
2038
+ first_four = src[pos:pos + 4]
2039
+ if first_four in {'-inf', '+inf', '-nan', '+nan'}:
2040
+ return pos + 4, parse_float(first_four)
1930
2041
 
2042
+ raise toml_suffixed_err(src, pos, 'Invalid value')
1931
2043
 
1932
- ########################################
1933
- # ../../../omlish/asyncs/asyncio/channels.py
1934
2044
 
2045
+ def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
2046
+ """Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
1935
2047
 
1936
- class AsyncioBytesChannelTransport(asyncio.Transport):
1937
- def __init__(self, reader: asyncio.StreamReader) -> None:
1938
- super().__init__()
2048
+ def coord_repr(src: str, pos: TomlPos) -> str:
2049
+ if pos >= len(src):
2050
+ return 'end of document'
2051
+ line = src.count('\n', 0, pos) + 1
2052
+ if line == 1:
2053
+ column = pos + 1
2054
+ else:
2055
+ column = pos - src.rindex('\n', 0, pos)
2056
+ return f'line {line}, column {column}'
1939
2057
 
1940
- self.reader = reader
1941
- self.closed: asyncio.Future = asyncio.Future()
2058
+ return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
1942
2059
 
1943
- # @ta.override
1944
- def write(self, data: bytes) -> None:
1945
- self.reader.feed_data(data)
1946
2060
 
1947
- # @ta.override
1948
- def close(self) -> None:
1949
- self.reader.feed_eof()
1950
- if not self.closed.done():
1951
- self.closed.set_result(True)
2061
+ def toml_is_unicode_scalar_value(codepoint: int) -> bool:
2062
+ return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
1952
2063
 
1953
- # @ta.override
1954
- def is_closing(self) -> bool:
1955
- return self.closed.done()
1956
2064
 
2065
+ def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
2066
+ """A decorator to make `parse_float` safe.
1957
2067
 
1958
- def asyncio_create_bytes_channel(
1959
- loop: ta.Any = None,
1960
- ) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
1961
- if loop is None:
1962
- loop = asyncio.get_running_loop()
2068
+ `parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
2069
+ thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
2070
+ """
2071
+ # The default `float` callable never returns illegal types. Optimize it.
2072
+ if parse_float is float:
2073
+ return float
1963
2074
 
1964
- reader = asyncio.StreamReader()
1965
- protocol = asyncio.StreamReaderProtocol(reader)
1966
- transport = AsyncioBytesChannelTransport(reader)
1967
- writer = asyncio.StreamWriter(transport, protocol, reader, loop)
2075
+ def safe_parse_float(float_str: str) -> ta.Any:
2076
+ float_value = parse_float(float_str)
2077
+ if isinstance(float_value, (dict, list)):
2078
+ raise ValueError('parse_float must not return dicts or lists') # noqa
2079
+ return float_value
1968
2080
 
1969
- return reader, writer
2081
+ return safe_parse_float
1970
2082
 
1971
2083
 
1972
2084
  ########################################
1973
- # ../../../omlish/asyncs/asyncio/streams.py
2085
+ # ../../../omlish/formats/toml/writer.py
1974
2086
 
1975
2087
 
1976
- ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
2088
+ class TomlWriter:
2089
+ @dc.dataclass(frozen=True)
2090
+ class Literal:
2091
+ s: str
1977
2092
 
2093
+ def __init__(self, out: ta.TextIO) -> None:
2094
+ super().__init__()
2095
+ self._out = out
1978
2096
 
1979
- async def asyncio_open_stream_reader(
1980
- f: ta.IO,
1981
- loop: ta.Any = None,
1982
- *,
1983
- limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1984
- ) -> asyncio.StreamReader:
1985
- if loop is None:
1986
- loop = asyncio.get_running_loop()
2097
+ self._indent = 0
2098
+ self._wrote_indent = False
1987
2099
 
1988
- reader = asyncio.StreamReader(limit=limit, loop=loop)
1989
- await loop.connect_read_pipe(
1990
- lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1991
- f,
1992
- )
2100
+ #
1993
2101
 
1994
- return reader
2102
+ def _w(self, s: str) -> None:
2103
+ if not self._wrote_indent:
2104
+ self._out.write(' ' * self._indent)
2105
+ self._wrote_indent = True
2106
+ self._out.write(s)
1995
2107
 
2108
+ def _nl(self) -> None:
2109
+ self._out.write('\n')
2110
+ self._wrote_indent = False
1996
2111
 
1997
- async def asyncio_open_stream_writer(
1998
- f: ta.IO,
1999
- loop: ta.Any = None,
2000
- ) -> asyncio.StreamWriter:
2001
- if loop is None:
2002
- loop = asyncio.get_running_loop()
2112
+ def _needs_quote(self, s: str) -> bool:
2113
+ return (
2114
+ not s or
2115
+ any(c in s for c in '\'"\n') or
2116
+ s[0] not in string.ascii_letters
2117
+ )
2003
2118
 
2004
- writer_transport, writer_protocol = await loop.connect_write_pipe(
2005
- lambda: asyncio.streams.FlowControlMixin(loop=loop),
2006
- f,
2007
- )
2119
+ def _maybe_quote(self, s: str) -> str:
2120
+ if self._needs_quote(s):
2121
+ return repr(s)
2122
+ else:
2123
+ return s
2008
2124
 
2009
- return asyncio.streams.StreamWriter(
2010
- writer_transport,
2011
- writer_protocol,
2012
- None,
2013
- loop,
2014
- )
2125
+ #
2015
2126
 
2127
+ def write_root(self, obj: ta.Mapping) -> None:
2128
+ for i, (k, v) in enumerate(obj.items()):
2129
+ if i:
2130
+ self._nl()
2131
+ self._w('[')
2132
+ self._w(self._maybe_quote(k))
2133
+ self._w(']')
2134
+ self._nl()
2135
+ self.write_table_contents(v)
2136
+
2137
+ def write_table_contents(self, obj: ta.Mapping) -> None:
2138
+ for k, v in obj.items():
2139
+ self.write_key(k)
2140
+ self._w(' = ')
2141
+ self.write_value(v)
2142
+ self._nl()
2143
+
2144
+ def write_array(self, obj: ta.Sequence) -> None:
2145
+ self._w('[')
2146
+ self._nl()
2147
+ self._indent += 1
2148
+ for e in obj:
2149
+ self.write_value(e)
2150
+ self._w(',')
2151
+ self._nl()
2152
+ self._indent -= 1
2153
+ self._w(']')
2154
+
2155
+ def write_inline_table(self, obj: ta.Mapping) -> None:
2156
+ self._w('{')
2157
+ for i, (k, v) in enumerate(obj.items()):
2158
+ if i:
2159
+ self._w(', ')
2160
+ self.write_key(k)
2161
+ self._w(' = ')
2162
+ self.write_value(v)
2163
+ self._w('}')
2164
+
2165
+ def write_inline_array(self, obj: ta.Sequence) -> None:
2166
+ self._w('[')
2167
+ for i, e in enumerate(obj):
2168
+ if i:
2169
+ self._w(', ')
2170
+ self.write_value(e)
2171
+ self._w(']')
2172
+
2173
+ def write_key(self, obj: ta.Any) -> None:
2174
+ if isinstance(obj, TomlWriter.Literal):
2175
+ self._w(obj.s)
2176
+ elif isinstance(obj, str):
2177
+ self._w(self._maybe_quote(obj.replace('_', '-')))
2178
+ elif isinstance(obj, int):
2179
+ self._w(repr(str(obj)))
2180
+ else:
2181
+ raise TypeError(obj)
2016
2182
 
2017
- ########################################
2018
- # ../../../omlish/asyncs/asyncio/timeouts.py
2183
+ def write_value(self, obj: ta.Any) -> None:
2184
+ if isinstance(obj, bool):
2185
+ self._w(str(obj).lower())
2186
+ elif isinstance(obj, (str, int, float)):
2187
+ self._w(repr(obj))
2188
+ elif isinstance(obj, ta.Mapping):
2189
+ self.write_inline_table(obj)
2190
+ elif isinstance(obj, ta.Sequence):
2191
+ if not obj:
2192
+ self.write_inline_array(obj)
2193
+ else:
2194
+ self.write_array(obj)
2195
+ else:
2196
+ raise TypeError(obj)
2019
2197
 
2198
+ #
2020
2199
 
2021
- def asyncio_maybe_timeout(
2022
- fut: AwaitableT,
2023
- timeout: ta.Optional[float] = None,
2024
- ) -> AwaitableT:
2025
- if timeout is not None:
2026
- fut = asyncio.wait_for(fut, timeout) # type: ignore
2027
- return fut
2200
+ @classmethod
2201
+ def write_str(cls, obj: ta.Any) -> str:
2202
+ out = io.StringIO()
2203
+ cls(out).write_value(obj)
2204
+ return out.getvalue()
2028
2205
 
2029
2206
 
2030
2207
  ########################################
@@ -4672,131 +4849,360 @@ class ArgparseCli:
4672
4849
  elif k in objs:
4673
4850
  del [k]
4674
4851
 
4675
- #
4852
+ #
4853
+
4854
+ anns = ta.get_type_hints(_ArgparseCliAnnotationBox({
4855
+ **{k: v for bcls in reversed(mro) for k, v in getattr(bcls, '__annotations__', {}).items()},
4856
+ **ns.get('__annotations__', {}),
4857
+ }), globalns=ns.get('__globals__', {}))
4858
+
4859
+ #
4860
+
4861
+ if '_parser' in ns:
4862
+ parser = check.isinstance(ns['_parser'], argparse.ArgumentParser)
4863
+ else:
4864
+ parser = argparse.ArgumentParser()
4865
+ setattr(cls, '_parser', parser)
4866
+
4867
+ #
4868
+
4869
+ subparsers = parser.add_subparsers()
4870
+
4871
+ for att, obj in objs.items():
4872
+ if isinstance(obj, ArgparseCmd):
4873
+ if obj.parent is not None:
4874
+ raise NotImplementedError
4875
+
4876
+ for cn in [obj.name, *(obj.aliases or [])]:
4877
+ subparser = subparsers.add_parser(cn)
4878
+
4879
+ for arg in (obj.args or []):
4880
+ if (
4881
+ len(arg.args) == 1 and
4882
+ isinstance(arg.args[0], str) and
4883
+ not (n := check.isinstance(arg.args[0], str)).startswith('-') and
4884
+ 'metavar' not in arg.kwargs
4885
+ ):
4886
+ subparser.add_argument(
4887
+ n.replace('-', '_'),
4888
+ **arg.kwargs,
4889
+ metavar=n,
4890
+ )
4891
+ else:
4892
+ subparser.add_argument(*arg.args, **arg.kwargs)
4893
+
4894
+ subparser.set_defaults(_cmd=obj)
4895
+
4896
+ elif isinstance(obj, ArgparseArg):
4897
+ if att in anns:
4898
+ ann_kwargs = _get_argparse_arg_ann_kwargs(anns[att])
4899
+ obj.kwargs = {**ann_kwargs, **obj.kwargs}
4900
+
4901
+ if not obj.dest:
4902
+ if 'dest' in obj.kwargs:
4903
+ obj.dest = obj.kwargs['dest']
4904
+ else:
4905
+ obj.dest = obj.kwargs['dest'] = att # type: ignore
4906
+
4907
+ parser.add_argument(*obj.args, **obj.kwargs)
4908
+
4909
+ else:
4910
+ raise TypeError(obj)
4911
+
4912
+ #
4913
+
4914
+ _parser: ta.ClassVar[argparse.ArgumentParser]
4915
+
4916
+ @classmethod
4917
+ def get_parser(cls) -> argparse.ArgumentParser:
4918
+ return cls._parser
4919
+
4920
+ @property
4921
+ def argv(self) -> ta.Sequence[str]:
4922
+ return self._argv
4923
+
4924
+ @property
4925
+ def args(self) -> argparse.Namespace:
4926
+ return self._args
4927
+
4928
+ @property
4929
+ def unknown_args(self) -> ta.Sequence[str]:
4930
+ return self._unknown_args
4931
+
4932
+ #
4933
+
4934
+ def _bind_cli_cmd(self, cmd: ArgparseCmd) -> ta.Callable:
4935
+ return cmd.__get__(self, type(self))
4936
+
4937
+ def prepare_cli_run(self) -> ta.Optional[ta.Callable]:
4938
+ cmd = getattr(self.args, '_cmd', None)
4939
+
4940
+ if self._unknown_args and not (cmd is not None and cmd.accepts_unknown):
4941
+ msg = f'unrecognized arguments: {" ".join(self._unknown_args)}'
4942
+ if (parser := self.get_parser()).exit_on_error: # type: ignore
4943
+ parser.error(msg)
4944
+ else:
4945
+ raise argparse.ArgumentError(None, msg)
4946
+
4947
+ if cmd is None:
4948
+ self.get_parser().print_help()
4949
+ return None
4950
+
4951
+ return self._bind_cli_cmd(cmd)
4952
+
4953
+ #
4954
+
4955
+ def cli_run(self) -> ta.Optional[int]:
4956
+ if (fn := self.prepare_cli_run()) is None:
4957
+ return 0
4958
+
4959
+ return fn()
4960
+
4961
+ def cli_run_and_exit(self) -> ta.NoReturn:
4962
+ sys.exit(rc if isinstance(rc := self.cli_run(), int) else 0)
4963
+
4964
+ def __call__(self, *, exit: bool = False) -> ta.Optional[int]: # noqa
4965
+ if exit:
4966
+ return self.cli_run_and_exit()
4967
+ else:
4968
+ return self.cli_run()
4969
+
4970
+ #
4971
+
4972
+ async def async_cli_run(self) -> ta.Optional[int]:
4973
+ if (fn := self.prepare_cli_run()) is None:
4974
+ return 0
4975
+
4976
+ return await fn()
4977
+
4978
+
4979
+ ########################################
4980
+ # ../../../omlish/configs/formats.py
4981
+ """
4982
+ Notes:
4983
+ - necessarily string-oriented
4984
+ - single file, as this is intended to be amalg'd and thus all included anyway
4985
+
4986
+ TODO:
4987
+ - ConfigDataMapper? to_map -> ConfigMap?
4988
+ - nginx ?
4989
+ - raw ?
4990
+ """
4991
+
4992
+
4993
+ ##
4994
+
4995
+
4996
+ @dc.dataclass(frozen=True)
4997
+ class ConfigData(abc.ABC): # noqa
4998
+ @abc.abstractmethod
4999
+ def as_map(self) -> ConfigMap:
5000
+ raise NotImplementedError
5001
+
5002
+
5003
+ #
5004
+
5005
+
5006
+ class ConfigLoader(abc.ABC, ta.Generic[ConfigDataT]):
5007
+ @property
5008
+ def file_exts(self) -> ta.Sequence[str]:
5009
+ return ()
5010
+
5011
+ def match_file(self, n: str) -> bool:
5012
+ return '.' in n and n.split('.')[-1] in check.not_isinstance(self.file_exts, str)
5013
+
5014
+ #
5015
+
5016
+ def load_file(self, p: str) -> ConfigDataT:
5017
+ with open(p) as f:
5018
+ return self.load_str(f.read())
5019
+
5020
+ @abc.abstractmethod
5021
+ def load_str(self, s: str) -> ConfigDataT:
5022
+ raise NotImplementedError
5023
+
5024
+
5025
+ #
5026
+
5027
+
5028
+ class ConfigRenderer(abc.ABC, ta.Generic[ConfigDataT]):
5029
+ @property
5030
+ @abc.abstractmethod
5031
+ def data_cls(self) -> ta.Type[ConfigDataT]:
5032
+ raise NotImplementedError
5033
+
5034
+ def match_data(self, d: ConfigDataT) -> bool:
5035
+ return isinstance(d, self.data_cls)
5036
+
5037
+ #
5038
+
5039
+ @abc.abstractmethod
5040
+ def render(self, d: ConfigDataT) -> str:
5041
+ raise NotImplementedError
5042
+
5043
+
5044
+ ##
5045
+
5046
+
5047
+ @dc.dataclass(frozen=True)
5048
+ class ObjConfigData(ConfigData, abc.ABC):
5049
+ obj: ta.Any
5050
+
5051
+ def as_map(self) -> ConfigMap:
5052
+ return check.isinstance(self.obj, collections.abc.Mapping)
5053
+
5054
+
5055
+ ##
5056
+
5057
+
5058
+ @dc.dataclass(frozen=True)
5059
+ class JsonConfigData(ObjConfigData):
5060
+ pass
5061
+
5062
+
5063
+ class JsonConfigLoader(ConfigLoader[JsonConfigData]):
5064
+ file_exts = ('json',)
5065
+
5066
+ def load_str(self, s: str) -> JsonConfigData:
5067
+ return JsonConfigData(json.loads(s))
5068
+
5069
+
5070
+ class JsonConfigRenderer(ConfigRenderer[JsonConfigData]):
5071
+ data_cls = JsonConfigData
5072
+
5073
+ def render(self, d: JsonConfigData) -> str:
5074
+ return json_dumps_pretty(d.obj)
5075
+
5076
+
5077
+ ##
5078
+
5079
+
5080
+ @dc.dataclass(frozen=True)
5081
+ class TomlConfigData(ObjConfigData):
5082
+ pass
5083
+
5084
+
5085
+ class TomlConfigLoader(ConfigLoader[TomlConfigData]):
5086
+ file_exts = ('toml',)
5087
+
5088
+ def load_str(self, s: str) -> TomlConfigData:
5089
+ return TomlConfigData(toml_loads(s))
5090
+
5091
+
5092
+ class TomlConfigRenderer(ConfigRenderer[TomlConfigData]):
5093
+ data_cls = TomlConfigData
5094
+
5095
+ def render(self, d: TomlConfigData) -> str:
5096
+ return TomlWriter.write_str(d.obj)
5097
+
5098
+
5099
+ ##
5100
+
4676
5101
 
4677
- anns = ta.get_type_hints(_ArgparseCliAnnotationBox({
4678
- **{k: v for bcls in reversed(mro) for k, v in getattr(bcls, '__annotations__', {}).items()},
4679
- **ns.get('__annotations__', {}),
4680
- }), globalns=ns.get('__globals__', {}))
5102
+ @dc.dataclass(frozen=True)
5103
+ class YamlConfigData(ObjConfigData):
5104
+ pass
4681
5105
 
4682
- #
4683
5106
 
4684
- if '_parser' in ns:
4685
- parser = check.isinstance(ns['_parser'], argparse.ArgumentParser)
4686
- else:
4687
- parser = argparse.ArgumentParser()
4688
- setattr(cls, '_parser', parser)
5107
+ class YamlConfigLoader(ConfigLoader[YamlConfigData]):
5108
+ file_exts = ('yaml', 'yml')
4689
5109
 
4690
- #
5110
+ def load_str(self, s: str) -> YamlConfigData:
5111
+ return YamlConfigData(__import__('yaml').safe_load(s))
4691
5112
 
4692
- subparsers = parser.add_subparsers()
4693
5113
 
4694
- for att, obj in objs.items():
4695
- if isinstance(obj, ArgparseCmd):
4696
- if obj.parent is not None:
4697
- raise NotImplementedError
5114
+ class YamlConfigRenderer(ConfigRenderer[YamlConfigData]):
5115
+ data_cls = YamlConfigData
4698
5116
 
4699
- for cn in [obj.name, *(obj.aliases or [])]:
4700
- subparser = subparsers.add_parser(cn)
5117
+ def render(self, d: YamlConfigData) -> str:
5118
+ return __import__('yaml').safe_dump(d.obj)
4701
5119
 
4702
- for arg in (obj.args or []):
4703
- if (
4704
- len(arg.args) == 1 and
4705
- isinstance(arg.args[0], str) and
4706
- not (n := check.isinstance(arg.args[0], str)).startswith('-') and
4707
- 'metavar' not in arg.kwargs
4708
- ):
4709
- subparser.add_argument(
4710
- n.replace('-', '_'),
4711
- **arg.kwargs,
4712
- metavar=n,
4713
- )
4714
- else:
4715
- subparser.add_argument(*arg.args, **arg.kwargs)
4716
5120
 
4717
- subparser.set_defaults(_cmd=obj)
5121
+ ##
4718
5122
 
4719
- elif isinstance(obj, ArgparseArg):
4720
- if att in anns:
4721
- ann_kwargs = _get_argparse_arg_ann_kwargs(anns[att])
4722
- obj.kwargs = {**ann_kwargs, **obj.kwargs}
4723
5123
 
4724
- if not obj.dest:
4725
- if 'dest' in obj.kwargs:
4726
- obj.dest = obj.kwargs['dest']
4727
- else:
4728
- obj.dest = obj.kwargs['dest'] = att # type: ignore
5124
+ @dc.dataclass(frozen=True)
5125
+ class IniConfigData(ConfigData):
5126
+ sections: IniSectionSettingsMap
4729
5127
 
4730
- parser.add_argument(*obj.args, **obj.kwargs)
5128
+ def as_map(self) -> ConfigMap:
5129
+ return self.sections
4731
5130
 
4732
- else:
4733
- raise TypeError(obj)
4734
5131
 
4735
- #
5132
+ class IniConfigLoader(ConfigLoader[IniConfigData]):
5133
+ file_exts = ('ini',)
4736
5134
 
4737
- _parser: ta.ClassVar[argparse.ArgumentParser]
5135
+ def load_str(self, s: str) -> IniConfigData:
5136
+ cp = configparser.ConfigParser()
5137
+ cp.read_string(s)
5138
+ return IniConfigData(extract_ini_sections(cp))
4738
5139
 
4739
- @classmethod
4740
- def get_parser(cls) -> argparse.ArgumentParser:
4741
- return cls._parser
4742
5140
 
4743
- @property
4744
- def argv(self) -> ta.Sequence[str]:
4745
- return self._argv
5141
+ class IniConfigRenderer(ConfigRenderer[IniConfigData]):
5142
+ data_cls = IniConfigData
4746
5143
 
4747
- @property
4748
- def args(self) -> argparse.Namespace:
4749
- return self._args
5144
+ def render(self, d: IniConfigData) -> str:
5145
+ return render_ini_sections(d.sections)
4750
5146
 
4751
- @property
4752
- def unknown_args(self) -> ta.Sequence[str]:
4753
- return self._unknown_args
4754
5147
 
4755
- #
5148
+ ##
4756
5149
 
4757
- def _bind_cli_cmd(self, cmd: ArgparseCmd) -> ta.Callable:
4758
- return cmd.__get__(self, type(self))
4759
5150
 
4760
- def prepare_cli_run(self) -> ta.Optional[ta.Callable]:
4761
- cmd = getattr(self.args, '_cmd', None)
5151
+ @dc.dataclass(frozen=True)
5152
+ class SwitchedConfigFileLoader:
5153
+ loaders: ta.Sequence[ConfigLoader]
5154
+ default: ta.Optional[ConfigLoader] = None
4762
5155
 
4763
- if self._unknown_args and not (cmd is not None and cmd.accepts_unknown):
4764
- msg = f'unrecognized arguments: {" ".join(self._unknown_args)}'
4765
- if (parser := self.get_parser()).exit_on_error: # type: ignore
4766
- parser.error(msg)
4767
- else:
4768
- raise argparse.ArgumentError(None, msg)
5156
+ def load_file(self, p: str) -> ConfigData:
5157
+ n = os.path.basename(p)
4769
5158
 
4770
- if cmd is None:
4771
- self.get_parser().print_help()
4772
- return None
5159
+ for l in self.loaders:
5160
+ if l.match_file(n):
5161
+ return l.load_file(p)
4773
5162
 
4774
- return self._bind_cli_cmd(cmd)
5163
+ if (d := self.default) is not None:
5164
+ return d.load_file(p)
4775
5165
 
4776
- #
5166
+ raise NameError(n)
4777
5167
 
4778
- def cli_run(self) -> ta.Optional[int]:
4779
- if (fn := self.prepare_cli_run()) is None:
4780
- return 0
4781
5168
 
4782
- return fn()
5169
+ DEFAULT_CONFIG_LOADERS: ta.Sequence[ConfigLoader] = [
5170
+ JsonConfigLoader(),
5171
+ TomlConfigLoader(),
5172
+ YamlConfigLoader(),
5173
+ IniConfigLoader(),
5174
+ ]
4783
5175
 
4784
- def cli_run_and_exit(self) -> ta.NoReturn:
4785
- sys.exit(rc if isinstance(rc := self.cli_run(), int) else 0)
5176
+ DEFAULT_CONFIG_LOADER: ConfigLoader = JsonConfigLoader()
4786
5177
 
4787
- def __call__(self, *, exit: bool = False) -> ta.Optional[int]: # noqa
4788
- if exit:
4789
- return self.cli_run_and_exit()
4790
- else:
4791
- return self.cli_run()
5178
+ DEFAULT_CONFIG_FILE_LOADER = SwitchedConfigFileLoader(
5179
+ loaders=DEFAULT_CONFIG_LOADERS,
5180
+ default=DEFAULT_CONFIG_LOADER,
5181
+ )
4792
5182
 
4793
- #
4794
5183
 
4795
- async def async_cli_run(self) -> ta.Optional[int]:
4796
- if (fn := self.prepare_cli_run()) is None:
4797
- return 0
5184
+ ##
4798
5185
 
4799
- return await fn()
5186
+
5187
+ @dc.dataclass(frozen=True)
5188
+ class SwitchedConfigRenderer:
5189
+ renderers: ta.Sequence[ConfigRenderer]
5190
+
5191
+ def render(self, d: ConfigData) -> str:
5192
+ for r in self.renderers:
5193
+ if r.match_data(d):
5194
+ return r.render(d)
5195
+ raise TypeError(d)
5196
+
5197
+
5198
+ DEFAULT_CONFIG_RENDERERS: ta.Sequence[ConfigRenderer] = [
5199
+ JsonConfigRenderer(),
5200
+ TomlConfigRenderer(),
5201
+ YamlConfigRenderer(),
5202
+ IniConfigRenderer(),
5203
+ ]
5204
+
5205
+ DEFAULT_CONFIG_RENDERER = SwitchedConfigRenderer(DEFAULT_CONFIG_RENDERERS)
4800
5206
 
4801
5207
 
4802
5208
  ########################################
@@ -6895,121 +7301,6 @@ def bind_interp_uv() -> InjectorBindings:
6895
7301
  return inj.as_bindings(*lst)
6896
7302
 
6897
7303
 
6898
- ########################################
6899
- # ../../configs.py
6900
-
6901
-
6902
- ##
6903
-
6904
-
6905
- def parse_config_file(
6906
- name: str,
6907
- f: ta.TextIO,
6908
- ) -> ConfigMapping:
6909
- if name.endswith('.toml'):
6910
- return toml_loads(f.read())
6911
-
6912
- elif any(name.endswith(e) for e in ('.yml', '.yaml')):
6913
- yaml = __import__('yaml')
6914
- return yaml.safe_load(f)
6915
-
6916
- elif name.endswith('.ini'):
6917
- import configparser
6918
- cp = configparser.ConfigParser()
6919
- cp.read_file(f)
6920
- config_dct: ta.Dict[str, ta.Any] = {}
6921
- for sec in cp.sections():
6922
- cd = config_dct
6923
- for k in sec.split('.'):
6924
- cd = cd.setdefault(k, {})
6925
- cd.update(cp.items(sec))
6926
- return config_dct
6927
-
6928
- else:
6929
- return json.loads(f.read())
6930
-
6931
-
6932
- def read_config_file(
6933
- path: str,
6934
- cls: ta.Type[T],
6935
- *,
6936
- prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
6937
- msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
6938
- ) -> T:
6939
- with open(path) as cf:
6940
- config_dct = parse_config_file(os.path.basename(path), cf)
6941
-
6942
- if prepare is not None:
6943
- config_dct = prepare(config_dct)
6944
-
6945
- return msh.unmarshal_obj(config_dct, cls)
6946
-
6947
-
6948
- ##
6949
-
6950
-
6951
- def build_config_named_children(
6952
- o: ta.Union[
6953
- ta.Sequence[ConfigMapping],
6954
- ta.Mapping[str, ConfigMapping],
6955
- None,
6956
- ],
6957
- *,
6958
- name_key: str = 'name',
6959
- ) -> ta.Optional[ta.Sequence[ConfigMapping]]:
6960
- if o is None:
6961
- return None
6962
-
6963
- lst: ta.List[ConfigMapping] = []
6964
- if isinstance(o, ta.Mapping):
6965
- for k, v in o.items():
6966
- check.isinstance(v, ta.Mapping)
6967
- if name_key in v:
6968
- n = v[name_key]
6969
- if k != n:
6970
- raise KeyError(f'Given names do not match: {n} != {k}')
6971
- lst.append(v)
6972
- else:
6973
- lst.append({name_key: k, **v})
6974
-
6975
- else:
6976
- check.not_isinstance(o, str)
6977
- lst.extend(o)
6978
-
6979
- seen = set()
6980
- for d in lst:
6981
- n = d['name']
6982
- if n in d:
6983
- raise KeyError(f'Duplicate name: {n}')
6984
- seen.add(n)
6985
-
6986
- return lst
6987
-
6988
-
6989
- ##
6990
-
6991
-
6992
- def render_ini_config(
6993
- settings_by_section: IniConfigSectionSettingsMap,
6994
- ) -> str:
6995
- out = io.StringIO()
6996
-
6997
- for i, (section, settings) in enumerate(settings_by_section.items()):
6998
- if i:
6999
- out.write('\n')
7000
-
7001
- out.write(f'[{section}]\n')
7002
-
7003
- for k, v in settings.items():
7004
- if isinstance(v, str):
7005
- out.write(f'{k}={v}\n')
7006
- else:
7007
- for vv in v:
7008
- out.write(f'{k}={vv}\n')
7009
-
7010
- return out.getvalue()
7011
-
7012
-
7013
7304
  ########################################
7014
7305
  # ../commands/marshal.py
7015
7306
 
@@ -7060,7 +7351,105 @@ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
7060
7351
  # ../commands/types.py
7061
7352
 
7062
7353
 
7063
- CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
7354
+ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
7355
+
7356
+
7357
+ ########################################
7358
+ # ../deploy/conf/specs.py
7359
+
7360
+
7361
+ ##
7362
+
7363
+
7364
+ class DeployAppConfContent(abc.ABC): # noqa
7365
+ pass
7366
+
7367
+
7368
+ #
7369
+
7370
+
7371
+ @register_single_field_type_obj_marshaler('body')
7372
+ @dc.dataclass(frozen=True)
7373
+ class RawDeployAppConfContent(DeployAppConfContent):
7374
+ body: str
7375
+
7376
+
7377
+ #
7378
+
7379
+
7380
+ @register_single_field_type_obj_marshaler('obj')
7381
+ @dc.dataclass(frozen=True)
7382
+ class JsonDeployAppConfContent(DeployAppConfContent):
7383
+ obj: ta.Any
7384
+
7385
+
7386
+ #
7387
+
7388
+
7389
+ @register_single_field_type_obj_marshaler('sections')
7390
+ @dc.dataclass(frozen=True)
7391
+ class IniDeployAppConfContent(DeployAppConfContent):
7392
+ sections: IniSectionSettingsMap
7393
+
7394
+
7395
+ #
7396
+
7397
+
7398
+ @register_single_field_type_obj_marshaler('items')
7399
+ @dc.dataclass(frozen=True)
7400
+ class NginxDeployAppConfContent(DeployAppConfContent):
7401
+ items: ta.Any
7402
+
7403
+
7404
+ ##
7405
+
7406
+
7407
+ @dc.dataclass(frozen=True)
7408
+ class DeployAppConfFile:
7409
+ path: str
7410
+ content: DeployAppConfContent
7411
+
7412
+ def __post_init__(self) -> None:
7413
+ check_valid_deploy_spec_path(self.path)
7414
+
7415
+
7416
+ ##
7417
+
7418
+
7419
+ @dc.dataclass(frozen=True)
7420
+ class DeployAppConfLink: # noqa
7421
+ """
7422
+ May be either:
7423
+ - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
7424
+ - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
7425
+ - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
7426
+ """
7427
+
7428
+ src: str
7429
+
7430
+ kind: ta.Literal['current_only', 'all_active'] = 'current_only'
7431
+
7432
+ def __post_init__(self) -> None:
7433
+ check_valid_deploy_spec_path(self.src)
7434
+ if '/' in self.src:
7435
+ check.equal(self.src.count('/'), 1)
7436
+
7437
+
7438
+ ##
7439
+
7440
+
7441
+ @dc.dataclass(frozen=True)
7442
+ class DeployAppConfSpec:
7443
+ files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
7444
+
7445
+ links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
7446
+
7447
+ def __post_init__(self) -> None:
7448
+ if self.files:
7449
+ seen: ta.Set[str] = set()
7450
+ for f in self.files:
7451
+ check.not_in(f.path, seen)
7452
+ seen.add(f.path)
7064
7453
 
7065
7454
 
7066
7455
  ########################################
@@ -7388,6 +7777,106 @@ class SystemConfig:
7388
7777
  platform: ta.Optional[Platform] = None
7389
7778
 
7390
7779
 
7780
+ ########################################
7781
+ # ../../../omlish/configs/nginx.py
7782
+ """
7783
+ See:
7784
+ - https://nginx.org/en/docs/dev/development_guide.html
7785
+ - https://nginx.org/en/docs/dev/development_guide.html#config_directives
7786
+ - https://nginx.org/en/docs/example.html
7787
+ """
7788
+
7789
+
7790
+ @dc.dataclass()
7791
+ class NginxConfigItems:
7792
+ lst: ta.List['NginxConfigItem']
7793
+
7794
+ @classmethod
7795
+ def of(cls, obj: ta.Any) -> 'NginxConfigItems':
7796
+ if isinstance(obj, NginxConfigItems):
7797
+ return obj
7798
+ return cls([NginxConfigItem.of(e) for e in check.isinstance(obj, list)])
7799
+
7800
+
7801
+ @dc.dataclass()
7802
+ class NginxConfigItem:
7803
+ name: str
7804
+ args: ta.Optional[ta.List[str]] = None
7805
+ block: ta.Optional[NginxConfigItems] = None
7806
+
7807
+ @classmethod
7808
+ def of(cls, obj: ta.Any) -> 'NginxConfigItem':
7809
+ if isinstance(obj, NginxConfigItem):
7810
+ return obj
7811
+ args = check.isinstance(check.not_isinstance(obj, str), collections.abc.Sequence)
7812
+ name, args = check.isinstance(args[0], str), args[1:]
7813
+ if args and not isinstance(args[-1], str):
7814
+ block, args = NginxConfigItems.of(args[-1]), args[:-1]
7815
+ else:
7816
+ block = None
7817
+ return NginxConfigItem(name, [check.isinstance(e, str) for e in args], block=block)
7818
+
7819
+
7820
+ def render_nginx_config(wr: IndentWriter, obj: ta.Any) -> None:
7821
+ if isinstance(obj, NginxConfigItem):
7822
+ wr.write(obj.name)
7823
+ for e in obj.args or ():
7824
+ wr.write(' ')
7825
+ wr.write(e)
7826
+ if obj.block:
7827
+ wr.write(' {\n')
7828
+ with wr.indent():
7829
+ render_nginx_config(wr, obj.block)
7830
+ wr.write('}\n')
7831
+ else:
7832
+ wr.write(';\n')
7833
+
7834
+ elif isinstance(obj, NginxConfigItems):
7835
+ for e2 in obj.lst:
7836
+ render_nginx_config(wr, e2)
7837
+
7838
+ else:
7839
+ raise TypeError(obj)
7840
+
7841
+
7842
+ def render_nginx_config_str(obj: ta.Any) -> str:
7843
+ iw = IndentWriter()
7844
+ render_nginx_config(iw, obj)
7845
+ return iw.getvalue()
7846
+
7847
+
7848
+ ########################################
7849
+ # ../../../omlish/lite/configs.py
7850
+
7851
+
7852
+ ##
7853
+
7854
+
7855
+ def load_config_file_obj(
7856
+ f: str,
7857
+ cls: ta.Type[T],
7858
+ *,
7859
+ prepare: ta.Union[
7860
+ ta.Callable[[ConfigMap], ConfigMap],
7861
+ ta.Iterable[ta.Callable[[ConfigMap], ConfigMap]],
7862
+ ] = (),
7863
+ msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
7864
+ ) -> T:
7865
+ config_data = DEFAULT_CONFIG_FILE_LOADER.load_file(f)
7866
+
7867
+ config_dct = config_data.as_map()
7868
+
7869
+ if prepare is not None:
7870
+ if isinstance(prepare, ta.Iterable):
7871
+ pfs = list(prepare)
7872
+ else:
7873
+ pfs = [prepare]
7874
+ for pf in pfs:
7875
+ config_dct = pf(config_dct)
7876
+
7877
+ return msh.unmarshal_obj(config_dct, cls)
7878
+
7879
+
7391
7880
  ########################################
7392
7881
  # ../../../omlish/logs/standard.py
7393
7882
  """
@@ -7838,78 +8327,6 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
7838
8327
  return ret.decode().strip()
7839
8328
 
7840
8329
 
7841
- ########################################
7842
- # ../../../omserv/nginx/configs.py
7843
- """
7844
- TODO:
7845
- - omnibus/jmespath
7846
-
7847
- https://nginx.org/en/docs/dev/development_guide.html
7848
- https://nginx.org/en/docs/dev/development_guide.html#config_directives
7849
- https://nginx.org/en/docs/example.html
7850
-
7851
- https://github.com/yandex/gixy
7852
- """
7853
-
7854
-
7855
- @dc.dataclass()
7856
- class NginxConfigItems:
7857
- lst: ta.List['NginxConfigItem']
7858
-
7859
- @classmethod
7860
- def of(cls, obj: ta.Any) -> 'NginxConfigItems':
7861
- if isinstance(obj, NginxConfigItems):
7862
- return obj
7863
- return cls([NginxConfigItem.of(e) for e in check.isinstance(obj, list)])
7864
-
7865
-
7866
- @dc.dataclass()
7867
- class NginxConfigItem:
7868
- name: str
7869
- args: ta.Optional[ta.List[str]] = None
7870
- block: ta.Optional[NginxConfigItems] = None
7871
-
7872
- @classmethod
7873
- def of(cls, obj: ta.Any) -> 'NginxConfigItem':
7874
- if isinstance(obj, NginxConfigItem):
7875
- return obj
7876
- args = check.isinstance(check.not_isinstance(obj, str), collections.abc.Sequence)
7877
- name, args = check.isinstance(args[0], str), args[1:]
7878
- if args and not isinstance(args[-1], str):
7879
- block, args = NginxConfigItems.of(args[-1]), args[:-1]
7880
- else:
7881
- block = None
7882
- return NginxConfigItem(name, [check.isinstance(e, str) for e in args], block=block)
7883
-
7884
-
7885
- def render_nginx_config(wr: IndentWriter, obj: ta.Any) -> None:
7886
- if isinstance(obj, NginxConfigItem):
7887
- wr.write(obj.name)
7888
- for e in obj.args or ():
7889
- wr.write(' ')
7890
- wr.write(e)
7891
- if obj.block:
7892
- wr.write(' {\n')
7893
- with wr.indent():
7894
- render_nginx_config(wr, obj.block)
7895
- wr.write('}\n')
7896
- else:
7897
- wr.write(';\n')
7898
-
7899
- elif isinstance(obj, NginxConfigItems):
7900
- for e2 in obj.lst:
7901
- render_nginx_config(wr, e2)
7902
-
7903
- else:
7904
- raise TypeError(obj)
7905
-
7906
-
7907
- def render_nginx_config_str(obj: ta.Any) -> str:
7908
- iw = IndentWriter()
7909
- render_nginx_config(iw, obj)
7910
- return iw.getvalue()
7911
-
7912
-
7913
8330
  ########################################
7914
8331
  # ../../../omdev/interp/providers/base.py
7915
8332
  """
@@ -7977,113 +8394,15 @@ class LocalCommandExecutor(CommandExecutor):
7977
8394
  def __init__(
7978
8395
  self,
7979
8396
  *,
7980
- command_executors: CommandExecutorMap,
7981
- ) -> None:
7982
- super().__init__()
7983
-
7984
- self._command_executors = command_executors
7985
-
7986
- async def execute(self, cmd: Command) -> Command.Output:
7987
- ce: CommandExecutor = self._command_executors[type(cmd)]
7988
- return await ce.execute(cmd)
7989
-
7990
-
7991
- ########################################
7992
- # ../deploy/conf/specs.py
7993
-
7994
-
7995
- ##
7996
-
7997
-
7998
- class DeployAppConfContent(abc.ABC): # noqa
7999
- pass
8000
-
8001
-
8002
- #
8003
-
8004
-
8005
- @register_single_field_type_obj_marshaler('body')
8006
- @dc.dataclass(frozen=True)
8007
- class RawDeployAppConfContent(DeployAppConfContent):
8008
- body: str
8009
-
8010
-
8011
- #
8012
-
8013
-
8014
- @register_single_field_type_obj_marshaler('obj')
8015
- @dc.dataclass(frozen=True)
8016
- class JsonDeployAppConfContent(DeployAppConfContent):
8017
- obj: ta.Any
8018
-
8019
-
8020
- #
8021
-
8022
-
8023
- @register_single_field_type_obj_marshaler('sections')
8024
- @dc.dataclass(frozen=True)
8025
- class IniDeployAppConfContent(DeployAppConfContent):
8026
- sections: IniConfigSectionSettingsMap
8027
-
8028
-
8029
- #
8030
-
8031
-
8032
- @register_single_field_type_obj_marshaler('items')
8033
- @dc.dataclass(frozen=True)
8034
- class NginxDeployAppConfContent(DeployAppConfContent):
8035
- items: ta.Any
8036
-
8037
-
8038
- ##
8039
-
8040
-
8041
- @dc.dataclass(frozen=True)
8042
- class DeployAppConfFile:
8043
- path: str
8044
- content: DeployAppConfContent
8045
-
8046
- def __post_init__(self) -> None:
8047
- check_valid_deploy_spec_path(self.path)
8048
-
8049
-
8050
- ##
8051
-
8052
-
8053
- @dc.dataclass(frozen=True)
8054
- class DeployAppConfLink: # noqa
8055
- """
8056
- May be either:
8057
- - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
8058
- - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
8059
- - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
8060
- """
8061
-
8062
- src: str
8063
-
8064
- kind: ta.Literal['current_only', 'all_active'] = 'current_only'
8065
-
8066
- def __post_init__(self) -> None:
8067
- check_valid_deploy_spec_path(self.src)
8068
- if '/' in self.src:
8069
- check.equal(self.src.count('/'), 1)
8070
-
8071
-
8072
- ##
8073
-
8074
-
8075
- @dc.dataclass(frozen=True)
8076
- class DeployAppConfSpec:
8077
- files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
8397
+ command_executors: CommandExecutorMap,
8398
+ ) -> None:
8399
+ super().__init__()
8078
8400
 
8079
- links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
8401
+ self._command_executors = command_executors
8080
8402
 
8081
- def __post_init__(self) -> None:
8082
- if self.files:
8083
- seen: ta.Set[str] = set()
8084
- for f in self.files:
8085
- check.not_in(f.path, seen)
8086
- seen.add(f.path)
8403
+ async def execute(self, cmd: Command) -> Command.Output:
8404
+ ce: CommandExecutor = self._command_executors[type(cmd)]
8405
+ return await ce.execute(cmd)
8087
8406
 
8088
8407
 
8089
8408
  ########################################
@@ -8288,6 +8607,127 @@ class DeployPath(DeployPathRenderable):
8288
8607
  return cls(tuple(DeployPathPart.parse(p) for p in ps))
8289
8608
 
8290
8609
 
8610
+ ########################################
8611
+ # ../deploy/specs.py
8612
+
8613
+
8614
+ ##
8615
+
8616
+
8617
+ class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
8618
+ @cached_nullary
8619
+ def _key_str(self) -> str:
8620
+ return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
8621
+
8622
+ @abc.abstractmethod
8623
+ def key(self) -> KeyDeployTagT:
8624
+ raise NotImplementedError
8625
+
8626
+
8627
+ ##
8628
+
8629
+
8630
+ @dc.dataclass(frozen=True)
8631
+ class DeployGitRepo:
8632
+ host: ta.Optional[str] = None
8633
+ username: ta.Optional[str] = None
8634
+ path: ta.Optional[str] = None
8635
+
8636
+ def __post_init__(self) -> None:
8637
+ check.not_in('..', check.non_empty_str(self.host))
8638
+ check.not_in('.', check.non_empty_str(self.path))
8639
+
8640
+
8641
+ @dc.dataclass(frozen=True)
8642
+ class DeployGitSpec:
8643
+ repo: DeployGitRepo
8644
+ rev: DeployRev
8645
+
8646
+ subtrees: ta.Optional[ta.Sequence[str]] = None
8647
+
8648
+ def __post_init__(self) -> None:
8649
+ check.non_empty_str(self.rev)
8650
+ if self.subtrees is not None:
8651
+ for st in self.subtrees:
8652
+ check.non_empty_str(st)
8653
+
8654
+
8655
+ ##
8656
+
8657
+
8658
+ @dc.dataclass(frozen=True)
8659
+ class DeployVenvSpec:
8660
+ interp: ta.Optional[str] = None
8661
+
8662
+ requirements_files: ta.Optional[ta.Sequence[str]] = None
8663
+ extra_dependencies: ta.Optional[ta.Sequence[str]] = None
8664
+
8665
+ use_uv: bool = False
8666
+
8667
+
8668
+ ##
8669
+
8670
+
8671
+ @dc.dataclass(frozen=True)
8672
+ class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
8673
+ app: DeployApp
8674
+
8675
+ git: DeployGitSpec
8676
+
8677
+ venv: ta.Optional[DeployVenvSpec] = None
8678
+
8679
+ conf: ta.Optional[DeployAppConfSpec] = None
8680
+
8681
+ # @ta.override
8682
+ def key(self) -> DeployAppKey:
8683
+ return DeployAppKey(self._key_str())
8684
+
8685
+
8686
+ @dc.dataclass(frozen=True)
8687
+ class DeployAppLinksSpec:
8688
+ apps: ta.Sequence[DeployApp] = ()
8689
+
8690
+ removed_apps: ta.Sequence[DeployApp] = ()
8691
+
8692
+ exclude_unspecified: bool = False
8693
+
8694
+
8695
+ ##
8696
+
8697
+
8698
+ @dc.dataclass(frozen=True)
8699
+ class DeploySystemdSpec:
8700
+ # ~/.config/systemd/user/
8701
+ unit_dir: ta.Optional[str] = None
8702
+
8703
+
8704
+ ##
8705
+
8706
+
8707
+ @dc.dataclass(frozen=True)
8708
+ class DeploySpec(DeploySpecKeyed[DeployKey]):
8709
+ home: DeployHome
8710
+
8711
+ apps: ta.Sequence[DeployAppSpec] = ()
8712
+
8713
+ app_links: DeployAppLinksSpec = DeployAppLinksSpec()
8714
+
8715
+ systemd: ta.Optional[DeploySystemdSpec] = None
8716
+
8717
+ def __post_init__(self) -> None:
8718
+ check.non_empty_str(self.home)
8719
+
8720
+ seen: ta.Set[DeployApp] = set()
8721
+ for a in self.apps:
8722
+ if a.app in seen:
8723
+ raise KeyError(a.app)
8724
+ seen.add(a.app)
8725
+
8726
+ # @ta.override
8727
+ def key(self) -> DeployKey:
8728
+ return DeployKey(self._key_str())
8729
+
8730
+
8291
8731
  ########################################
8292
8732
  # ../remote/execution.py
8293
8733
  """
@@ -9320,7 +9760,7 @@ class DeployConfManager:
9320
9760
 
9321
9761
  elif isinstance(ac, IniDeployAppConfContent):
9322
9762
  ini_sections = pcc(ac.sections)
9323
- return strip_with_newline(render_ini_config(ini_sections))
9763
+ return strip_with_newline(render_ini_sections(ini_sections))
9324
9764
 
9325
9765
  elif isinstance(ac, NginxDeployAppConfContent):
9326
9766
  nginx_items = NginxConfigItems.of(pcc(ac.items))
@@ -9534,127 +9974,6 @@ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
9534
9974
  return self._owned_deploy_paths
9535
9975
 
9536
9976
 
9537
- ########################################
9538
- # ../deploy/specs.py
9539
-
9540
-
9541
- ##
9542
-
9543
-
9544
- class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
9545
- @cached_nullary
9546
- def _key_str(self) -> str:
9547
- return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
9548
-
9549
- @abc.abstractmethod
9550
- def key(self) -> KeyDeployTagT:
9551
- raise NotImplementedError
9552
-
9553
-
9554
- ##
9555
-
9556
-
9557
- @dc.dataclass(frozen=True)
9558
- class DeployGitRepo:
9559
- host: ta.Optional[str] = None
9560
- username: ta.Optional[str] = None
9561
- path: ta.Optional[str] = None
9562
-
9563
- def __post_init__(self) -> None:
9564
- check.not_in('..', check.non_empty_str(self.host))
9565
- check.not_in('.', check.non_empty_str(self.path))
9566
-
9567
-
9568
- @dc.dataclass(frozen=True)
9569
- class DeployGitSpec:
9570
- repo: DeployGitRepo
9571
- rev: DeployRev
9572
-
9573
- subtrees: ta.Optional[ta.Sequence[str]] = None
9574
-
9575
- def __post_init__(self) -> None:
9576
- check.non_empty_str(self.rev)
9577
- if self.subtrees is not None:
9578
- for st in self.subtrees:
9579
- check.non_empty_str(st)
9580
-
9581
-
9582
- ##
9583
-
9584
-
9585
- @dc.dataclass(frozen=True)
9586
- class DeployVenvSpec:
9587
- interp: ta.Optional[str] = None
9588
-
9589
- requirements_files: ta.Optional[ta.Sequence[str]] = None
9590
- extra_dependencies: ta.Optional[ta.Sequence[str]] = None
9591
-
9592
- use_uv: bool = False
9593
-
9594
-
9595
- ##
9596
-
9597
-
9598
- @dc.dataclass(frozen=True)
9599
- class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
9600
- app: DeployApp
9601
-
9602
- git: DeployGitSpec
9603
-
9604
- venv: ta.Optional[DeployVenvSpec] = None
9605
-
9606
- conf: ta.Optional[DeployAppConfSpec] = None
9607
-
9608
- # @ta.override
9609
- def key(self) -> DeployAppKey:
9610
- return DeployAppKey(self._key_str())
9611
-
9612
-
9613
- @dc.dataclass(frozen=True)
9614
- class DeployAppLinksSpec:
9615
- apps: ta.Sequence[DeployApp] = ()
9616
-
9617
- removed_apps: ta.Sequence[DeployApp] = ()
9618
-
9619
- exclude_unspecified: bool = False
9620
-
9621
-
9622
- ##
9623
-
9624
-
9625
- @dc.dataclass(frozen=True)
9626
- class DeploySystemdSpec:
9627
- # ~/.config/systemd/user/
9628
- unit_dir: ta.Optional[str] = None
9629
-
9630
-
9631
- ##
9632
-
9633
-
9634
- @dc.dataclass(frozen=True)
9635
- class DeploySpec(DeploySpecKeyed[DeployKey]):
9636
- home: DeployHome
9637
-
9638
- apps: ta.Sequence[DeployAppSpec] = ()
9639
-
9640
- app_links: DeployAppLinksSpec = DeployAppLinksSpec()
9641
-
9642
- systemd: ta.Optional[DeploySystemdSpec] = None
9643
-
9644
- def __post_init__(self) -> None:
9645
- check.non_empty_str(self.home)
9646
-
9647
- seen: ta.Set[DeployApp] = set()
9648
- for a in self.apps:
9649
- if a.app in seen:
9650
- raise KeyError(a.app)
9651
- seen.add(a.app)
9652
-
9653
- # @ta.override
9654
- def key(self) -> DeployKey:
9655
- return DeployKey(self._key_str())
9656
-
9657
-
9658
9977
  ########################################
9659
9978
  # ../remote/_main.py
9660
9979
 
@@ -12294,7 +12613,7 @@ class MainCli(ArgparseCli):
12294
12613
  if cf is None:
12295
12614
  return ManageConfig()
12296
12615
  else:
12297
- return read_config_file(cf, ManageConfig)
12616
+ return load_config_file_obj(cf, ManageConfig)
12298
12617
 
12299
12618
  #
12300
12619
 
@@ -12365,7 +12684,7 @@ class MainCli(ArgparseCli):
12365
12684
  cmds.append(cmd)
12366
12685
 
12367
12686
  for cf in self.args.command_file or []:
12368
- cmd = read_config_file(cf, Command, msh=msh)
12687
+ cmd = load_config_file_obj(cf, Command, msh=msh)
12369
12688
  cmds.append(cmd)
12370
12689
 
12371
12690
  #