ominfra 0.0.0.dev192__py3-none-any.whl → 0.0.0.dev194__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ominfra/scripts/manage.py CHANGED
@@ -16,6 +16,7 @@ import asyncio.subprocess
16
16
  import base64
17
17
  import collections
18
18
  import collections.abc
19
+ import configparser
19
20
  import contextlib
20
21
  import contextvars
21
22
  import ctypes as ct
@@ -73,17 +74,23 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
73
74
  VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType] # noqa
74
75
  VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
75
76
 
76
- # ../../omdev/toml/parser.py
77
- TomlParseFloat = ta.Callable[[str], ta.Any]
78
- TomlKey = ta.Tuple[str, ...]
79
- TomlPos = int # ta.TypeAlias
80
-
81
77
  # deploy/paths/types.py
82
78
  DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
83
79
 
84
80
  # ../../omlish/asyncs/asyncio/timeouts.py
85
81
  AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
86
82
 
83
+ # ../../omlish/configs/types.py
84
+ ConfigMap = ta.Mapping[str, ta.Any]
85
+
86
+ # ../../omlish/formats/ini/sections.py
87
+ IniSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
88
+
89
+ # ../../omlish/formats/toml/parser.py
90
+ TomlParseFloat = ta.Callable[[str], ta.Any]
91
+ TomlKey = ta.Tuple[str, ...]
92
+ TomlPos = int # ta.TypeAlias
93
+
87
94
  # ../../omlish/lite/cached.py
88
95
  T = ta.TypeVar('T')
89
96
  CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
@@ -113,6 +120,9 @@ CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
113
120
  # ../../omlish/argparse/cli.py
114
121
  ArgparseCmdFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
115
122
 
123
+ # ../../omlish/configs/formats.py
124
+ ConfigDataT = ta.TypeVar('ConfigDataT', bound='ConfigData')
125
+
116
126
  # ../../omlish/lite/contextmanagers.py
117
127
  ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
118
128
 
@@ -127,10 +137,6 @@ InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
127
137
  AtomicPathSwapKind = ta.Literal['dir', 'file']
128
138
  AtomicPathSwapState = ta.Literal['open', 'committed', 'aborted'] # ta.TypeAlias
129
139
 
130
- # ../configs.py
131
- ConfigMapping = ta.Mapping[str, ta.Any]
132
- IniConfigSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
133
-
134
140
  # ../../omlish/subprocesses.py
135
141
  SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
136
142
 
@@ -549,1482 +555,1653 @@ def canonicalize_version(
549
555
 
550
556
 
551
557
  ########################################
552
- # ../../../omdev/toml/parser.py
553
- # SPDX-License-Identifier: MIT
554
- # SPDX-FileCopyrightText: 2021 Taneli Hukkinen
555
- # Licensed to PSF under a Contributor Agreement.
556
- #
557
- # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
558
- # --------------------------------------------
559
- #
560
- # 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
561
- # ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
562
- # documentation.
563
- #
564
- # 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
565
- # royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
566
- # works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
567
- # Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
568
- # 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
569
- # Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
570
- #
571
- # 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
572
- # wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
573
- # any such work a brief summary of the changes made to Python.
574
- #
575
- # 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
576
- # EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
577
- # OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
578
- # RIGHTS.
579
- #
580
- # 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
581
- # DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
582
- # ADVISED OF THE POSSIBILITY THEREOF.
583
- #
584
- # 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
585
- #
586
- # 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
587
- # venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
588
- # name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
589
- #
590
- # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
591
- # License Agreement.
592
- #
593
- # https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
558
+ # ../config.py
594
559
 
595
560
 
596
- ##
561
+ @dc.dataclass(frozen=True)
562
+ class MainConfig:
563
+ log_level: ta.Optional[str] = 'INFO'
597
564
 
565
+ debug: bool = False
598
566
 
599
- _TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
600
567
 
601
- TOML_RE_NUMBER = re.compile(
602
- r"""
603
- 0
604
- (?:
605
- x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
606
- |
607
- b[01](?:_?[01])* # bin
608
- |
609
- o[0-7](?:_?[0-7])* # oct
610
- )
611
- |
612
- [+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
613
- (?P<floatpart>
614
- (?:\.[0-9](?:_?[0-9])*)? # optional fractional part
615
- (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
616
- )
617
- """,
618
- flags=re.VERBOSE,
619
- )
620
- TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
621
- TOML_RE_DATETIME = re.compile(
622
- rf"""
623
- ([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
624
- (?:
625
- [Tt ]
626
- {_TOML_TIME_RE_STR}
627
- (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
628
- )?
629
- """,
630
- flags=re.VERBOSE,
631
- )
568
+ ########################################
569
+ # ../deploy/config.py
632
570
 
633
571
 
634
- def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
635
- """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
572
+ ##
636
573
 
637
- Raises ValueError if the match does not correspond to a valid date or datetime.
638
- """
639
- (
640
- year_str,
641
- month_str,
642
- day_str,
643
- hour_str,
644
- minute_str,
645
- sec_str,
646
- micros_str,
647
- zulu_time,
648
- offset_sign_str,
649
- offset_hour_str,
650
- offset_minute_str,
651
- ) = match.groups()
652
- year, month, day = int(year_str), int(month_str), int(day_str)
653
- if hour_str is None:
654
- return datetime.date(year, month, day)
655
- hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
656
- micros = int(micros_str.ljust(6, '0')) if micros_str else 0
657
- if offset_sign_str:
658
- tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
659
- offset_hour_str, offset_minute_str, offset_sign_str,
660
- )
661
- elif zulu_time:
662
- tz = datetime.UTC
663
- else: # local date-time
664
- tz = None
665
- return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
666
574
 
575
+ @dc.dataclass(frozen=True)
576
+ class DeployConfig:
577
+ pass
667
578
 
668
- @functools.lru_cache() # noqa
669
- def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
670
- sign = 1 if sign_str == '+' else -1
671
- return datetime.timezone(
672
- datetime.timedelta(
673
- hours=sign * int(hour_str),
674
- minutes=sign * int(minute_str),
675
- ),
676
- )
677
579
 
580
+ ########################################
581
+ # ../deploy/paths/types.py
678
582
 
679
- def toml_match_to_localtime(match: re.Match) -> datetime.time:
680
- hour_str, minute_str, sec_str, micros_str = match.groups()
681
- micros = int(micros_str.ljust(6, '0')) if micros_str else 0
682
- return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
683
583
 
584
+ ##
684
585
 
685
- def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
686
- if match.group('floatpart'):
687
- return parse_float(match.group())
688
- return int(match.group(), 0)
689
586
 
587
+ ########################################
588
+ # ../deploy/types.py
690
589
 
691
- TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
692
590
 
693
- # Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
694
- # functions.
695
- TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
696
- TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
591
+ ##
697
592
 
698
- TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
699
- TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
700
593
 
701
- TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
594
+ DeployHome = ta.NewType('DeployHome', str)
702
595
 
703
- TOML_WS = frozenset(' \t')
704
- TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
705
- TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
706
- TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
707
- TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
596
+ DeployRev = ta.NewType('DeployRev', str)
708
597
 
709
- TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
710
- {
711
- '\\b': '\u0008', # backspace
712
- '\\t': '\u0009', # tab
713
- '\\n': '\u000A', # linefeed
714
- '\\f': '\u000C', # form feed
715
- '\\r': '\u000D', # carriage return
716
- '\\"': '\u0022', # quote
717
- '\\\\': '\u005C', # backslash
718
- },
719
- )
720
598
 
599
+ ########################################
600
+ # ../../pyremote.py
601
+ """
602
+ Basically this: https://mitogen.networkgenomics.com/howitworks.html
721
603
 
722
- class TomlDecodeError(ValueError):
723
- """An error raised if a document is not valid TOML."""
604
+ TODO:
605
+ - log: ta.Optional[logging.Logger] = None + log.debug's
606
+ """
724
607
 
725
608
 
726
- def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
727
- """Parse TOML from a binary file object."""
728
- b = fp.read()
729
- try:
730
- s = b.decode()
731
- except AttributeError:
732
- raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
733
- return toml_loads(s, parse_float=parse_float)
609
+ ##
734
610
 
735
611
 
736
- def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
737
- """Parse TOML from a string."""
612
+ @dc.dataclass(frozen=True)
613
+ class PyremoteBootstrapOptions:
614
+ debug: bool = False
738
615
 
739
- # The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
740
- try:
741
- src = s.replace('\r\n', '\n')
742
- except (AttributeError, TypeError):
743
- raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
744
- pos = 0
745
- out = TomlOutput(TomlNestedDict(), TomlFlags())
746
- header: TomlKey = ()
747
- parse_float = toml_make_safe_parse_float(parse_float)
616
+ DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
617
+ main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
748
618
 
749
- # Parse one statement at a time (typically means one line in TOML source)
750
- while True:
751
- # 1. Skip line leading whitespace
752
- pos = toml_skip_chars(src, pos, TOML_WS)
753
619
 
754
- # 2. Parse rules. Expect one of the following:
755
- # - end of file
756
- # - end of line
757
- # - comment
758
- # - key/value pair
759
- # - append dict to list (and move to its namespace)
760
- # - create dict (and move to its namespace)
761
- # Skip trailing whitespace when applicable.
762
- try:
763
- char = src[pos]
764
- except IndexError:
765
- break
766
- if char == '\n':
767
- pos += 1
768
- continue
769
- if char in TOML_KEY_INITIAL_CHARS:
770
- pos = toml_key_value_rule(src, pos, out, header, parse_float)
771
- pos = toml_skip_chars(src, pos, TOML_WS)
772
- elif char == '[':
773
- try:
774
- second_char: ta.Optional[str] = src[pos + 1]
775
- except IndexError:
776
- second_char = None
777
- out.flags.finalize_pending()
778
- if second_char == '[':
779
- pos, header = toml_create_list_rule(src, pos, out)
780
- else:
781
- pos, header = toml_create_dict_rule(src, pos, out)
782
- pos = toml_skip_chars(src, pos, TOML_WS)
783
- elif char != '#':
784
- raise toml_suffixed_err(src, pos, 'Invalid statement')
620
+ ##
785
621
 
786
- # 3. Skip comment
787
- pos = toml_skip_comment(src, pos)
788
622
 
789
- # 4. Expect end of line or end of file
790
- try:
791
- char = src[pos]
792
- except IndexError:
793
- break
794
- if char != '\n':
795
- raise toml_suffixed_err(
796
- src, pos, 'Expected newline or end of document after a statement',
797
- )
798
- pos += 1
623
+ @dc.dataclass(frozen=True)
624
+ class PyremoteEnvInfo:
625
+ sys_base_prefix: str
626
+ sys_byteorder: str
627
+ sys_defaultencoding: str
628
+ sys_exec_prefix: str
629
+ sys_executable: str
630
+ sys_implementation_name: str
631
+ sys_path: ta.List[str]
632
+ sys_platform: str
633
+ sys_prefix: str
634
+ sys_version: str
635
+ sys_version_info: ta.List[ta.Union[int, str]]
799
636
 
800
- return out.data.dict
637
+ platform_architecture: ta.List[str]
638
+ platform_machine: str
639
+ platform_platform: str
640
+ platform_processor: str
641
+ platform_system: str
642
+ platform_release: str
643
+ platform_version: str
801
644
 
645
+ site_userbase: str
802
646
 
803
- class TomlFlags:
804
- """Flags that map to parsed keys/namespaces."""
647
+ os_cwd: str
648
+ os_gid: int
649
+ os_loadavg: ta.List[float]
650
+ os_login: ta.Optional[str]
651
+ os_pgrp: int
652
+ os_pid: int
653
+ os_ppid: int
654
+ os_uid: int
805
655
 
806
- # Marks an immutable namespace (inline array or inline table).
807
- FROZEN = 0
808
- # Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
809
- EXPLICIT_NEST = 1
656
+ pw_name: str
657
+ pw_uid: int
658
+ pw_gid: int
659
+ pw_gecos: str
660
+ pw_dir: str
661
+ pw_shell: str
810
662
 
811
- def __init__(self) -> None:
812
- self._flags: ta.Dict[str, dict] = {}
813
- self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
663
+ env_path: ta.Optional[str]
814
664
 
815
- def add_pending(self, key: TomlKey, flag: int) -> None:
816
- self._pending_flags.add((key, flag))
817
665
 
818
- def finalize_pending(self) -> None:
819
- for key, flag in self._pending_flags:
820
- self.set(key, flag, recursive=False)
821
- self._pending_flags.clear()
666
+ def _get_pyremote_env_info() -> PyremoteEnvInfo:
667
+ os_uid = os.getuid()
822
668
 
823
- def unset_all(self, key: TomlKey) -> None:
824
- cont = self._flags
825
- for k in key[:-1]:
826
- if k not in cont:
827
- return
828
- cont = cont[k]['nested']
829
- cont.pop(key[-1], None)
669
+ pw = pwd.getpwuid(os_uid)
830
670
 
831
- def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
832
- cont = self._flags
833
- key_parent, key_stem = key[:-1], key[-1]
834
- for k in key_parent:
835
- if k not in cont:
836
- cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
837
- cont = cont[k]['nested']
838
- if key_stem not in cont:
839
- cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
840
- cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
671
+ os_login: ta.Optional[str]
672
+ try:
673
+ os_login = os.getlogin()
674
+ except OSError:
675
+ os_login = None
841
676
 
842
- def is_(self, key: TomlKey, flag: int) -> bool:
843
- if not key:
844
- return False # document root has no flags
845
- cont = self._flags
846
- for k in key[:-1]:
847
- if k not in cont:
848
- return False
849
- inner_cont = cont[k]
850
- if flag in inner_cont['recursive_flags']:
851
- return True
852
- cont = inner_cont['nested']
853
- key_stem = key[-1]
854
- if key_stem in cont:
855
- cont = cont[key_stem]
856
- return flag in cont['flags'] or flag in cont['recursive_flags']
857
- return False
677
+ return PyremoteEnvInfo(
678
+ sys_base_prefix=sys.base_prefix,
679
+ sys_byteorder=sys.byteorder,
680
+ sys_defaultencoding=sys.getdefaultencoding(),
681
+ sys_exec_prefix=sys.exec_prefix,
682
+ sys_executable=sys.executable,
683
+ sys_implementation_name=sys.implementation.name,
684
+ sys_path=sys.path,
685
+ sys_platform=sys.platform,
686
+ sys_prefix=sys.prefix,
687
+ sys_version=sys.version,
688
+ sys_version_info=list(sys.version_info),
858
689
 
690
+ platform_architecture=list(platform.architecture()),
691
+ platform_machine=platform.machine(),
692
+ platform_platform=platform.platform(),
693
+ platform_processor=platform.processor(),
694
+ platform_system=platform.system(),
695
+ platform_release=platform.release(),
696
+ platform_version=platform.version(),
859
697
 
860
- class TomlNestedDict:
861
- def __init__(self) -> None:
862
- # The parsed content of the TOML document
863
- self.dict: ta.Dict[str, ta.Any] = {}
698
+ site_userbase=site.getuserbase(),
864
699
 
865
- def get_or_create_nest(
866
- self,
867
- key: TomlKey,
868
- *,
869
- access_lists: bool = True,
870
- ) -> dict:
871
- cont: ta.Any = self.dict
872
- for k in key:
873
- if k not in cont:
874
- cont[k] = {}
875
- cont = cont[k]
876
- if access_lists and isinstance(cont, list):
877
- cont = cont[-1]
878
- if not isinstance(cont, dict):
879
- raise KeyError('There is no nest behind this key')
880
- return cont
700
+ os_cwd=os.getcwd(),
701
+ os_gid=os.getgid(),
702
+ os_loadavg=list(os.getloadavg()),
703
+ os_login=os_login,
704
+ os_pgrp=os.getpgrp(),
705
+ os_pid=os.getpid(),
706
+ os_ppid=os.getppid(),
707
+ os_uid=os_uid,
881
708
 
882
- def append_nest_to_list(self, key: TomlKey) -> None:
883
- cont = self.get_or_create_nest(key[:-1])
884
- last_key = key[-1]
885
- if last_key in cont:
886
- list_ = cont[last_key]
887
- if not isinstance(list_, list):
888
- raise KeyError('An object other than list found behind this key')
889
- list_.append({})
890
- else:
891
- cont[last_key] = [{}]
709
+ pw_name=pw.pw_name,
710
+ pw_uid=pw.pw_uid,
711
+ pw_gid=pw.pw_gid,
712
+ pw_gecos=pw.pw_gecos,
713
+ pw_dir=pw.pw_dir,
714
+ pw_shell=pw.pw_shell,
892
715
 
716
+ env_path=os.environ.get('PATH'),
717
+ )
893
718
 
894
- class TomlOutput(ta.NamedTuple):
895
- data: TomlNestedDict
896
- flags: TomlFlags
897
719
 
720
+ ##
898
721
 
899
- def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
900
- try:
901
- while src[pos] in chars:
902
- pos += 1
903
- except IndexError:
904
- pass
905
- return pos
906
722
 
723
+ _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
724
+ _PYREMOTE_BOOTSTRAP_SRC_FD = 101
907
725
 
908
- def toml_skip_until(
909
- src: str,
910
- pos: TomlPos,
911
- expect: str,
912
- *,
913
- error_on: ta.FrozenSet[str],
914
- error_on_eof: bool,
915
- ) -> TomlPos:
916
- try:
917
- new_pos = src.index(expect, pos)
918
- except ValueError:
919
- new_pos = len(src)
920
- if error_on_eof:
921
- raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
726
+ _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
727
+ _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
728
+ _PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
729
+ _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
730
+ _PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
922
731
 
923
- if not error_on.isdisjoint(src[pos:new_pos]):
924
- while src[pos] not in error_on:
925
- pos += 1
926
- raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
927
- return new_pos
732
+ _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
733
+ _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
734
+ _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
735
+ _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
928
736
 
737
+ _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
929
738
 
930
- def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
931
- try:
932
- char: ta.Optional[str] = src[pos]
933
- except IndexError:
934
- char = None
935
- if char == '#':
936
- return toml_skip_until(
937
- src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
938
- )
939
- return pos
739
+ _PYREMOTE_BOOTSTRAP_IMPORTS = [
740
+ 'base64',
741
+ 'os',
742
+ 'struct',
743
+ 'sys',
744
+ 'zlib',
745
+ ]
940
746
 
941
747
 
942
- def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
943
- while True:
944
- pos_before_skip = pos
945
- pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
946
- pos = toml_skip_comment(src, pos)
947
- if pos == pos_before_skip:
948
- return pos
748
+ def _pyremote_bootstrap_main(context_name: str) -> None:
749
+ # Get pid
750
+ pid = os.getpid()
949
751
 
752
+ # Two copies of payload src to be sent to parent
753
+ r0, w0 = os.pipe()
754
+ r1, w1 = os.pipe()
950
755
 
951
- def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
952
- pos += 1 # Skip "["
953
- pos = toml_skip_chars(src, pos, TOML_WS)
954
- pos, key = toml_parse_key(src, pos)
756
+ if (cp := os.fork()):
757
+ # Parent process
955
758
 
956
- if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
957
- raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
958
- out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
959
- try:
960
- out.data.get_or_create_nest(key)
961
- except KeyError:
962
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
759
+ # Dup original stdin to comm_fd for use as comm channel
760
+ os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
963
761
 
964
- if not src.startswith(']', pos):
965
- raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
966
- return pos + 1, key
762
+ # Overwrite stdin (fed to python repl) with first copy of src
763
+ os.dup2(r0, 0)
967
764
 
765
+ # Dup second copy of src to src_fd to recover after launch
766
+ os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
968
767
 
969
- def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
970
- pos += 2 # Skip "[["
971
- pos = toml_skip_chars(src, pos, TOML_WS)
972
- pos, key = toml_parse_key(src, pos)
768
+ # Close remaining fd's
769
+ for f in [r0, w0, r1, w1]:
770
+ os.close(f)
973
771
 
974
- if out.flags.is_(key, TomlFlags.FROZEN):
975
- raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
976
- # Free the namespace now that it points to another empty list item...
977
- out.flags.unset_all(key)
978
- # ...but this key precisely is still prohibited from table declaration
979
- out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
980
- try:
981
- out.data.append_nest_to_list(key)
982
- except KeyError:
983
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
772
+ # Save vars
773
+ env = os.environ
774
+ exe = sys.executable
775
+ env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
776
+ env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
777
+ env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
984
778
 
985
- if not src.startswith(']]', pos):
986
- raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
987
- return pos + 2, key
779
+ # Start repl reading stdin from r0
780
+ os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
988
781
 
782
+ else:
783
+ # Child process
989
784
 
990
- def toml_key_value_rule(
991
- src: str,
992
- pos: TomlPos,
993
- out: TomlOutput,
994
- header: TomlKey,
995
- parse_float: TomlParseFloat,
996
- ) -> TomlPos:
997
- pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
998
- key_parent, key_stem = key[:-1], key[-1]
999
- abs_key_parent = header + key_parent
785
+ # Write first ack
786
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
1000
787
 
1001
- relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
1002
- for cont_key in relative_path_cont_keys:
1003
- # Check that dotted key syntax does not redefine an existing table
1004
- if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
1005
- raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
1006
- # Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
1007
- # table sections.
1008
- out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
788
+ # Write pid
789
+ os.write(1, struct.pack('<Q', pid))
1009
790
 
1010
- if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
1011
- raise toml_suffixed_err(
1012
- src,
1013
- pos,
1014
- f'Cannot mutate immutable namespace {abs_key_parent}',
1015
- )
791
+ # Read payload src from stdin
792
+ payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
793
+ if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
794
+ raise EOFError
795
+ payload_src = zlib.decompress(payload_z)
1016
796
 
1017
- try:
1018
- nest = out.data.get_or_create_nest(abs_key_parent)
1019
- except KeyError:
1020
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1021
- if key_stem in nest:
1022
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
1023
- # Mark inline table and array namespaces recursively immutable
1024
- if isinstance(value, (dict, list)):
1025
- out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
1026
- nest[key_stem] = value
1027
- return pos
797
+ # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
798
+ # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
799
+ for w in [w0, w1]:
800
+ fp = os.fdopen(w, 'wb', 0)
801
+ fp.write(payload_src)
802
+ fp.close()
1028
803
 
804
+ # Write second ack
805
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
1029
806
 
1030
- def toml_parse_key_value_pair(
1031
- src: str,
1032
- pos: TomlPos,
1033
- parse_float: TomlParseFloat,
1034
- ) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
1035
- pos, key = toml_parse_key(src, pos)
1036
- try:
1037
- char: ta.Optional[str] = src[pos]
1038
- except IndexError:
1039
- char = None
1040
- if char != '=':
1041
- raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
1042
- pos += 1
1043
- pos = toml_skip_chars(src, pos, TOML_WS)
1044
- pos, value = toml_parse_value(src, pos, parse_float)
1045
- return pos, key, value
807
+ # Exit child
808
+ sys.exit(0)
1046
809
 
1047
810
 
1048
- def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
1049
- pos, key_part = toml_parse_key_part(src, pos)
1050
- key: TomlKey = (key_part,)
1051
- pos = toml_skip_chars(src, pos, TOML_WS)
1052
- while True:
1053
- try:
1054
- char: ta.Optional[str] = src[pos]
1055
- except IndexError:
1056
- char = None
1057
- if char != '.':
1058
- return pos, key
1059
- pos += 1
1060
- pos = toml_skip_chars(src, pos, TOML_WS)
1061
- pos, key_part = toml_parse_key_part(src, pos)
1062
- key += (key_part,)
1063
- pos = toml_skip_chars(src, pos, TOML_WS)
811
+ ##
1064
812
 
1065
813
 
1066
- def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1067
- try:
1068
- char: ta.Optional[str] = src[pos]
1069
- except IndexError:
1070
- char = None
1071
- if char in TOML_BARE_KEY_CHARS:
1072
- start_pos = pos
1073
- pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
1074
- return pos, src[start_pos:pos]
1075
- if char == "'":
1076
- return toml_parse_literal_str(src, pos)
1077
- if char == '"':
1078
- return toml_parse_one_line_basic_str(src, pos)
1079
- raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
814
+ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
815
+ if any(c in context_name for c in '\'"'):
816
+ raise NameError(context_name)
1080
817
 
818
+ import inspect
819
+ import textwrap
820
+ bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
1081
821
 
1082
- def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1083
- pos += 1
1084
- return toml_parse_basic_str(src, pos, multiline=False)
822
+ for gl in [
823
+ '_PYREMOTE_BOOTSTRAP_INPUT_FD',
824
+ '_PYREMOTE_BOOTSTRAP_SRC_FD',
1085
825
 
826
+ '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
827
+ '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
828
+ '_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
1086
829
 
1087
- def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
1088
- pos += 1
1089
- array: list = []
830
+ '_PYREMOTE_BOOTSTRAP_ACK0',
831
+ '_PYREMOTE_BOOTSTRAP_ACK1',
1090
832
 
1091
- pos = toml_skip_comments_and_array_ws(src, pos)
1092
- if src.startswith(']', pos):
1093
- return pos + 1, array
1094
- while True:
1095
- pos, val = toml_parse_value(src, pos, parse_float)
1096
- array.append(val)
1097
- pos = toml_skip_comments_and_array_ws(src, pos)
833
+ '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
834
+ ]:
835
+ bs_src = bs_src.replace(gl, repr(globals()[gl]))
1098
836
 
1099
- c = src[pos:pos + 1]
1100
- if c == ']':
1101
- return pos + 1, array
1102
- if c != ',':
1103
- raise toml_suffixed_err(src, pos, 'Unclosed array')
1104
- pos += 1
837
+ bs_src = '\n'.join(
838
+ cl
839
+ for l in bs_src.splitlines()
840
+ if (cl := (l.split('#')[0]).rstrip())
841
+ if cl.strip()
842
+ )
1105
843
 
1106
- pos = toml_skip_comments_and_array_ws(src, pos)
1107
- if src.startswith(']', pos):
1108
- return pos + 1, array
844
+ bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
845
+ bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
846
+ if b'"' in bs_z85:
847
+ raise ValueError(bs_z85)
1109
848
 
849
+ stmts = [
850
+ f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
851
+ f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
852
+ f'_pyremote_bootstrap_main("{context_name}")',
853
+ ]
1110
854
 
1111
- def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
1112
- pos += 1
1113
- nested_dict = TomlNestedDict()
1114
- flags = TomlFlags()
855
+ cmd = '; '.join(stmts)
856
+ return cmd
1115
857
 
1116
- pos = toml_skip_chars(src, pos, TOML_WS)
1117
- if src.startswith('}', pos):
1118
- return pos + 1, nested_dict.dict
1119
- while True:
1120
- pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1121
- key_parent, key_stem = key[:-1], key[-1]
1122
- if flags.is_(key, TomlFlags.FROZEN):
1123
- raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1124
- try:
1125
- nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
1126
- except KeyError:
1127
- raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1128
- if key_stem in nest:
1129
- raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
1130
- nest[key_stem] = value
1131
- pos = toml_skip_chars(src, pos, TOML_WS)
1132
- c = src[pos:pos + 1]
1133
- if c == '}':
1134
- return pos + 1, nested_dict.dict
1135
- if c != ',':
1136
- raise toml_suffixed_err(src, pos, 'Unclosed inline table')
1137
- if isinstance(value, (dict, list)):
1138
- flags.set(key, TomlFlags.FROZEN, recursive=True)
1139
- pos += 1
1140
- pos = toml_skip_chars(src, pos, TOML_WS)
1141
858
 
859
+ ##
1142
860
 
1143
- def toml_parse_basic_str_escape(
1144
- src: str,
1145
- pos: TomlPos,
1146
- *,
1147
- multiline: bool = False,
1148
- ) -> ta.Tuple[TomlPos, str]:
1149
- escape_id = src[pos:pos + 2]
1150
- pos += 2
1151
- if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
1152
- # Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
1153
- # newline.
1154
- if escape_id != '\\\n':
1155
- pos = toml_skip_chars(src, pos, TOML_WS)
1156
- try:
1157
- char = src[pos]
1158
- except IndexError:
1159
- return pos, ''
1160
- if char != '\n':
1161
- raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
1162
- pos += 1
1163
- pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1164
- return pos, ''
1165
- if escape_id == '\\u':
1166
- return toml_parse_hex_char(src, pos, 4)
1167
- if escape_id == '\\U':
1168
- return toml_parse_hex_char(src, pos, 8)
1169
- try:
1170
- return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
1171
- except KeyError:
1172
- raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
1173
861
 
862
+ @dc.dataclass(frozen=True)
863
+ class PyremotePayloadRuntime:
864
+ input: ta.BinaryIO
865
+ output: ta.BinaryIO
866
+ context_name: str
867
+ payload_src: str
868
+ options: PyremoteBootstrapOptions
869
+ env_info: PyremoteEnvInfo
1174
870
 
1175
- def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1176
- return toml_parse_basic_str_escape(src, pos, multiline=True)
1177
871
 
872
+ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
873
+ # If src file var is not present we need to do initial finalization
874
+ if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
875
+ # Read second copy of payload src
876
+ r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
877
+ payload_src = r1.read().decode('utf-8')
878
+ r1.close()
1178
879
 
1179
- def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
1180
- hex_str = src[pos:pos + hex_len]
1181
- if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
1182
- raise toml_suffixed_err(src, pos, 'Invalid hex value')
1183
- pos += hex_len
1184
- hex_int = int(hex_str, 16)
1185
- if not toml_is_unicode_scalar_value(hex_int):
1186
- raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
1187
- return pos, chr(hex_int)
880
+ # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
881
+ # a pipe at once.
882
+ os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
1188
883
 
884
+ # Read options
885
+ options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
886
+ if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
887
+ raise EOFError
888
+ options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
1189
889
 
1190
- def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1191
- pos += 1 # Skip starting apostrophe
1192
- start_pos = pos
1193
- pos = toml_skip_until(
1194
- src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
1195
- )
1196
- return pos + 1, src[start_pos:pos] # Skip ending apostrophe
890
+ # If debugging, re-exec as file
891
+ if options.debug:
892
+ # Write temp source file
893
+ import tempfile
894
+ tfd, tfn = tempfile.mkstemp('-pyremote.py')
895
+ os.write(tfd, payload_src.encode('utf-8'))
896
+ os.close(tfd)
1197
897
 
898
+ # Set vars
899
+ os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
900
+ os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
1198
901
 
1199
- def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
1200
- pos += 3
1201
- if src.startswith('\n', pos):
1202
- pos += 1
902
+ # Re-exec temp file
903
+ exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
904
+ context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
905
+ os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
1203
906
 
1204
- if literal:
1205
- delim = "'"
1206
- end_pos = toml_skip_until(
1207
- src,
1208
- pos,
1209
- "'''",
1210
- error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
1211
- error_on_eof=True,
1212
- )
1213
- result = src[pos:end_pos]
1214
- pos = end_pos + 3
1215
907
  else:
1216
- delim = '"'
1217
- pos, result = toml_parse_basic_str(src, pos, multiline=True)
908
+ # Load options json var
909
+ options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
910
+ options = PyremoteBootstrapOptions(**json.loads(options_json_str))
1218
911
 
1219
- # Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
1220
- if not src.startswith(delim, pos):
1221
- return pos, result
1222
- pos += 1
1223
- if not src.startswith(delim, pos):
1224
- return pos, result + delim
1225
- pos += 1
1226
- return pos, result + (delim * 2)
912
+ # Read temp source file
913
+ with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
914
+ payload_src = sf.read()
1227
915
 
916
+ # Restore vars
917
+ sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
918
+ context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
1228
919
 
1229
- def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
1230
- if multiline:
1231
- error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1232
- parse_escapes = toml_parse_basic_str_escape_multiline
1233
- else:
1234
- error_on = TOML_ILLEGAL_BASIC_STR_CHARS
1235
- parse_escapes = toml_parse_basic_str_escape
1236
- result = ''
1237
- start_pos = pos
1238
- while True:
1239
- try:
1240
- char = src[pos]
1241
- except IndexError:
1242
- raise toml_suffixed_err(src, pos, 'Unterminated string') from None
1243
- if char == '"':
1244
- if not multiline:
1245
- return pos + 1, result + src[start_pos:pos]
1246
- if src.startswith('"""', pos):
1247
- return pos + 3, result + src[start_pos:pos]
1248
- pos += 1
1249
- continue
1250
- if char == '\\':
1251
- result += src[start_pos:pos]
1252
- pos, parsed_escape = parse_escapes(src, pos)
1253
- result += parsed_escape
1254
- start_pos = pos
1255
- continue
1256
- if char in error_on:
1257
- raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
1258
- pos += 1
920
+ # Write third ack
921
+ os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
1259
922
 
923
+ # Write env info
924
+ env_info = _get_pyremote_env_info()
925
+ env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
926
+ os.write(1, struct.pack('<I', len(env_info_json)))
927
+ os.write(1, env_info_json.encode('utf-8'))
1260
928
 
1261
- def toml_parse_value( # noqa: C901
1262
- src: str,
1263
- pos: TomlPos,
1264
- parse_float: TomlParseFloat,
1265
- ) -> ta.Tuple[TomlPos, ta.Any]:
1266
- try:
1267
- char: ta.Optional[str] = src[pos]
1268
- except IndexError:
1269
- char = None
929
+ # Setup IO
930
+ input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
931
+ output = os.fdopen(os.dup(1), 'wb', 0) # noqa
932
+ os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
933
+ os.close(nfd)
1270
934
 
1271
- # IMPORTANT: order conditions based on speed of checking and likelihood
935
+ if (mn := options.main_name_override) is not None:
936
+ # Inspections like typing.get_type_hints need an entry in sys.modules.
937
+ sys.modules[mn] = sys.modules['__main__']
1272
938
 
1273
- # Basic strings
1274
- if char == '"':
1275
- if src.startswith('"""', pos):
1276
- return toml_parse_multiline_str(src, pos, literal=False)
1277
- return toml_parse_one_line_basic_str(src, pos)
939
+ # Write fourth ack
940
+ output.write(_PYREMOTE_BOOTSTRAP_ACK3)
1278
941
 
1279
- # Literal strings
1280
- if char == "'":
1281
- if src.startswith("'''", pos):
1282
- return toml_parse_multiline_str(src, pos, literal=True)
1283
- return toml_parse_literal_str(src, pos)
942
+ # Return
943
+ return PyremotePayloadRuntime(
944
+ input=input,
945
+ output=output,
946
+ context_name=context_name,
947
+ payload_src=payload_src,
948
+ options=options,
949
+ env_info=env_info,
950
+ )
1284
951
 
1285
- # Booleans
1286
- if char == 't':
1287
- if src.startswith('true', pos):
1288
- return pos + 4, True
1289
- if char == 'f':
1290
- if src.startswith('false', pos):
1291
- return pos + 5, False
1292
952
 
1293
- # Arrays
1294
- if char == '[':
1295
- return toml_parse_array(src, pos, parse_float)
953
+ ##
1296
954
 
1297
- # Inline tables
1298
- if char == '{':
1299
- return toml_parse_inline_table(src, pos, parse_float)
1300
955
 
1301
- # Dates and times
1302
- datetime_match = TOML_RE_DATETIME.match(src, pos)
1303
- if datetime_match:
1304
- try:
1305
- datetime_obj = toml_match_to_datetime(datetime_match)
1306
- except ValueError as e:
1307
- raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
1308
- return datetime_match.end(), datetime_obj
1309
- localtime_match = TOML_RE_LOCALTIME.match(src, pos)
1310
- if localtime_match:
1311
- return localtime_match.end(), toml_match_to_localtime(localtime_match)
956
+ class PyremoteBootstrapDriver:
957
+ def __init__(
958
+ self,
959
+ payload_src: ta.Union[str, ta.Sequence[str]],
960
+ options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
961
+ ) -> None:
962
+ super().__init__()
1312
963
 
1313
- # Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
1314
- # located after handling of dates and times.
1315
- number_match = TOML_RE_NUMBER.match(src, pos)
1316
- if number_match:
1317
- return number_match.end(), toml_match_to_number(number_match, parse_float)
964
+ self._payload_src = payload_src
965
+ self._options = options
1318
966
 
1319
- # Special floats
1320
- first_three = src[pos:pos + 3]
1321
- if first_three in {'inf', 'nan'}:
1322
- return pos + 3, parse_float(first_three)
1323
- first_four = src[pos:pos + 4]
1324
- if first_four in {'-inf', '+inf', '-nan', '+nan'}:
1325
- return pos + 4, parse_float(first_four)
967
+ self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
968
+ self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
1326
969
 
1327
- raise toml_suffixed_err(src, pos, 'Invalid value')
970
+ self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
971
+ #
1328
972
 
973
+ @classmethod
974
+ def _prepare_payload_src(
975
+ cls,
976
+ payload_src: ta.Union[str, ta.Sequence[str]],
977
+ options: PyremoteBootstrapOptions,
978
+ ) -> str:
979
+ parts: ta.List[str]
980
+ if isinstance(payload_src, str):
981
+ parts = [payload_src]
982
+ else:
983
+ parts = list(payload_src)
1329
984
 
1330
- def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
1331
- """Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
985
+ if (mn := options.main_name_override) is not None:
986
+ parts.insert(0, f'__name__ = {mn!r}')
1332
987
 
1333
- def coord_repr(src: str, pos: TomlPos) -> str:
1334
- if pos >= len(src):
1335
- return 'end of document'
1336
- line = src.count('\n', 0, pos) + 1
1337
- if line == 1:
1338
- column = pos + 1
988
+ if len(parts) == 1:
989
+ return parts[0]
1339
990
  else:
1340
- column = pos - src.rindex('\n', 0, pos)
1341
- return f'line {line}, column {column}'
991
+ return '\n\n'.join(parts)
1342
992
 
1343
- return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
993
+ #
1344
994
 
995
+ @dc.dataclass(frozen=True)
996
+ class Read:
997
+ sz: int
1345
998
 
1346
- def toml_is_unicode_scalar_value(codepoint: int) -> bool:
1347
- return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
999
+ @dc.dataclass(frozen=True)
1000
+ class Write:
1001
+ d: bytes
1348
1002
 
1003
+ class ProtocolError(Exception):
1004
+ pass
1349
1005
 
1350
- def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
1351
- """A decorator to make `parse_float` safe.
1006
+ @dc.dataclass(frozen=True)
1007
+ class Result:
1008
+ pid: int
1009
+ env_info: PyremoteEnvInfo
1352
1010
 
1353
- `parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
1354
- thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
1355
- """
1356
- # The default `float` callable never returns illegal types. Optimize it.
1357
- if parse_float is float:
1358
- return float
1011
+ def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
1012
+ # Read first ack (after fork)
1013
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
1359
1014
 
1360
- def safe_parse_float(float_str: str) -> ta.Any:
1361
- float_value = parse_float(float_str)
1362
- if isinstance(float_value, (dict, list)):
1363
- raise ValueError('parse_float must not return dicts or lists') # noqa
1364
- return float_value
1015
+ # Read pid
1016
+ d = yield from self._read(8)
1017
+ pid = struct.unpack('<Q', d)[0]
1365
1018
 
1366
- return safe_parse_float
1019
+ # Write payload src
1020
+ yield from self._write(struct.pack('<I', len(self._payload_z)))
1021
+ yield from self._write(self._payload_z)
1367
1022
 
1023
+ # Read second ack (after writing src copies)
1024
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
1368
1025
 
1369
- ########################################
1370
- # ../config.py
1026
+ # Write options
1027
+ yield from self._write(struct.pack('<I', len(self._options_json)))
1028
+ yield from self._write(self._options_json)
1371
1029
 
1030
+ # Read third ack (after reaping child process)
1031
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
1372
1032
 
1373
- @dc.dataclass(frozen=True)
1374
- class MainConfig:
1375
- log_level: ta.Optional[str] = 'INFO'
1033
+ # Read env info
1034
+ d = yield from self._read(4)
1035
+ env_info_json_len = struct.unpack('<I', d)[0]
1036
+ d = yield from self._read(env_info_json_len)
1037
+ env_info_json = d.decode('utf-8')
1038
+ env_info = PyremoteEnvInfo(**json.loads(env_info_json))
1376
1039
 
1377
- debug: bool = False
1040
+ # Read fourth ack (after finalization completed)
1041
+ yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
1378
1042
 
1043
+ # Return
1044
+ return self.Result(
1045
+ pid=pid,
1046
+ env_info=env_info,
1047
+ )
1379
1048
 
1380
- ########################################
1381
- # ../deploy/config.py
1049
+ def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
1050
+ d = yield self.Read(sz)
1051
+ if not isinstance(d, bytes):
1052
+ raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
1053
+ if len(d) != sz:
1054
+ raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
1055
+ return d
1382
1056
 
1057
+ def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
1058
+ d = yield from self._read(len(e))
1059
+ if d != e:
1060
+ raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
1383
1061
 
1384
- ##
1062
+ def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
1063
+ i = yield self.Write(d)
1064
+ if i is not None:
1065
+ raise self.ProtocolError('Unexpected input after write')
1385
1066
 
1067
+ #
1386
1068
 
1387
- @dc.dataclass(frozen=True)
1388
- class DeployConfig:
1389
- pass
1069
+ def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
1070
+ gen = self.gen()
1390
1071
 
1072
+ gi: ta.Optional[bytes] = None
1073
+ while True:
1074
+ try:
1075
+ if gi is not None:
1076
+ go = gen.send(gi)
1077
+ else:
1078
+ go = next(gen)
1079
+ except StopIteration as e:
1080
+ return e.value
1391
1081
 
1392
- ########################################
1393
- # ../deploy/paths/types.py
1082
+ if isinstance(go, self.Read):
1083
+ if len(gi := input.read(go.sz)) != go.sz:
1084
+ raise EOFError
1085
+ elif isinstance(go, self.Write):
1086
+ gi = None
1087
+ output.write(go.d)
1088
+ output.flush()
1089
+ else:
1090
+ raise TypeError(go)
1394
1091
 
1092
+ async def async_run(
1093
+ self,
1094
+ input: ta.Any, # asyncio.StreamWriter # noqa
1095
+ output: ta.Any, # asyncio.StreamReader
1096
+ ) -> Result:
1097
+ gen = self.gen()
1395
1098
 
1396
- ##
1099
+ gi: ta.Optional[bytes] = None
1100
+ while True:
1101
+ try:
1102
+ if gi is not None:
1103
+ go = gen.send(gi)
1104
+ else:
1105
+ go = next(gen)
1106
+ except StopIteration as e:
1107
+ return e.value
1108
+
1109
+ if isinstance(go, self.Read):
1110
+ if len(gi := await input.read(go.sz)) != go.sz:
1111
+ raise EOFError
1112
+ elif isinstance(go, self.Write):
1113
+ gi = None
1114
+ output.write(go.d)
1115
+ await output.drain()
1116
+ else:
1117
+ raise TypeError(go)
1397
1118
 
1398
1119
 
1399
1120
  ########################################
1400
- # ../deploy/types.py
1121
+ # ../../../omlish/asyncs/asyncio/channels.py
1401
1122
 
1402
1123
 
1403
- ##
1124
+ class AsyncioBytesChannelTransport(asyncio.Transport):
1125
+ def __init__(self, reader: asyncio.StreamReader) -> None:
1126
+ super().__init__()
1404
1127
 
1128
+ self.reader = reader
1129
+ self.closed: asyncio.Future = asyncio.Future()
1405
1130
 
1406
- DeployHome = ta.NewType('DeployHome', str)
1131
+ # @ta.override
1132
+ def write(self, data: bytes) -> None:
1133
+ self.reader.feed_data(data)
1407
1134
 
1408
- DeployRev = ta.NewType('DeployRev', str)
1135
+ # @ta.override
1136
+ def close(self) -> None:
1137
+ self.reader.feed_eof()
1138
+ if not self.closed.done():
1139
+ self.closed.set_result(True)
1409
1140
 
1141
+ # @ta.override
1142
+ def is_closing(self) -> bool:
1143
+ return self.closed.done()
1410
1144
 
1411
- ########################################
1412
- # ../../pyremote.py
1413
- """
1414
- Basically this: https://mitogen.networkgenomics.com/howitworks.html
1415
1145
 
1416
- TODO:
1417
- - log: ta.Optional[logging.Logger] = None + log.debug's
1418
- """
1146
+ def asyncio_create_bytes_channel(
1147
+ loop: ta.Any = None,
1148
+ ) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
1149
+ if loop is None:
1150
+ loop = asyncio.get_running_loop()
1419
1151
 
1152
+ reader = asyncio.StreamReader()
1153
+ protocol = asyncio.StreamReaderProtocol(reader)
1154
+ transport = AsyncioBytesChannelTransport(reader)
1155
+ writer = asyncio.StreamWriter(transport, protocol, reader, loop)
1420
1156
 
1421
- ##
1157
+ return reader, writer
1422
1158
 
1423
1159
 
1424
- @dc.dataclass(frozen=True)
1425
- class PyremoteBootstrapOptions:
1426
- debug: bool = False
1160
+ ########################################
1161
+ # ../../../omlish/asyncs/asyncio/streams.py
1427
1162
 
1428
- DEFAULT_MAIN_NAME_OVERRIDE: ta.ClassVar[str] = '__pyremote__'
1429
- main_name_override: ta.Optional[str] = DEFAULT_MAIN_NAME_OVERRIDE
1430
1163
 
1164
+ ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
1431
1165
 
1432
- ##
1433
1166
 
1167
+ async def asyncio_open_stream_reader(
1168
+ f: ta.IO,
1169
+ loop: ta.Any = None,
1170
+ *,
1171
+ limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1172
+ ) -> asyncio.StreamReader:
1173
+ if loop is None:
1174
+ loop = asyncio.get_running_loop()
1434
1175
 
1435
- @dc.dataclass(frozen=True)
1436
- class PyremoteEnvInfo:
1437
- sys_base_prefix: str
1438
- sys_byteorder: str
1439
- sys_defaultencoding: str
1440
- sys_exec_prefix: str
1441
- sys_executable: str
1442
- sys_implementation_name: str
1443
- sys_path: ta.List[str]
1444
- sys_platform: str
1445
- sys_prefix: str
1446
- sys_version: str
1447
- sys_version_info: ta.List[ta.Union[int, str]]
1176
+ reader = asyncio.StreamReader(limit=limit, loop=loop)
1177
+ await loop.connect_read_pipe(
1178
+ lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1179
+ f,
1180
+ )
1448
1181
 
1449
- platform_architecture: ta.List[str]
1450
- platform_machine: str
1451
- platform_platform: str
1452
- platform_processor: str
1453
- platform_system: str
1454
- platform_release: str
1455
- platform_version: str
1182
+ return reader
1456
1183
 
1457
- site_userbase: str
1458
1184
 
1459
- os_cwd: str
1460
- os_gid: int
1461
- os_loadavg: ta.List[float]
1462
- os_login: ta.Optional[str]
1463
- os_pgrp: int
1464
- os_pid: int
1465
- os_ppid: int
1466
- os_uid: int
1185
+ async def asyncio_open_stream_writer(
1186
+ f: ta.IO,
1187
+ loop: ta.Any = None,
1188
+ ) -> asyncio.StreamWriter:
1189
+ if loop is None:
1190
+ loop = asyncio.get_running_loop()
1467
1191
 
1468
- pw_name: str
1469
- pw_uid: int
1470
- pw_gid: int
1471
- pw_gecos: str
1472
- pw_dir: str
1473
- pw_shell: str
1192
+ writer_transport, writer_protocol = await loop.connect_write_pipe(
1193
+ lambda: asyncio.streams.FlowControlMixin(loop=loop),
1194
+ f,
1195
+ )
1474
1196
 
1475
- env_path: ta.Optional[str]
1197
+ return asyncio.streams.StreamWriter(
1198
+ writer_transport,
1199
+ writer_protocol,
1200
+ None,
1201
+ loop,
1202
+ )
1476
1203
 
1477
1204
 
1478
- def _get_pyremote_env_info() -> PyremoteEnvInfo:
1479
- os_uid = os.getuid()
1205
+ ########################################
1206
+ # ../../../omlish/asyncs/asyncio/timeouts.py
1480
1207
 
1481
- pw = pwd.getpwuid(os_uid)
1482
1208
 
1483
- os_login: ta.Optional[str]
1484
- try:
1485
- os_login = os.getlogin()
1486
- except OSError:
1487
- os_login = None
1209
+ def asyncio_maybe_timeout(
1210
+ fut: AwaitableT,
1211
+ timeout: ta.Optional[float] = None,
1212
+ ) -> AwaitableT:
1213
+ if timeout is not None:
1214
+ fut = asyncio.wait_for(fut, timeout) # type: ignore
1215
+ return fut
1488
1216
 
1489
- return PyremoteEnvInfo(
1490
- sys_base_prefix=sys.base_prefix,
1491
- sys_byteorder=sys.byteorder,
1492
- sys_defaultencoding=sys.getdefaultencoding(),
1493
- sys_exec_prefix=sys.exec_prefix,
1494
- sys_executable=sys.executable,
1495
- sys_implementation_name=sys.implementation.name,
1496
- sys_path=sys.path,
1497
- sys_platform=sys.platform,
1498
- sys_prefix=sys.prefix,
1499
- sys_version=sys.version,
1500
- sys_version_info=list(sys.version_info),
1501
1217
 
1502
- platform_architecture=list(platform.architecture()),
1503
- platform_machine=platform.machine(),
1504
- platform_platform=platform.platform(),
1505
- platform_processor=platform.processor(),
1506
- platform_system=platform.system(),
1507
- platform_release=platform.release(),
1508
- platform_version=platform.version(),
1218
+ ########################################
1219
+ # ../../../omlish/configs/types.py
1509
1220
 
1510
- site_userbase=site.getuserbase(),
1511
1221
 
1512
- os_cwd=os.getcwd(),
1513
- os_gid=os.getgid(),
1514
- os_loadavg=list(os.getloadavg()),
1515
- os_login=os_login,
1516
- os_pgrp=os.getpgrp(),
1517
- os_pid=os.getpid(),
1518
- os_ppid=os.getppid(),
1519
- os_uid=os_uid,
1222
+ #
1520
1223
 
1521
- pw_name=pw.pw_name,
1522
- pw_uid=pw.pw_uid,
1523
- pw_gid=pw.pw_gid,
1524
- pw_gecos=pw.pw_gecos,
1525
- pw_dir=pw.pw_dir,
1526
- pw_shell=pw.pw_shell,
1527
1224
 
1528
- env_path=os.environ.get('PATH'),
1529
- )
1225
+ ########################################
1226
+ # ../../../omlish/formats/ini/sections.py
1530
1227
 
1531
1228
 
1532
1229
  ##
1533
1230
 
1534
1231
 
1535
- _PYREMOTE_BOOTSTRAP_INPUT_FD = 100
1536
- _PYREMOTE_BOOTSTRAP_SRC_FD = 101
1537
-
1538
- _PYREMOTE_BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CHILD_PID'
1539
- _PYREMOTE_BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
1540
- _PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR = '_OPYR_CONTEXT_NAME'
1541
- _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR = '_OPYR_SRC_FILE'
1542
- _PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR = '_OPYR_OPTIONS_JSON'
1232
+ def extract_ini_sections(cp: configparser.ConfigParser) -> IniSectionSettingsMap:
1233
+ config_dct: ta.Dict[str, ta.Any] = {}
1234
+ for sec in cp.sections():
1235
+ cd = config_dct
1236
+ for k in sec.split('.'):
1237
+ cd = cd.setdefault(k, {})
1238
+ cd.update(cp.items(sec))
1239
+ return config_dct
1543
1240
 
1544
- _PYREMOTE_BOOTSTRAP_ACK0 = b'OPYR000\n'
1545
- _PYREMOTE_BOOTSTRAP_ACK1 = b'OPYR001\n'
1546
- _PYREMOTE_BOOTSTRAP_ACK2 = b'OPYR002\n'
1547
- _PYREMOTE_BOOTSTRAP_ACK3 = b'OPYR003\n'
1548
1241
 
1549
- _PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
1242
+ ##
1550
1243
 
1551
- _PYREMOTE_BOOTSTRAP_IMPORTS = [
1552
- 'base64',
1553
- 'os',
1554
- 'struct',
1555
- 'sys',
1556
- 'zlib',
1557
- ]
1558
1244
 
1245
+ def render_ini_sections(
1246
+ settings_by_section: IniSectionSettingsMap,
1247
+ ) -> str:
1248
+ out = io.StringIO()
1559
1249
 
1560
- def _pyremote_bootstrap_main(context_name: str) -> None:
1561
- # Get pid
1562
- pid = os.getpid()
1250
+ for i, (section, settings) in enumerate(settings_by_section.items()):
1251
+ if i:
1252
+ out.write('\n')
1563
1253
 
1564
- # Two copies of payload src to be sent to parent
1565
- r0, w0 = os.pipe()
1566
- r1, w1 = os.pipe()
1254
+ out.write(f'[{section}]\n')
1567
1255
 
1568
- if (cp := os.fork()):
1569
- # Parent process
1256
+ for k, v in settings.items():
1257
+ if isinstance(v, str):
1258
+ out.write(f'{k}={v}\n')
1259
+ else:
1260
+ for vv in v:
1261
+ out.write(f'{k}={vv}\n')
1570
1262
 
1571
- # Dup original stdin to comm_fd for use as comm channel
1572
- os.dup2(0, _PYREMOTE_BOOTSTRAP_INPUT_FD)
1263
+ return out.getvalue()
1573
1264
 
1574
- # Overwrite stdin (fed to python repl) with first copy of src
1575
- os.dup2(r0, 0)
1576
1265
 
1577
- # Dup second copy of src to src_fd to recover after launch
1578
- os.dup2(r1, _PYREMOTE_BOOTSTRAP_SRC_FD)
1266
+ ########################################
1267
+ # ../../../omlish/formats/toml/parser.py
1268
+ # SPDX-License-Identifier: MIT
1269
+ # SPDX-FileCopyrightText: 2021 Taneli Hukkinen
1270
+ # Licensed to PSF under a Contributor Agreement.
1271
+ #
1272
+ # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
1273
+ # --------------------------------------------
1274
+ #
1275
+ # 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
1276
+ # ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
1277
+ # documentation.
1278
+ #
1279
+ # 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
1280
+ # royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
1281
+ # works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
1282
+ # Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
1283
+ # 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
1284
+ # Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
1285
+ #
1286
+ # 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
1287
+ # wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
1288
+ # any such work a brief summary of the changes made to Python.
1289
+ #
1290
+ # 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
1291
+ # EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
1292
+ # OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
1293
+ # RIGHTS.
1294
+ #
1295
+ # 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
1296
+ # DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
1297
+ # ADVISED OF THE POSSIBILITY THEREOF.
1298
+ #
1299
+ # 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
1300
+ #
1301
+ # 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
1302
+ # venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
1303
+ # name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
1304
+ #
1305
+ # 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
1306
+ # License Agreement.
1307
+ #
1308
+ # https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
1579
1309
 
1580
- # Close remaining fd's
1581
- for f in [r0, w0, r1, w1]:
1582
- os.close(f)
1583
1310
 
1584
- # Save vars
1585
- env = os.environ
1586
- exe = sys.executable
1587
- env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
1588
- env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
1589
- env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
1311
+ ##
1590
1312
 
1591
- # Start repl reading stdin from r0
1592
- os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
1593
1313
 
1594
- else:
1595
- # Child process
1314
+ _TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
1596
1315
 
1597
- # Write first ack
1598
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK0)
1316
+ TOML_RE_NUMBER = re.compile(
1317
+ r"""
1318
+ 0
1319
+ (?:
1320
+ x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
1321
+ |
1322
+ b[01](?:_?[01])* # bin
1323
+ |
1324
+ o[0-7](?:_?[0-7])* # oct
1325
+ )
1326
+ |
1327
+ [+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
1328
+ (?P<floatpart>
1329
+ (?:\.[0-9](?:_?[0-9])*)? # optional fractional part
1330
+ (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
1331
+ )
1332
+ """,
1333
+ flags=re.VERBOSE,
1334
+ )
1335
+ TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
1336
+ TOML_RE_DATETIME = re.compile(
1337
+ rf"""
1338
+ ([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
1339
+ (?:
1340
+ [Tt ]
1341
+ {_TOML_TIME_RE_STR}
1342
+ (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
1343
+ )?
1344
+ """,
1345
+ flags=re.VERBOSE,
1346
+ )
1599
1347
 
1600
- # Write pid
1601
- os.write(1, struct.pack('<Q', pid))
1602
1348
 
1603
- # Read payload src from stdin
1604
- payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
1605
- if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
1606
- raise EOFError
1607
- payload_src = zlib.decompress(payload_z)
1349
+ def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
1350
+ """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
1608
1351
 
1609
- # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
1610
- # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
1611
- for w in [w0, w1]:
1612
- fp = os.fdopen(w, 'wb', 0)
1613
- fp.write(payload_src)
1614
- fp.close()
1352
+ Raises ValueError if the match does not correspond to a valid date or datetime.
1353
+ """
1354
+ (
1355
+ year_str,
1356
+ month_str,
1357
+ day_str,
1358
+ hour_str,
1359
+ minute_str,
1360
+ sec_str,
1361
+ micros_str,
1362
+ zulu_time,
1363
+ offset_sign_str,
1364
+ offset_hour_str,
1365
+ offset_minute_str,
1366
+ ) = match.groups()
1367
+ year, month, day = int(year_str), int(month_str), int(day_str)
1368
+ if hour_str is None:
1369
+ return datetime.date(year, month, day)
1370
+ hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
1371
+ micros = int(micros_str.ljust(6, '0')) if micros_str else 0
1372
+ if offset_sign_str:
1373
+ tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
1374
+ offset_hour_str, offset_minute_str, offset_sign_str,
1375
+ )
1376
+ elif zulu_time:
1377
+ tz = datetime.UTC
1378
+ else: # local date-time
1379
+ tz = None
1380
+ return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
1615
1381
 
1616
- # Write second ack
1617
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK1)
1618
1382
 
1619
- # Exit child
1620
- sys.exit(0)
1383
+ @functools.lru_cache() # noqa
1384
+ def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
1385
+ sign = 1 if sign_str == '+' else -1
1386
+ return datetime.timezone(
1387
+ datetime.timedelta(
1388
+ hours=sign * int(hour_str),
1389
+ minutes=sign * int(minute_str),
1390
+ ),
1391
+ )
1621
1392
 
1622
1393
 
1623
- ##
1394
+ def toml_match_to_localtime(match: re.Match) -> datetime.time:
1395
+ hour_str, minute_str, sec_str, micros_str = match.groups()
1396
+ micros = int(micros_str.ljust(6, '0')) if micros_str else 0
1397
+ return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
1624
1398
 
1625
1399
 
1626
- def pyremote_build_bootstrap_cmd(context_name: str) -> str:
1627
- if any(c in context_name for c in '\'"'):
1628
- raise NameError(context_name)
1400
+ def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
1401
+ if match.group('floatpart'):
1402
+ return parse_float(match.group())
1403
+ return int(match.group(), 0)
1629
1404
 
1630
- import inspect
1631
- import textwrap
1632
- bs_src = textwrap.dedent(inspect.getsource(_pyremote_bootstrap_main))
1633
1405
 
1634
- for gl in [
1635
- '_PYREMOTE_BOOTSTRAP_INPUT_FD',
1636
- '_PYREMOTE_BOOTSTRAP_SRC_FD',
1406
+ TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
1637
1407
 
1638
- '_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR',
1639
- '_PYREMOTE_BOOTSTRAP_ARGV0_VAR',
1640
- '_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR',
1408
+ # Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
1409
+ # functions.
1410
+ TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
1411
+ TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
1641
1412
 
1642
- '_PYREMOTE_BOOTSTRAP_ACK0',
1643
- '_PYREMOTE_BOOTSTRAP_ACK1',
1413
+ TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
1414
+ TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1644
1415
 
1645
- '_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT',
1646
- ]:
1647
- bs_src = bs_src.replace(gl, repr(globals()[gl]))
1416
+ TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
1648
1417
 
1649
- bs_src = '\n'.join(
1650
- cl
1651
- for l in bs_src.splitlines()
1652
- if (cl := (l.split('#')[0]).rstrip())
1653
- if cl.strip()
1654
- )
1655
-
1656
- bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
1657
- bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
1658
- if b'"' in bs_z85:
1659
- raise ValueError(bs_z85)
1660
-
1661
- stmts = [
1662
- f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
1663
- f'exec(zlib.decompress(base64.b85decode(b"{bs_z85.decode("ascii")}")))',
1664
- f'_pyremote_bootstrap_main("{context_name}")',
1665
- ]
1418
+ TOML_WS = frozenset(' \t')
1419
+ TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
1420
+ TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
1421
+ TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
1422
+ TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
1666
1423
 
1667
- cmd = '; '.join(stmts)
1668
- return cmd
1424
+ TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
1425
+ {
1426
+ '\\b': '\u0008', # backspace
1427
+ '\\t': '\u0009', # tab
1428
+ '\\n': '\u000A', # linefeed
1429
+ '\\f': '\u000C', # form feed
1430
+ '\\r': '\u000D', # carriage return
1431
+ '\\"': '\u0022', # quote
1432
+ '\\\\': '\u005C', # backslash
1433
+ },
1434
+ )
1669
1435
 
1670
1436
 
1671
- ##
1437
+ class TomlDecodeError(ValueError):
1438
+ """An error raised if a document is not valid TOML."""
1672
1439
 
1673
1440
 
1674
- @dc.dataclass(frozen=True)
1675
- class PyremotePayloadRuntime:
1676
- input: ta.BinaryIO
1677
- output: ta.BinaryIO
1678
- context_name: str
1679
- payload_src: str
1680
- options: PyremoteBootstrapOptions
1681
- env_info: PyremoteEnvInfo
1441
+ def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
1442
+ """Parse TOML from a binary file object."""
1443
+ b = fp.read()
1444
+ try:
1445
+ s = b.decode()
1446
+ except AttributeError:
1447
+ raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
1448
+ return toml_loads(s, parse_float=parse_float)
1682
1449
 
1683
1450
 
1684
- def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
1685
- # If src file var is not present we need to do initial finalization
1686
- if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
1687
- # Read second copy of payload src
1688
- r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
1689
- payload_src = r1.read().decode('utf-8')
1690
- r1.close()
1451
+ def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
1452
+ """Parse TOML from a string."""
1691
1453
 
1692
- # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
1693
- # a pipe at once.
1694
- os.waitpid(int(os.environ.pop(_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR)), 0)
1454
+ # The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
1455
+ try:
1456
+ src = s.replace('\r\n', '\n')
1457
+ except (AttributeError, TypeError):
1458
+ raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
1459
+ pos = 0
1460
+ out = TomlOutput(TomlNestedDict(), TomlFlags())
1461
+ header: TomlKey = ()
1462
+ parse_float = toml_make_safe_parse_float(parse_float)
1695
1463
 
1696
- # Read options
1697
- options_json_len = struct.unpack('<I', os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, 4))[0]
1698
- if len(options_json := os.read(_PYREMOTE_BOOTSTRAP_INPUT_FD, options_json_len)) != options_json_len:
1699
- raise EOFError
1700
- options = PyremoteBootstrapOptions(**json.loads(options_json.decode('utf-8')))
1464
+ # Parse one statement at a time (typically means one line in TOML source)
1465
+ while True:
1466
+ # 1. Skip line leading whitespace
1467
+ pos = toml_skip_chars(src, pos, TOML_WS)
1701
1468
 
1702
- # If debugging, re-exec as file
1703
- if options.debug:
1704
- # Write temp source file
1705
- import tempfile
1706
- tfd, tfn = tempfile.mkstemp('-pyremote.py')
1707
- os.write(tfd, payload_src.encode('utf-8'))
1708
- os.close(tfd)
1469
+ # 2. Parse rules. Expect one of the following:
1470
+ # - end of file
1471
+ # - end of line
1472
+ # - comment
1473
+ # - key/value pair
1474
+ # - append dict to list (and move to its namespace)
1475
+ # - create dict (and move to its namespace)
1476
+ # Skip trailing whitespace when applicable.
1477
+ try:
1478
+ char = src[pos]
1479
+ except IndexError:
1480
+ break
1481
+ if char == '\n':
1482
+ pos += 1
1483
+ continue
1484
+ if char in TOML_KEY_INITIAL_CHARS:
1485
+ pos = toml_key_value_rule(src, pos, out, header, parse_float)
1486
+ pos = toml_skip_chars(src, pos, TOML_WS)
1487
+ elif char == '[':
1488
+ try:
1489
+ second_char: ta.Optional[str] = src[pos + 1]
1490
+ except IndexError:
1491
+ second_char = None
1492
+ out.flags.finalize_pending()
1493
+ if second_char == '[':
1494
+ pos, header = toml_create_list_rule(src, pos, out)
1495
+ else:
1496
+ pos, header = toml_create_dict_rule(src, pos, out)
1497
+ pos = toml_skip_chars(src, pos, TOML_WS)
1498
+ elif char != '#':
1499
+ raise toml_suffixed_err(src, pos, 'Invalid statement')
1709
1500
 
1710
- # Set vars
1711
- os.environ[_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR] = tfn
1712
- os.environ[_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR] = options_json.decode('utf-8')
1501
+ # 3. Skip comment
1502
+ pos = toml_skip_comment(src, pos)
1713
1503
 
1714
- # Re-exec temp file
1715
- exe = os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR]
1716
- context_name = os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR]
1717
- os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)), tfn)
1504
+ # 4. Expect end of line or end of file
1505
+ try:
1506
+ char = src[pos]
1507
+ except IndexError:
1508
+ break
1509
+ if char != '\n':
1510
+ raise toml_suffixed_err(
1511
+ src, pos, 'Expected newline or end of document after a statement',
1512
+ )
1513
+ pos += 1
1718
1514
 
1719
- else:
1720
- # Load options json var
1721
- options_json_str = os.environ.pop(_PYREMOTE_BOOTSTRAP_OPTIONS_JSON_VAR)
1722
- options = PyremoteBootstrapOptions(**json.loads(options_json_str))
1515
+ return out.data.dict
1723
1516
 
1724
- # Read temp source file
1725
- with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
1726
- payload_src = sf.read()
1727
1517
 
1728
- # Restore vars
1729
- sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
1730
- context_name = os.environ.pop(_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR)
1518
+ class TomlFlags:
1519
+ """Flags that map to parsed keys/namespaces."""
1731
1520
 
1732
- # Write third ack
1733
- os.write(1, _PYREMOTE_BOOTSTRAP_ACK2)
1521
+ # Marks an immutable namespace (inline array or inline table).
1522
+ FROZEN = 0
1523
+ # Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
1524
+ EXPLICIT_NEST = 1
1734
1525
 
1735
- # Write env info
1736
- env_info = _get_pyremote_env_info()
1737
- env_info_json = json.dumps(dc.asdict(env_info), indent=None, separators=(',', ':')) # noqa
1738
- os.write(1, struct.pack('<I', len(env_info_json)))
1739
- os.write(1, env_info_json.encode('utf-8'))
1526
+ def __init__(self) -> None:
1527
+ self._flags: ta.Dict[str, dict] = {}
1528
+ self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
1740
1529
 
1741
- # Setup IO
1742
- input = os.fdopen(_PYREMOTE_BOOTSTRAP_INPUT_FD, 'rb', 0) # noqa
1743
- output = os.fdopen(os.dup(1), 'wb', 0) # noqa
1744
- os.dup2(nfd := os.open('/dev/null', os.O_WRONLY), 1)
1745
- os.close(nfd)
1530
+ def add_pending(self, key: TomlKey, flag: int) -> None:
1531
+ self._pending_flags.add((key, flag))
1746
1532
 
1747
- if (mn := options.main_name_override) is not None:
1748
- # Inspections like typing.get_type_hints need an entry in sys.modules.
1749
- sys.modules[mn] = sys.modules['__main__']
1533
+ def finalize_pending(self) -> None:
1534
+ for key, flag in self._pending_flags:
1535
+ self.set(key, flag, recursive=False)
1536
+ self._pending_flags.clear()
1750
1537
 
1751
- # Write fourth ack
1752
- output.write(_PYREMOTE_BOOTSTRAP_ACK3)
1538
+ def unset_all(self, key: TomlKey) -> None:
1539
+ cont = self._flags
1540
+ for k in key[:-1]:
1541
+ if k not in cont:
1542
+ return
1543
+ cont = cont[k]['nested']
1544
+ cont.pop(key[-1], None)
1753
1545
 
1754
- # Return
1755
- return PyremotePayloadRuntime(
1756
- input=input,
1757
- output=output,
1758
- context_name=context_name,
1759
- payload_src=payload_src,
1760
- options=options,
1761
- env_info=env_info,
1762
- )
1546
+ def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
1547
+ cont = self._flags
1548
+ key_parent, key_stem = key[:-1], key[-1]
1549
+ for k in key_parent:
1550
+ if k not in cont:
1551
+ cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
1552
+ cont = cont[k]['nested']
1553
+ if key_stem not in cont:
1554
+ cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
1555
+ cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
1763
1556
 
1557
+ def is_(self, key: TomlKey, flag: int) -> bool:
1558
+ if not key:
1559
+ return False # document root has no flags
1560
+ cont = self._flags
1561
+ for k in key[:-1]:
1562
+ if k not in cont:
1563
+ return False
1564
+ inner_cont = cont[k]
1565
+ if flag in inner_cont['recursive_flags']:
1566
+ return True
1567
+ cont = inner_cont['nested']
1568
+ key_stem = key[-1]
1569
+ if key_stem in cont:
1570
+ cont = cont[key_stem]
1571
+ return flag in cont['flags'] or flag in cont['recursive_flags']
1572
+ return False
1764
1573
 
1765
- ##
1766
1574
 
1575
+ class TomlNestedDict:
1576
+ def __init__(self) -> None:
1577
+ # The parsed content of the TOML document
1578
+ self.dict: ta.Dict[str, ta.Any] = {}
1767
1579
 
1768
- class PyremoteBootstrapDriver:
1769
- def __init__(
1580
+ def get_or_create_nest(
1770
1581
  self,
1771
- payload_src: ta.Union[str, ta.Sequence[str]],
1772
- options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
1773
- ) -> None:
1774
- super().__init__()
1775
-
1776
- self._payload_src = payload_src
1777
- self._options = options
1582
+ key: TomlKey,
1583
+ *,
1584
+ access_lists: bool = True,
1585
+ ) -> dict:
1586
+ cont: ta.Any = self.dict
1587
+ for k in key:
1588
+ if k not in cont:
1589
+ cont[k] = {}
1590
+ cont = cont[k]
1591
+ if access_lists and isinstance(cont, list):
1592
+ cont = cont[-1]
1593
+ if not isinstance(cont, dict):
1594
+ raise KeyError('There is no nest behind this key')
1595
+ return cont
1778
1596
 
1779
- self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
1780
- self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
1597
+ def append_nest_to_list(self, key: TomlKey) -> None:
1598
+ cont = self.get_or_create_nest(key[:-1])
1599
+ last_key = key[-1]
1600
+ if last_key in cont:
1601
+ list_ = cont[last_key]
1602
+ if not isinstance(list_, list):
1603
+ raise KeyError('An object other than list found behind this key')
1604
+ list_.append({})
1605
+ else:
1606
+ cont[last_key] = [{}]
1781
1607
 
1782
- self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
1783
- #
1784
1608
 
1785
- @classmethod
1786
- def _prepare_payload_src(
1787
- cls,
1788
- payload_src: ta.Union[str, ta.Sequence[str]],
1789
- options: PyremoteBootstrapOptions,
1790
- ) -> str:
1791
- parts: ta.List[str]
1792
- if isinstance(payload_src, str):
1793
- parts = [payload_src]
1794
- else:
1795
- parts = list(payload_src)
1609
+ class TomlOutput(ta.NamedTuple):
1610
+ data: TomlNestedDict
1611
+ flags: TomlFlags
1796
1612
 
1797
- if (mn := options.main_name_override) is not None:
1798
- parts.insert(0, f'__name__ = {mn!r}')
1799
1613
 
1800
- if len(parts) == 1:
1801
- return parts[0]
1802
- else:
1803
- return '\n\n'.join(parts)
1614
+ def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
1615
+ try:
1616
+ while src[pos] in chars:
1617
+ pos += 1
1618
+ except IndexError:
1619
+ pass
1620
+ return pos
1804
1621
 
1805
- #
1806
1622
 
1807
- @dc.dataclass(frozen=True)
1808
- class Read:
1809
- sz: int
1623
+ def toml_skip_until(
1624
+ src: str,
1625
+ pos: TomlPos,
1626
+ expect: str,
1627
+ *,
1628
+ error_on: ta.FrozenSet[str],
1629
+ error_on_eof: bool,
1630
+ ) -> TomlPos:
1631
+ try:
1632
+ new_pos = src.index(expect, pos)
1633
+ except ValueError:
1634
+ new_pos = len(src)
1635
+ if error_on_eof:
1636
+ raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
1810
1637
 
1811
- @dc.dataclass(frozen=True)
1812
- class Write:
1813
- d: bytes
1638
+ if not error_on.isdisjoint(src[pos:new_pos]):
1639
+ while src[pos] not in error_on:
1640
+ pos += 1
1641
+ raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
1642
+ return new_pos
1814
1643
 
1815
- class ProtocolError(Exception):
1816
- pass
1817
1644
 
1818
- @dc.dataclass(frozen=True)
1819
- class Result:
1820
- pid: int
1821
- env_info: PyremoteEnvInfo
1645
+ def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
1646
+ try:
1647
+ char: ta.Optional[str] = src[pos]
1648
+ except IndexError:
1649
+ char = None
1650
+ if char == '#':
1651
+ return toml_skip_until(
1652
+ src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
1653
+ )
1654
+ return pos
1822
1655
 
1823
- def gen(self) -> ta.Generator[ta.Union[Read, Write], ta.Optional[bytes], Result]:
1824
- # Read first ack (after fork)
1825
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK0)
1826
1656
 
1827
- # Read pid
1828
- d = yield from self._read(8)
1829
- pid = struct.unpack('<Q', d)[0]
1657
+ def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
1658
+ while True:
1659
+ pos_before_skip = pos
1660
+ pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1661
+ pos = toml_skip_comment(src, pos)
1662
+ if pos == pos_before_skip:
1663
+ return pos
1664
+
1665
+
1666
+ def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
1667
+ pos += 1 # Skip "["
1668
+ pos = toml_skip_chars(src, pos, TOML_WS)
1669
+ pos, key = toml_parse_key(src, pos)
1670
+
1671
+ if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
1672
+ raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
1673
+ out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
1674
+ try:
1675
+ out.data.get_or_create_nest(key)
1676
+ except KeyError:
1677
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1678
+
1679
+ if not src.startswith(']', pos):
1680
+ raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
1681
+ return pos + 1, key
1682
+
1683
+
1684
+ def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
1685
+ pos += 2 # Skip "[["
1686
+ pos = toml_skip_chars(src, pos, TOML_WS)
1687
+ pos, key = toml_parse_key(src, pos)
1688
+
1689
+ if out.flags.is_(key, TomlFlags.FROZEN):
1690
+ raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1691
+ # Free the namespace now that it points to another empty list item...
1692
+ out.flags.unset_all(key)
1693
+ # ...but this key precisely is still prohibited from table declaration
1694
+ out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
1695
+ try:
1696
+ out.data.append_nest_to_list(key)
1697
+ except KeyError:
1698
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1699
+
1700
+ if not src.startswith(']]', pos):
1701
+ raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
1702
+ return pos + 2, key
1703
+
1704
+
1705
+ def toml_key_value_rule(
1706
+ src: str,
1707
+ pos: TomlPos,
1708
+ out: TomlOutput,
1709
+ header: TomlKey,
1710
+ parse_float: TomlParseFloat,
1711
+ ) -> TomlPos:
1712
+ pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1713
+ key_parent, key_stem = key[:-1], key[-1]
1714
+ abs_key_parent = header + key_parent
1715
+
1716
+ relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
1717
+ for cont_key in relative_path_cont_keys:
1718
+ # Check that dotted key syntax does not redefine an existing table
1719
+ if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
1720
+ raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
1721
+ # Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
1722
+ # table sections.
1723
+ out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
1724
+
1725
+ if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
1726
+ raise toml_suffixed_err(
1727
+ src,
1728
+ pos,
1729
+ f'Cannot mutate immutable namespace {abs_key_parent}',
1730
+ )
1731
+
1732
+ try:
1733
+ nest = out.data.get_or_create_nest(abs_key_parent)
1734
+ except KeyError:
1735
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1736
+ if key_stem in nest:
1737
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
1738
+ # Mark inline table and array namespaces recursively immutable
1739
+ if isinstance(value, (dict, list)):
1740
+ out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
1741
+ nest[key_stem] = value
1742
+ return pos
1743
+
1744
+
1745
+ def toml_parse_key_value_pair(
1746
+ src: str,
1747
+ pos: TomlPos,
1748
+ parse_float: TomlParseFloat,
1749
+ ) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
1750
+ pos, key = toml_parse_key(src, pos)
1751
+ try:
1752
+ char: ta.Optional[str] = src[pos]
1753
+ except IndexError:
1754
+ char = None
1755
+ if char != '=':
1756
+ raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
1757
+ pos += 1
1758
+ pos = toml_skip_chars(src, pos, TOML_WS)
1759
+ pos, value = toml_parse_value(src, pos, parse_float)
1760
+ return pos, key, value
1761
+
1762
+
1763
+ def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
1764
+ pos, key_part = toml_parse_key_part(src, pos)
1765
+ key: TomlKey = (key_part,)
1766
+ pos = toml_skip_chars(src, pos, TOML_WS)
1767
+ while True:
1768
+ try:
1769
+ char: ta.Optional[str] = src[pos]
1770
+ except IndexError:
1771
+ char = None
1772
+ if char != '.':
1773
+ return pos, key
1774
+ pos += 1
1775
+ pos = toml_skip_chars(src, pos, TOML_WS)
1776
+ pos, key_part = toml_parse_key_part(src, pos)
1777
+ key += (key_part,)
1778
+ pos = toml_skip_chars(src, pos, TOML_WS)
1779
+
1780
+
1781
+ def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1782
+ try:
1783
+ char: ta.Optional[str] = src[pos]
1784
+ except IndexError:
1785
+ char = None
1786
+ if char in TOML_BARE_KEY_CHARS:
1787
+ start_pos = pos
1788
+ pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
1789
+ return pos, src[start_pos:pos]
1790
+ if char == "'":
1791
+ return toml_parse_literal_str(src, pos)
1792
+ if char == '"':
1793
+ return toml_parse_one_line_basic_str(src, pos)
1794
+ raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
1795
+
1796
+
1797
+ def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1798
+ pos += 1
1799
+ return toml_parse_basic_str(src, pos, multiline=False)
1800
+
1801
+
1802
+ def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
1803
+ pos += 1
1804
+ array: list = []
1805
+
1806
+ pos = toml_skip_comments_and_array_ws(src, pos)
1807
+ if src.startswith(']', pos):
1808
+ return pos + 1, array
1809
+ while True:
1810
+ pos, val = toml_parse_value(src, pos, parse_float)
1811
+ array.append(val)
1812
+ pos = toml_skip_comments_and_array_ws(src, pos)
1813
+
1814
+ c = src[pos:pos + 1]
1815
+ if c == ']':
1816
+ return pos + 1, array
1817
+ if c != ',':
1818
+ raise toml_suffixed_err(src, pos, 'Unclosed array')
1819
+ pos += 1
1820
+
1821
+ pos = toml_skip_comments_and_array_ws(src, pos)
1822
+ if src.startswith(']', pos):
1823
+ return pos + 1, array
1824
+
1825
+
1826
+ def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
1827
+ pos += 1
1828
+ nested_dict = TomlNestedDict()
1829
+ flags = TomlFlags()
1830
+
1831
+ pos = toml_skip_chars(src, pos, TOML_WS)
1832
+ if src.startswith('}', pos):
1833
+ return pos + 1, nested_dict.dict
1834
+ while True:
1835
+ pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
1836
+ key_parent, key_stem = key[:-1], key[-1]
1837
+ if flags.is_(key, TomlFlags.FROZEN):
1838
+ raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
1839
+ try:
1840
+ nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
1841
+ except KeyError:
1842
+ raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
1843
+ if key_stem in nest:
1844
+ raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
1845
+ nest[key_stem] = value
1846
+ pos = toml_skip_chars(src, pos, TOML_WS)
1847
+ c = src[pos:pos + 1]
1848
+ if c == '}':
1849
+ return pos + 1, nested_dict.dict
1850
+ if c != ',':
1851
+ raise toml_suffixed_err(src, pos, 'Unclosed inline table')
1852
+ if isinstance(value, (dict, list)):
1853
+ flags.set(key, TomlFlags.FROZEN, recursive=True)
1854
+ pos += 1
1855
+ pos = toml_skip_chars(src, pos, TOML_WS)
1856
+
1857
+
1858
+ def toml_parse_basic_str_escape(
1859
+ src: str,
1860
+ pos: TomlPos,
1861
+ *,
1862
+ multiline: bool = False,
1863
+ ) -> ta.Tuple[TomlPos, str]:
1864
+ escape_id = src[pos:pos + 2]
1865
+ pos += 2
1866
+ if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
1867
+ # Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
1868
+ # newline.
1869
+ if escape_id != '\\\n':
1870
+ pos = toml_skip_chars(src, pos, TOML_WS)
1871
+ try:
1872
+ char = src[pos]
1873
+ except IndexError:
1874
+ return pos, ''
1875
+ if char != '\n':
1876
+ raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
1877
+ pos += 1
1878
+ pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
1879
+ return pos, ''
1880
+ if escape_id == '\\u':
1881
+ return toml_parse_hex_char(src, pos, 4)
1882
+ if escape_id == '\\U':
1883
+ return toml_parse_hex_char(src, pos, 8)
1884
+ try:
1885
+ return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
1886
+ except KeyError:
1887
+ raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
1888
+
1889
+
1890
+ def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1891
+ return toml_parse_basic_str_escape(src, pos, multiline=True)
1892
+
1893
+
1894
+ def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
1895
+ hex_str = src[pos:pos + hex_len]
1896
+ if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
1897
+ raise toml_suffixed_err(src, pos, 'Invalid hex value')
1898
+ pos += hex_len
1899
+ hex_int = int(hex_str, 16)
1900
+ if not toml_is_unicode_scalar_value(hex_int):
1901
+ raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
1902
+ return pos, chr(hex_int)
1903
+
1904
+
1905
+ def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
1906
+ pos += 1 # Skip starting apostrophe
1907
+ start_pos = pos
1908
+ pos = toml_skip_until(
1909
+ src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
1910
+ )
1911
+ return pos + 1, src[start_pos:pos] # Skip ending apostrophe
1830
1912
 
1831
- # Write payload src
1832
- yield from self._write(struct.pack('<I', len(self._payload_z)))
1833
- yield from self._write(self._payload_z)
1834
1913
 
1835
- # Read second ack (after writing src copies)
1836
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
1914
+ def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
1915
+ pos += 3
1916
+ if src.startswith('\n', pos):
1917
+ pos += 1
1837
1918
 
1838
- # Write options
1839
- yield from self._write(struct.pack('<I', len(self._options_json)))
1840
- yield from self._write(self._options_json)
1919
+ if literal:
1920
+ delim = "'"
1921
+ end_pos = toml_skip_until(
1922
+ src,
1923
+ pos,
1924
+ "'''",
1925
+ error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
1926
+ error_on_eof=True,
1927
+ )
1928
+ result = src[pos:end_pos]
1929
+ pos = end_pos + 3
1930
+ else:
1931
+ delim = '"'
1932
+ pos, result = toml_parse_basic_str(src, pos, multiline=True)
1841
1933
 
1842
- # Read third ack (after reaping child process)
1843
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK2)
1934
+ # Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
1935
+ if not src.startswith(delim, pos):
1936
+ return pos, result
1937
+ pos += 1
1938
+ if not src.startswith(delim, pos):
1939
+ return pos, result + delim
1940
+ pos += 1
1941
+ return pos, result + (delim * 2)
1844
1942
 
1845
- # Read env info
1846
- d = yield from self._read(4)
1847
- env_info_json_len = struct.unpack('<I', d)[0]
1848
- d = yield from self._read(env_info_json_len)
1849
- env_info_json = d.decode('utf-8')
1850
- env_info = PyremoteEnvInfo(**json.loads(env_info_json))
1851
1943
 
1852
- # Read fourth ack (after finalization completed)
1853
- yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK3)
1944
+ def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
1945
+ if multiline:
1946
+ error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
1947
+ parse_escapes = toml_parse_basic_str_escape_multiline
1948
+ else:
1949
+ error_on = TOML_ILLEGAL_BASIC_STR_CHARS
1950
+ parse_escapes = toml_parse_basic_str_escape
1951
+ result = ''
1952
+ start_pos = pos
1953
+ while True:
1954
+ try:
1955
+ char = src[pos]
1956
+ except IndexError:
1957
+ raise toml_suffixed_err(src, pos, 'Unterminated string') from None
1958
+ if char == '"':
1959
+ if not multiline:
1960
+ return pos + 1, result + src[start_pos:pos]
1961
+ if src.startswith('"""', pos):
1962
+ return pos + 3, result + src[start_pos:pos]
1963
+ pos += 1
1964
+ continue
1965
+ if char == '\\':
1966
+ result += src[start_pos:pos]
1967
+ pos, parsed_escape = parse_escapes(src, pos)
1968
+ result += parsed_escape
1969
+ start_pos = pos
1970
+ continue
1971
+ if char in error_on:
1972
+ raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
1973
+ pos += 1
1854
1974
 
1855
- # Return
1856
- return self.Result(
1857
- pid=pid,
1858
- env_info=env_info,
1859
- )
1860
1975
 
1861
- def _read(self, sz: int) -> ta.Generator[Read, bytes, bytes]:
1862
- d = yield self.Read(sz)
1863
- if not isinstance(d, bytes):
1864
- raise self.ProtocolError(f'Expected bytes after read, got {d!r}')
1865
- if len(d) != sz:
1866
- raise self.ProtocolError(f'Read {len(d)} bytes, expected {sz}')
1867
- return d
1976
+ def toml_parse_value( # noqa: C901
1977
+ src: str,
1978
+ pos: TomlPos,
1979
+ parse_float: TomlParseFloat,
1980
+ ) -> ta.Tuple[TomlPos, ta.Any]:
1981
+ try:
1982
+ char: ta.Optional[str] = src[pos]
1983
+ except IndexError:
1984
+ char = None
1868
1985
 
1869
- def _expect(self, e: bytes) -> ta.Generator[Read, bytes, None]:
1870
- d = yield from self._read(len(e))
1871
- if d != e:
1872
- raise self.ProtocolError(f'Read {d!r}, expected {e!r}')
1986
+ # IMPORTANT: order conditions based on speed of checking and likelihood
1873
1987
 
1874
- def _write(self, d: bytes) -> ta.Generator[Write, ta.Optional[bytes], None]:
1875
- i = yield self.Write(d)
1876
- if i is not None:
1877
- raise self.ProtocolError('Unexpected input after write')
1988
+ # Basic strings
1989
+ if char == '"':
1990
+ if src.startswith('"""', pos):
1991
+ return toml_parse_multiline_str(src, pos, literal=False)
1992
+ return toml_parse_one_line_basic_str(src, pos)
1878
1993
 
1879
- #
1994
+ # Literal strings
1995
+ if char == "'":
1996
+ if src.startswith("'''", pos):
1997
+ return toml_parse_multiline_str(src, pos, literal=True)
1998
+ return toml_parse_literal_str(src, pos)
1880
1999
 
1881
- def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
1882
- gen = self.gen()
2000
+ # Booleans
2001
+ if char == 't':
2002
+ if src.startswith('true', pos):
2003
+ return pos + 4, True
2004
+ if char == 'f':
2005
+ if src.startswith('false', pos):
2006
+ return pos + 5, False
1883
2007
 
1884
- gi: ta.Optional[bytes] = None
1885
- while True:
1886
- try:
1887
- if gi is not None:
1888
- go = gen.send(gi)
1889
- else:
1890
- go = next(gen)
1891
- except StopIteration as e:
1892
- return e.value
2008
+ # Arrays
2009
+ if char == '[':
2010
+ return toml_parse_array(src, pos, parse_float)
1893
2011
 
1894
- if isinstance(go, self.Read):
1895
- if len(gi := input.read(go.sz)) != go.sz:
1896
- raise EOFError
1897
- elif isinstance(go, self.Write):
1898
- gi = None
1899
- output.write(go.d)
1900
- output.flush()
1901
- else:
1902
- raise TypeError(go)
2012
+ # Inline tables
2013
+ if char == '{':
2014
+ return toml_parse_inline_table(src, pos, parse_float)
1903
2015
 
1904
- async def async_run(
1905
- self,
1906
- input: ta.Any, # asyncio.StreamWriter # noqa
1907
- output: ta.Any, # asyncio.StreamReader
1908
- ) -> Result:
1909
- gen = self.gen()
2016
+ # Dates and times
2017
+ datetime_match = TOML_RE_DATETIME.match(src, pos)
2018
+ if datetime_match:
2019
+ try:
2020
+ datetime_obj = toml_match_to_datetime(datetime_match)
2021
+ except ValueError as e:
2022
+ raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
2023
+ return datetime_match.end(), datetime_obj
2024
+ localtime_match = TOML_RE_LOCALTIME.match(src, pos)
2025
+ if localtime_match:
2026
+ return localtime_match.end(), toml_match_to_localtime(localtime_match)
1910
2027
 
1911
- gi: ta.Optional[bytes] = None
1912
- while True:
1913
- try:
1914
- if gi is not None:
1915
- go = gen.send(gi)
1916
- else:
1917
- go = next(gen)
1918
- except StopIteration as e:
1919
- return e.value
2028
+ # Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
2029
+ # located after handling of dates and times.
2030
+ number_match = TOML_RE_NUMBER.match(src, pos)
2031
+ if number_match:
2032
+ return number_match.end(), toml_match_to_number(number_match, parse_float)
1920
2033
 
1921
- if isinstance(go, self.Read):
1922
- if len(gi := await input.read(go.sz)) != go.sz:
1923
- raise EOFError
1924
- elif isinstance(go, self.Write):
1925
- gi = None
1926
- output.write(go.d)
1927
- await output.drain()
1928
- else:
1929
- raise TypeError(go)
2034
+ # Special floats
2035
+ first_three = src[pos:pos + 3]
2036
+ if first_three in {'inf', 'nan'}:
2037
+ return pos + 3, parse_float(first_three)
2038
+ first_four = src[pos:pos + 4]
2039
+ if first_four in {'-inf', '+inf', '-nan', '+nan'}:
2040
+ return pos + 4, parse_float(first_four)
1930
2041
 
2042
+ raise toml_suffixed_err(src, pos, 'Invalid value')
1931
2043
 
1932
- ########################################
1933
- # ../../../omlish/asyncs/asyncio/channels.py
1934
2044
 
2045
+ def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
2046
+ """Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
1935
2047
 
1936
- class AsyncioBytesChannelTransport(asyncio.Transport):
1937
- def __init__(self, reader: asyncio.StreamReader) -> None:
1938
- super().__init__()
2048
+ def coord_repr(src: str, pos: TomlPos) -> str:
2049
+ if pos >= len(src):
2050
+ return 'end of document'
2051
+ line = src.count('\n', 0, pos) + 1
2052
+ if line == 1:
2053
+ column = pos + 1
2054
+ else:
2055
+ column = pos - src.rindex('\n', 0, pos)
2056
+ return f'line {line}, column {column}'
1939
2057
 
1940
- self.reader = reader
1941
- self.closed: asyncio.Future = asyncio.Future()
2058
+ return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
1942
2059
 
1943
- # @ta.override
1944
- def write(self, data: bytes) -> None:
1945
- self.reader.feed_data(data)
1946
2060
 
1947
- # @ta.override
1948
- def close(self) -> None:
1949
- self.reader.feed_eof()
1950
- if not self.closed.done():
1951
- self.closed.set_result(True)
2061
+ def toml_is_unicode_scalar_value(codepoint: int) -> bool:
2062
+ return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
1952
2063
 
1953
- # @ta.override
1954
- def is_closing(self) -> bool:
1955
- return self.closed.done()
1956
2064
 
2065
+ def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
2066
+ """A decorator to make `parse_float` safe.
1957
2067
 
1958
- def asyncio_create_bytes_channel(
1959
- loop: ta.Any = None,
1960
- ) -> ta.Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
1961
- if loop is None:
1962
- loop = asyncio.get_running_loop()
2068
+ `parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
2069
+ thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
2070
+ """
2071
+ # The default `float` callable never returns illegal types. Optimize it.
2072
+ if parse_float is float:
2073
+ return float
1963
2074
 
1964
- reader = asyncio.StreamReader()
1965
- protocol = asyncio.StreamReaderProtocol(reader)
1966
- transport = AsyncioBytesChannelTransport(reader)
1967
- writer = asyncio.StreamWriter(transport, protocol, reader, loop)
2075
+ def safe_parse_float(float_str: str) -> ta.Any:
2076
+ float_value = parse_float(float_str)
2077
+ if isinstance(float_value, (dict, list)):
2078
+ raise ValueError('parse_float must not return dicts or lists') # noqa
2079
+ return float_value
1968
2080
 
1969
- return reader, writer
2081
+ return safe_parse_float
1970
2082
 
1971
2083
 
1972
2084
  ########################################
1973
- # ../../../omlish/asyncs/asyncio/streams.py
2085
+ # ../../../omlish/formats/toml/writer.py
1974
2086
 
1975
2087
 
1976
- ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
2088
+ class TomlWriter:
2089
+ @dc.dataclass(frozen=True)
2090
+ class Literal:
2091
+ s: str
1977
2092
 
2093
+ def __init__(self, out: ta.TextIO) -> None:
2094
+ super().__init__()
2095
+ self._out = out
1978
2096
 
1979
- async def asyncio_open_stream_reader(
1980
- f: ta.IO,
1981
- loop: ta.Any = None,
1982
- *,
1983
- limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
1984
- ) -> asyncio.StreamReader:
1985
- if loop is None:
1986
- loop = asyncio.get_running_loop()
2097
+ self._indent = 0
2098
+ self._wrote_indent = False
1987
2099
 
1988
- reader = asyncio.StreamReader(limit=limit, loop=loop)
1989
- await loop.connect_read_pipe(
1990
- lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
1991
- f,
1992
- )
2100
+ #
1993
2101
 
1994
- return reader
2102
+ def _w(self, s: str) -> None:
2103
+ if not self._wrote_indent:
2104
+ self._out.write(' ' * self._indent)
2105
+ self._wrote_indent = True
2106
+ self._out.write(s)
1995
2107
 
2108
+ def _nl(self) -> None:
2109
+ self._out.write('\n')
2110
+ self._wrote_indent = False
1996
2111
 
1997
- async def asyncio_open_stream_writer(
1998
- f: ta.IO,
1999
- loop: ta.Any = None,
2000
- ) -> asyncio.StreamWriter:
2001
- if loop is None:
2002
- loop = asyncio.get_running_loop()
2112
+ def _needs_quote(self, s: str) -> bool:
2113
+ return (
2114
+ not s or
2115
+ any(c in s for c in '\'"\n') or
2116
+ s[0] not in string.ascii_letters
2117
+ )
2003
2118
 
2004
- writer_transport, writer_protocol = await loop.connect_write_pipe(
2005
- lambda: asyncio.streams.FlowControlMixin(loop=loop),
2006
- f,
2007
- )
2119
+ def _maybe_quote(self, s: str) -> str:
2120
+ if self._needs_quote(s):
2121
+ return repr(s)
2122
+ else:
2123
+ return s
2008
2124
 
2009
- return asyncio.streams.StreamWriter(
2010
- writer_transport,
2011
- writer_protocol,
2012
- None,
2013
- loop,
2014
- )
2125
+ #
2015
2126
 
2127
+ def write_root(self, obj: ta.Mapping) -> None:
2128
+ for i, (k, v) in enumerate(obj.items()):
2129
+ if i:
2130
+ self._nl()
2131
+ self._w('[')
2132
+ self._w(self._maybe_quote(k))
2133
+ self._w(']')
2134
+ self._nl()
2135
+ self.write_table_contents(v)
2136
+
2137
+ def write_table_contents(self, obj: ta.Mapping) -> None:
2138
+ for k, v in obj.items():
2139
+ self.write_key(k)
2140
+ self._w(' = ')
2141
+ self.write_value(v)
2142
+ self._nl()
2143
+
2144
+ def write_array(self, obj: ta.Sequence) -> None:
2145
+ self._w('[')
2146
+ self._nl()
2147
+ self._indent += 1
2148
+ for e in obj:
2149
+ self.write_value(e)
2150
+ self._w(',')
2151
+ self._nl()
2152
+ self._indent -= 1
2153
+ self._w(']')
2154
+
2155
+ def write_inline_table(self, obj: ta.Mapping) -> None:
2156
+ self._w('{')
2157
+ for i, (k, v) in enumerate(obj.items()):
2158
+ if i:
2159
+ self._w(', ')
2160
+ self.write_key(k)
2161
+ self._w(' = ')
2162
+ self.write_value(v)
2163
+ self._w('}')
2164
+
2165
+ def write_inline_array(self, obj: ta.Sequence) -> None:
2166
+ self._w('[')
2167
+ for i, e in enumerate(obj):
2168
+ if i:
2169
+ self._w(', ')
2170
+ self.write_value(e)
2171
+ self._w(']')
2172
+
2173
+ def write_key(self, obj: ta.Any) -> None:
2174
+ if isinstance(obj, TomlWriter.Literal):
2175
+ self._w(obj.s)
2176
+ elif isinstance(obj, str):
2177
+ self._w(self._maybe_quote(obj.replace('_', '-')))
2178
+ elif isinstance(obj, int):
2179
+ self._w(repr(str(obj)))
2180
+ else:
2181
+ raise TypeError(obj)
2016
2182
 
2017
- ########################################
2018
- # ../../../omlish/asyncs/asyncio/timeouts.py
2183
+ def write_value(self, obj: ta.Any) -> None:
2184
+ if isinstance(obj, bool):
2185
+ self._w(str(obj).lower())
2186
+ elif isinstance(obj, (str, int, float)):
2187
+ self._w(repr(obj))
2188
+ elif isinstance(obj, ta.Mapping):
2189
+ self.write_inline_table(obj)
2190
+ elif isinstance(obj, ta.Sequence):
2191
+ if not obj:
2192
+ self.write_inline_array(obj)
2193
+ else:
2194
+ self.write_array(obj)
2195
+ else:
2196
+ raise TypeError(obj)
2019
2197
 
2198
+ #
2020
2199
 
2021
- def asyncio_maybe_timeout(
2022
- fut: AwaitableT,
2023
- timeout: ta.Optional[float] = None,
2024
- ) -> AwaitableT:
2025
- if timeout is not None:
2026
- fut = asyncio.wait_for(fut, timeout) # type: ignore
2027
- return fut
2200
+ @classmethod
2201
+ def write_str(cls, obj: ta.Any) -> str:
2202
+ out = io.StringIO()
2203
+ cls(out).write_value(obj)
2204
+ return out.getvalue()
2028
2205
 
2029
2206
 
2030
2207
  ########################################
@@ -4672,131 +4849,360 @@ class ArgparseCli:
4672
4849
  elif k in objs:
4673
4850
  del [k]
4674
4851
 
4675
- #
4852
+ #
4853
+
4854
+ anns = ta.get_type_hints(_ArgparseCliAnnotationBox({
4855
+ **{k: v for bcls in reversed(mro) for k, v in getattr(bcls, '__annotations__', {}).items()},
4856
+ **ns.get('__annotations__', {}),
4857
+ }), globalns=ns.get('__globals__', {}))
4858
+
4859
+ #
4860
+
4861
+ if '_parser' in ns:
4862
+ parser = check.isinstance(ns['_parser'], argparse.ArgumentParser)
4863
+ else:
4864
+ parser = argparse.ArgumentParser()
4865
+ setattr(cls, '_parser', parser)
4866
+
4867
+ #
4868
+
4869
+ subparsers = parser.add_subparsers()
4870
+
4871
+ for att, obj in objs.items():
4872
+ if isinstance(obj, ArgparseCmd):
4873
+ if obj.parent is not None:
4874
+ raise NotImplementedError
4875
+
4876
+ for cn in [obj.name, *(obj.aliases or [])]:
4877
+ subparser = subparsers.add_parser(cn)
4878
+
4879
+ for arg in (obj.args or []):
4880
+ if (
4881
+ len(arg.args) == 1 and
4882
+ isinstance(arg.args[0], str) and
4883
+ not (n := check.isinstance(arg.args[0], str)).startswith('-') and
4884
+ 'metavar' not in arg.kwargs
4885
+ ):
4886
+ subparser.add_argument(
4887
+ n.replace('-', '_'),
4888
+ **arg.kwargs,
4889
+ metavar=n,
4890
+ )
4891
+ else:
4892
+ subparser.add_argument(*arg.args, **arg.kwargs)
4893
+
4894
+ subparser.set_defaults(_cmd=obj)
4895
+
4896
+ elif isinstance(obj, ArgparseArg):
4897
+ if att in anns:
4898
+ ann_kwargs = _get_argparse_arg_ann_kwargs(anns[att])
4899
+ obj.kwargs = {**ann_kwargs, **obj.kwargs}
4900
+
4901
+ if not obj.dest:
4902
+ if 'dest' in obj.kwargs:
4903
+ obj.dest = obj.kwargs['dest']
4904
+ else:
4905
+ obj.dest = obj.kwargs['dest'] = att # type: ignore
4906
+
4907
+ parser.add_argument(*obj.args, **obj.kwargs)
4908
+
4909
+ else:
4910
+ raise TypeError(obj)
4911
+
4912
+ #
4913
+
4914
+ _parser: ta.ClassVar[argparse.ArgumentParser]
4915
+
4916
+ @classmethod
4917
+ def get_parser(cls) -> argparse.ArgumentParser:
4918
+ return cls._parser
4919
+
4920
+ @property
4921
+ def argv(self) -> ta.Sequence[str]:
4922
+ return self._argv
4923
+
4924
+ @property
4925
+ def args(self) -> argparse.Namespace:
4926
+ return self._args
4927
+
4928
+ @property
4929
+ def unknown_args(self) -> ta.Sequence[str]:
4930
+ return self._unknown_args
4931
+
4932
+ #
4933
+
4934
+ def _bind_cli_cmd(self, cmd: ArgparseCmd) -> ta.Callable:
4935
+ return cmd.__get__(self, type(self))
4936
+
4937
+ def prepare_cli_run(self) -> ta.Optional[ta.Callable]:
4938
+ cmd = getattr(self.args, '_cmd', None)
4939
+
4940
+ if self._unknown_args and not (cmd is not None and cmd.accepts_unknown):
4941
+ msg = f'unrecognized arguments: {" ".join(self._unknown_args)}'
4942
+ if (parser := self.get_parser()).exit_on_error: # type: ignore
4943
+ parser.error(msg)
4944
+ else:
4945
+ raise argparse.ArgumentError(None, msg)
4946
+
4947
+ if cmd is None:
4948
+ self.get_parser().print_help()
4949
+ return None
4950
+
4951
+ return self._bind_cli_cmd(cmd)
4952
+
4953
+ #
4954
+
4955
+ def cli_run(self) -> ta.Optional[int]:
4956
+ if (fn := self.prepare_cli_run()) is None:
4957
+ return 0
4958
+
4959
+ return fn()
4960
+
4961
+ def cli_run_and_exit(self) -> ta.NoReturn:
4962
+ sys.exit(rc if isinstance(rc := self.cli_run(), int) else 0)
4963
+
4964
+ def __call__(self, *, exit: bool = False) -> ta.Optional[int]: # noqa
4965
+ if exit:
4966
+ return self.cli_run_and_exit()
4967
+ else:
4968
+ return self.cli_run()
4969
+
4970
+ #
4971
+
4972
+ async def async_cli_run(self) -> ta.Optional[int]:
4973
+ if (fn := self.prepare_cli_run()) is None:
4974
+ return 0
4975
+
4976
+ return await fn()
4977
+
4978
+
4979
+ ########################################
4980
+ # ../../../omlish/configs/formats.py
4981
+ """
4982
+ Notes:
4983
+ - necessarily string-oriented
4984
+ - single file, as this is intended to be amalg'd and thus all included anyway
4985
+
4986
+ TODO:
4987
+ - ConfigDataMapper? to_map -> ConfigMap?
4988
+ - nginx ?
4989
+ - raw ?
4990
+ """
4991
+
4992
+
4993
+ ##
4994
+
4995
+
4996
+ @dc.dataclass(frozen=True)
4997
+ class ConfigData(abc.ABC): # noqa
4998
+ @abc.abstractmethod
4999
+ def as_map(self) -> ConfigMap:
5000
+ raise NotImplementedError
5001
+
5002
+
5003
+ #
5004
+
5005
+
5006
+ class ConfigLoader(abc.ABC, ta.Generic[ConfigDataT]):
5007
+ @property
5008
+ def file_exts(self) -> ta.Sequence[str]:
5009
+ return ()
5010
+
5011
+ def match_file(self, n: str) -> bool:
5012
+ return '.' in n and n.split('.')[-1] in check.not_isinstance(self.file_exts, str)
5013
+
5014
+ #
5015
+
5016
+ def load_file(self, p: str) -> ConfigDataT:
5017
+ with open(p) as f:
5018
+ return self.load_str(f.read())
5019
+
5020
+ @abc.abstractmethod
5021
+ def load_str(self, s: str) -> ConfigDataT:
5022
+ raise NotImplementedError
5023
+
5024
+
5025
+ #
5026
+
5027
+
5028
+ class ConfigRenderer(abc.ABC, ta.Generic[ConfigDataT]):
5029
+ @property
5030
+ @abc.abstractmethod
5031
+ def data_cls(self) -> ta.Type[ConfigDataT]:
5032
+ raise NotImplementedError
5033
+
5034
+ def match_data(self, d: ConfigDataT) -> bool:
5035
+ return isinstance(d, self.data_cls)
5036
+
5037
+ #
5038
+
5039
+ @abc.abstractmethod
5040
+ def render(self, d: ConfigDataT) -> str:
5041
+ raise NotImplementedError
5042
+
5043
+
5044
+ ##
5045
+
5046
+
5047
+ @dc.dataclass(frozen=True)
5048
+ class ObjConfigData(ConfigData, abc.ABC):
5049
+ obj: ta.Any
5050
+
5051
+ def as_map(self) -> ConfigMap:
5052
+ return check.isinstance(self.obj, collections.abc.Mapping)
5053
+
5054
+
5055
+ ##
5056
+
5057
+
5058
+ @dc.dataclass(frozen=True)
5059
+ class JsonConfigData(ObjConfigData):
5060
+ pass
5061
+
5062
+
5063
+ class JsonConfigLoader(ConfigLoader[JsonConfigData]):
5064
+ file_exts = ('json',)
5065
+
5066
+ def load_str(self, s: str) -> JsonConfigData:
5067
+ return JsonConfigData(json.loads(s))
5068
+
5069
+
5070
+ class JsonConfigRenderer(ConfigRenderer[JsonConfigData]):
5071
+ data_cls = JsonConfigData
5072
+
5073
+ def render(self, d: JsonConfigData) -> str:
5074
+ return json_dumps_pretty(d.obj)
5075
+
5076
+
5077
+ ##
5078
+
5079
+
5080
+ @dc.dataclass(frozen=True)
5081
+ class TomlConfigData(ObjConfigData):
5082
+ pass
5083
+
5084
+
5085
+ class TomlConfigLoader(ConfigLoader[TomlConfigData]):
5086
+ file_exts = ('toml',)
5087
+
5088
+ def load_str(self, s: str) -> TomlConfigData:
5089
+ return TomlConfigData(toml_loads(s))
5090
+
5091
+
5092
+ class TomlConfigRenderer(ConfigRenderer[TomlConfigData]):
5093
+ data_cls = TomlConfigData
5094
+
5095
+ def render(self, d: TomlConfigData) -> str:
5096
+ return TomlWriter.write_str(d.obj)
5097
+
5098
+
5099
+ ##
5100
+
4676
5101
 
4677
- anns = ta.get_type_hints(_ArgparseCliAnnotationBox({
4678
- **{k: v for bcls in reversed(mro) for k, v in getattr(bcls, '__annotations__', {}).items()},
4679
- **ns.get('__annotations__', {}),
4680
- }), globalns=ns.get('__globals__', {}))
5102
+ @dc.dataclass(frozen=True)
5103
+ class YamlConfigData(ObjConfigData):
5104
+ pass
4681
5105
 
4682
- #
4683
5106
 
4684
- if '_parser' in ns:
4685
- parser = check.isinstance(ns['_parser'], argparse.ArgumentParser)
4686
- else:
4687
- parser = argparse.ArgumentParser()
4688
- setattr(cls, '_parser', parser)
5107
+ class YamlConfigLoader(ConfigLoader[YamlConfigData]):
5108
+ file_exts = ('yaml', 'yml')
4689
5109
 
4690
- #
5110
+ def load_str(self, s: str) -> YamlConfigData:
5111
+ return YamlConfigData(__import__('yaml').safe_load(s))
4691
5112
 
4692
- subparsers = parser.add_subparsers()
4693
5113
 
4694
- for att, obj in objs.items():
4695
- if isinstance(obj, ArgparseCmd):
4696
- if obj.parent is not None:
4697
- raise NotImplementedError
5114
+ class YamlConfigRenderer(ConfigRenderer[YamlConfigData]):
5115
+ data_cls = YamlConfigData
4698
5116
 
4699
- for cn in [obj.name, *(obj.aliases or [])]:
4700
- subparser = subparsers.add_parser(cn)
5117
+ def render(self, d: YamlConfigData) -> str:
5118
+ return __import__('yaml').safe_dump(d.obj)
4701
5119
 
4702
- for arg in (obj.args or []):
4703
- if (
4704
- len(arg.args) == 1 and
4705
- isinstance(arg.args[0], str) and
4706
- not (n := check.isinstance(arg.args[0], str)).startswith('-') and
4707
- 'metavar' not in arg.kwargs
4708
- ):
4709
- subparser.add_argument(
4710
- n.replace('-', '_'),
4711
- **arg.kwargs,
4712
- metavar=n,
4713
- )
4714
- else:
4715
- subparser.add_argument(*arg.args, **arg.kwargs)
4716
5120
 
4717
- subparser.set_defaults(_cmd=obj)
5121
+ ##
4718
5122
 
4719
- elif isinstance(obj, ArgparseArg):
4720
- if att in anns:
4721
- ann_kwargs = _get_argparse_arg_ann_kwargs(anns[att])
4722
- obj.kwargs = {**ann_kwargs, **obj.kwargs}
4723
5123
 
4724
- if not obj.dest:
4725
- if 'dest' in obj.kwargs:
4726
- obj.dest = obj.kwargs['dest']
4727
- else:
4728
- obj.dest = obj.kwargs['dest'] = att # type: ignore
5124
+ @dc.dataclass(frozen=True)
5125
+ class IniConfigData(ConfigData):
5126
+ sections: IniSectionSettingsMap
4729
5127
 
4730
- parser.add_argument(*obj.args, **obj.kwargs)
5128
+ def as_map(self) -> ConfigMap:
5129
+ return self.sections
4731
5130
 
4732
- else:
4733
- raise TypeError(obj)
4734
5131
 
4735
- #
5132
+ class IniConfigLoader(ConfigLoader[IniConfigData]):
5133
+ file_exts = ('ini',)
4736
5134
 
4737
- _parser: ta.ClassVar[argparse.ArgumentParser]
5135
+ def load_str(self, s: str) -> IniConfigData:
5136
+ cp = configparser.ConfigParser()
5137
+ cp.read_string(s)
5138
+ return IniConfigData(extract_ini_sections(cp))
4738
5139
 
4739
- @classmethod
4740
- def get_parser(cls) -> argparse.ArgumentParser:
4741
- return cls._parser
4742
5140
 
4743
- @property
4744
- def argv(self) -> ta.Sequence[str]:
4745
- return self._argv
5141
+ class IniConfigRenderer(ConfigRenderer[IniConfigData]):
5142
+ data_cls = IniConfigData
4746
5143
 
4747
- @property
4748
- def args(self) -> argparse.Namespace:
4749
- return self._args
5144
+ def render(self, d: IniConfigData) -> str:
5145
+ return render_ini_sections(d.sections)
4750
5146
 
4751
- @property
4752
- def unknown_args(self) -> ta.Sequence[str]:
4753
- return self._unknown_args
4754
5147
 
4755
- #
5148
+ ##
4756
5149
 
4757
- def _bind_cli_cmd(self, cmd: ArgparseCmd) -> ta.Callable:
4758
- return cmd.__get__(self, type(self))
4759
5150
 
4760
- def prepare_cli_run(self) -> ta.Optional[ta.Callable]:
4761
- cmd = getattr(self.args, '_cmd', None)
5151
+ @dc.dataclass(frozen=True)
5152
+ class SwitchedConfigFileLoader:
5153
+ loaders: ta.Sequence[ConfigLoader]
5154
+ default: ta.Optional[ConfigLoader] = None
4762
5155
 
4763
- if self._unknown_args and not (cmd is not None and cmd.accepts_unknown):
4764
- msg = f'unrecognized arguments: {" ".join(self._unknown_args)}'
4765
- if (parser := self.get_parser()).exit_on_error: # type: ignore
4766
- parser.error(msg)
4767
- else:
4768
- raise argparse.ArgumentError(None, msg)
5156
+ def load_file(self, p: str) -> ConfigData:
5157
+ n = os.path.basename(p)
4769
5158
 
4770
- if cmd is None:
4771
- self.get_parser().print_help()
4772
- return None
5159
+ for l in self.loaders:
5160
+ if l.match_file(n):
5161
+ return l.load_file(p)
4773
5162
 
4774
- return self._bind_cli_cmd(cmd)
5163
+ if (d := self.default) is not None:
5164
+ return d.load_file(p)
4775
5165
 
4776
- #
5166
+ raise NameError(n)
4777
5167
 
4778
- def cli_run(self) -> ta.Optional[int]:
4779
- if (fn := self.prepare_cli_run()) is None:
4780
- return 0
4781
5168
 
4782
- return fn()
5169
+ DEFAULT_CONFIG_LOADERS: ta.Sequence[ConfigLoader] = [
5170
+ JsonConfigLoader(),
5171
+ TomlConfigLoader(),
5172
+ YamlConfigLoader(),
5173
+ IniConfigLoader(),
5174
+ ]
4783
5175
 
4784
- def cli_run_and_exit(self) -> ta.NoReturn:
4785
- sys.exit(rc if isinstance(rc := self.cli_run(), int) else 0)
5176
+ DEFAULT_CONFIG_LOADER: ConfigLoader = JsonConfigLoader()
4786
5177
 
4787
- def __call__(self, *, exit: bool = False) -> ta.Optional[int]: # noqa
4788
- if exit:
4789
- return self.cli_run_and_exit()
4790
- else:
4791
- return self.cli_run()
5178
+ DEFAULT_CONFIG_FILE_LOADER = SwitchedConfigFileLoader(
5179
+ loaders=DEFAULT_CONFIG_LOADERS,
5180
+ default=DEFAULT_CONFIG_LOADER,
5181
+ )
4792
5182
 
4793
- #
4794
5183
 
4795
- async def async_cli_run(self) -> ta.Optional[int]:
4796
- if (fn := self.prepare_cli_run()) is None:
4797
- return 0
5184
+ ##
4798
5185
 
4799
- return await fn()
5186
+
5187
+ @dc.dataclass(frozen=True)
5188
+ class SwitchedConfigRenderer:
5189
+ renderers: ta.Sequence[ConfigRenderer]
5190
+
5191
+ def render(self, d: ConfigData) -> str:
5192
+ for r in self.renderers:
5193
+ if r.match_data(d):
5194
+ return r.render(d)
5195
+ raise TypeError(d)
5196
+
5197
+
5198
+ DEFAULT_CONFIG_RENDERERS: ta.Sequence[ConfigRenderer] = [
5199
+ JsonConfigRenderer(),
5200
+ TomlConfigRenderer(),
5201
+ YamlConfigRenderer(),
5202
+ IniConfigRenderer(),
5203
+ ]
5204
+
5205
+ DEFAULT_CONFIG_RENDERER = SwitchedConfigRenderer(DEFAULT_CONFIG_RENDERERS)
4800
5206
 
4801
5207
 
4802
5208
  ########################################
@@ -6895,121 +7301,6 @@ def bind_interp_uv() -> InjectorBindings:
6895
7301
  return inj.as_bindings(*lst)
6896
7302
 
6897
7303
 
6898
- ########################################
6899
- # ../../configs.py
6900
-
6901
-
6902
- ##
6903
-
6904
-
6905
- def parse_config_file(
6906
- name: str,
6907
- f: ta.TextIO,
6908
- ) -> ConfigMapping:
6909
- if name.endswith('.toml'):
6910
- return toml_loads(f.read())
6911
-
6912
- elif any(name.endswith(e) for e in ('.yml', '.yaml')):
6913
- yaml = __import__('yaml')
6914
- return yaml.safe_load(f)
6915
-
6916
- elif name.endswith('.ini'):
6917
- import configparser
6918
- cp = configparser.ConfigParser()
6919
- cp.read_file(f)
6920
- config_dct: ta.Dict[str, ta.Any] = {}
6921
- for sec in cp.sections():
6922
- cd = config_dct
6923
- for k in sec.split('.'):
6924
- cd = cd.setdefault(k, {})
6925
- cd.update(cp.items(sec))
6926
- return config_dct
6927
-
6928
- else:
6929
- return json.loads(f.read())
6930
-
6931
-
6932
- def read_config_file(
6933
- path: str,
6934
- cls: ta.Type[T],
6935
- *,
6936
- prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
6937
- msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
6938
- ) -> T:
6939
- with open(path) as cf:
6940
- config_dct = parse_config_file(os.path.basename(path), cf)
6941
-
6942
- if prepare is not None:
6943
- config_dct = prepare(config_dct)
6944
-
6945
- return msh.unmarshal_obj(config_dct, cls)
6946
-
6947
-
6948
- ##
6949
-
6950
-
6951
- def build_config_named_children(
6952
- o: ta.Union[
6953
- ta.Sequence[ConfigMapping],
6954
- ta.Mapping[str, ConfigMapping],
6955
- None,
6956
- ],
6957
- *,
6958
- name_key: str = 'name',
6959
- ) -> ta.Optional[ta.Sequence[ConfigMapping]]:
6960
- if o is None:
6961
- return None
6962
-
6963
- lst: ta.List[ConfigMapping] = []
6964
- if isinstance(o, ta.Mapping):
6965
- for k, v in o.items():
6966
- check.isinstance(v, ta.Mapping)
6967
- if name_key in v:
6968
- n = v[name_key]
6969
- if k != n:
6970
- raise KeyError(f'Given names do not match: {n} != {k}')
6971
- lst.append(v)
6972
- else:
6973
- lst.append({name_key: k, **v})
6974
-
6975
- else:
6976
- check.not_isinstance(o, str)
6977
- lst.extend(o)
6978
-
6979
- seen = set()
6980
- for d in lst:
6981
- n = d['name']
6982
- if n in d:
6983
- raise KeyError(f'Duplicate name: {n}')
6984
- seen.add(n)
6985
-
6986
- return lst
6987
-
6988
-
6989
- ##
6990
-
6991
-
6992
- def render_ini_config(
6993
- settings_by_section: IniConfigSectionSettingsMap,
6994
- ) -> str:
6995
- out = io.StringIO()
6996
-
6997
- for i, (section, settings) in enumerate(settings_by_section.items()):
6998
- if i:
6999
- out.write('\n')
7000
-
7001
- out.write(f'[{section}]\n')
7002
-
7003
- for k, v in settings.items():
7004
- if isinstance(v, str):
7005
- out.write(f'{k}={v}\n')
7006
- else:
7007
- for vv in v:
7008
- out.write(f'{k}={vv}\n')
7009
-
7010
- return out.getvalue()
7011
-
7012
-
7013
7304
  ########################################
7014
7305
  # ../commands/marshal.py
7015
7306
 
@@ -7060,7 +7351,105 @@ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
7060
7351
  # ../commands/types.py
7061
7352
 
7062
7353
 
7063
- CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
7354
+ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
7355
+
7356
+
7357
+ ########################################
7358
+ # ../deploy/conf/specs.py
7359
+
7360
+
7361
+ ##
7362
+
7363
+
7364
+ class DeployAppConfContent(abc.ABC): # noqa
7365
+ pass
7366
+
7367
+
7368
+ #
7369
+
7370
+
7371
+ @register_single_field_type_obj_marshaler('body')
7372
+ @dc.dataclass(frozen=True)
7373
+ class RawDeployAppConfContent(DeployAppConfContent):
7374
+ body: str
7375
+
7376
+
7377
+ #
7378
+
7379
+
7380
+ @register_single_field_type_obj_marshaler('obj')
7381
+ @dc.dataclass(frozen=True)
7382
+ class JsonDeployAppConfContent(DeployAppConfContent):
7383
+ obj: ta.Any
7384
+
7385
+
7386
+ #
7387
+
7388
+
7389
+ @register_single_field_type_obj_marshaler('sections')
7390
+ @dc.dataclass(frozen=True)
7391
+ class IniDeployAppConfContent(DeployAppConfContent):
7392
+ sections: IniSectionSettingsMap
7393
+
7394
+
7395
+ #
7396
+
7397
+
7398
+ @register_single_field_type_obj_marshaler('items')
7399
+ @dc.dataclass(frozen=True)
7400
+ class NginxDeployAppConfContent(DeployAppConfContent):
7401
+ items: ta.Any
7402
+
7403
+
7404
+ ##
7405
+
7406
+
7407
+ @dc.dataclass(frozen=True)
7408
+ class DeployAppConfFile:
7409
+ path: str
7410
+ content: DeployAppConfContent
7411
+
7412
+ def __post_init__(self) -> None:
7413
+ check_valid_deploy_spec_path(self.path)
7414
+
7415
+
7416
+ ##
7417
+
7418
+
7419
+ @dc.dataclass(frozen=True)
7420
+ class DeployAppConfLink: # noqa
7421
+ """
7422
+ May be either:
7423
+ - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
7424
+ - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
7425
+ - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
7426
+ """
7427
+
7428
+ src: str
7429
+
7430
+ kind: ta.Literal['current_only', 'all_active'] = 'current_only'
7431
+
7432
+ def __post_init__(self) -> None:
7433
+ check_valid_deploy_spec_path(self.src)
7434
+ if '/' in self.src:
7435
+ check.equal(self.src.count('/'), 1)
7436
+
7437
+
7438
+ ##
7439
+
7440
+
7441
+ @dc.dataclass(frozen=True)
7442
+ class DeployAppConfSpec:
7443
+ files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
7444
+
7445
+ links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
7446
+
7447
+ def __post_init__(self) -> None:
7448
+ if self.files:
7449
+ seen: ta.Set[str] = set()
7450
+ for f in self.files:
7451
+ check.not_in(f.path, seen)
7452
+ seen.add(f.path)
7064
7453
 
7065
7454
 
7066
7455
  ########################################
@@ -7388,6 +7777,106 @@ class SystemConfig:
7388
7777
  platform: ta.Optional[Platform] = None
7389
7778
 
7390
7779
 
7780
+ ########################################
7781
+ # ../../../omlish/configs/nginx.py
7782
+ """
7783
+ See:
7784
+ - https://nginx.org/en/docs/dev/development_guide.html
7785
+ - https://nginx.org/en/docs/dev/development_guide.html#config_directives
7786
+ - https://nginx.org/en/docs/example.html
7787
+ """
7788
+
7789
+
7790
+ @dc.dataclass()
7791
+ class NginxConfigItems:
7792
+ lst: ta.List['NginxConfigItem']
7793
+
7794
+ @classmethod
7795
+ def of(cls, obj: ta.Any) -> 'NginxConfigItems':
7796
+ if isinstance(obj, NginxConfigItems):
7797
+ return obj
7798
+ return cls([NginxConfigItem.of(e) for e in check.isinstance(obj, list)])
7799
+
7800
+
7801
+ @dc.dataclass()
7802
+ class NginxConfigItem:
7803
+ name: str
7804
+ args: ta.Optional[ta.List[str]] = None
7805
+ block: ta.Optional[NginxConfigItems] = None
7806
+
7807
+ @classmethod
7808
+ def of(cls, obj: ta.Any) -> 'NginxConfigItem':
7809
+ if isinstance(obj, NginxConfigItem):
7810
+ return obj
7811
+ args = check.isinstance(check.not_isinstance(obj, str), collections.abc.Sequence)
7812
+ name, args = check.isinstance(args[0], str), args[1:]
7813
+ if args and not isinstance(args[-1], str):
7814
+ block, args = NginxConfigItems.of(args[-1]), args[:-1]
7815
+ else:
7816
+ block = None
7817
+ return NginxConfigItem(name, [check.isinstance(e, str) for e in args], block=block)
7818
+
7819
+
7820
+ def render_nginx_config(wr: IndentWriter, obj: ta.Any) -> None:
7821
+ if isinstance(obj, NginxConfigItem):
7822
+ wr.write(obj.name)
7823
+ for e in obj.args or ():
7824
+ wr.write(' ')
7825
+ wr.write(e)
7826
+ if obj.block:
7827
+ wr.write(' {\n')
7828
+ with wr.indent():
7829
+ render_nginx_config(wr, obj.block)
7830
+ wr.write('}\n')
7831
+ else:
7832
+ wr.write(';\n')
7833
+
7834
+ elif isinstance(obj, NginxConfigItems):
7835
+ for e2 in obj.lst:
7836
+ render_nginx_config(wr, e2)
7837
+
7838
+ else:
7839
+ raise TypeError(obj)
7840
+
7841
+
7842
+ def render_nginx_config_str(obj: ta.Any) -> str:
7843
+ iw = IndentWriter()
7844
+ render_nginx_config(iw, obj)
7845
+ return iw.getvalue()
7846
+
7847
+
7848
+ ########################################
7849
+ # ../../../omlish/lite/configs.py
7850
+
7851
+
7852
+ ##
7853
+
7854
+
7855
+ def load_config_file_obj(
7856
+ f: str,
7857
+ cls: ta.Type[T],
7858
+ *,
7859
+ prepare: ta.Union[
7860
+ ta.Callable[[ConfigMap], ConfigMap],
7861
+ ta.Iterable[ta.Callable[[ConfigMap], ConfigMap]],
7862
+ ] = (),
7863
+ msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
7864
+ ) -> T:
7865
+ config_data = DEFAULT_CONFIG_FILE_LOADER.load_file(f)
7866
+
7867
+ config_dct = config_data.as_map()
7868
+
7869
+ if prepare is not None:
7870
+ if isinstance(prepare, ta.Iterable):
7871
+ pfs = list(prepare)
7872
+ else:
7873
+ pfs = [prepare]
7874
+ for pf in pfs:
7875
+ config_dct = pf(config_dct)
7876
+
7877
+ return msh.unmarshal_obj(config_dct, cls)
7878
+
7879
+
7391
7880
  ########################################
7392
7881
  # ../../../omlish/logs/standard.py
7393
7882
  """
@@ -7838,78 +8327,6 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
7838
8327
  return ret.decode().strip()
7839
8328
 
7840
8329
 
7841
- ########################################
7842
- # ../../../omserv/nginx/configs.py
7843
- """
7844
- TODO:
7845
- - omnibus/jmespath
7846
-
7847
- https://nginx.org/en/docs/dev/development_guide.html
7848
- https://nginx.org/en/docs/dev/development_guide.html#config_directives
7849
- https://nginx.org/en/docs/example.html
7850
-
7851
- https://github.com/yandex/gixy
7852
- """
7853
-
7854
-
7855
- @dc.dataclass()
7856
- class NginxConfigItems:
7857
- lst: ta.List['NginxConfigItem']
7858
-
7859
- @classmethod
7860
- def of(cls, obj: ta.Any) -> 'NginxConfigItems':
7861
- if isinstance(obj, NginxConfigItems):
7862
- return obj
7863
- return cls([NginxConfigItem.of(e) for e in check.isinstance(obj, list)])
7864
-
7865
-
7866
- @dc.dataclass()
7867
- class NginxConfigItem:
7868
- name: str
7869
- args: ta.Optional[ta.List[str]] = None
7870
- block: ta.Optional[NginxConfigItems] = None
7871
-
7872
- @classmethod
7873
- def of(cls, obj: ta.Any) -> 'NginxConfigItem':
7874
- if isinstance(obj, NginxConfigItem):
7875
- return obj
7876
- args = check.isinstance(check.not_isinstance(obj, str), collections.abc.Sequence)
7877
- name, args = check.isinstance(args[0], str), args[1:]
7878
- if args and not isinstance(args[-1], str):
7879
- block, args = NginxConfigItems.of(args[-1]), args[:-1]
7880
- else:
7881
- block = None
7882
- return NginxConfigItem(name, [check.isinstance(e, str) for e in args], block=block)
7883
-
7884
-
7885
- def render_nginx_config(wr: IndentWriter, obj: ta.Any) -> None:
7886
- if isinstance(obj, NginxConfigItem):
7887
- wr.write(obj.name)
7888
- for e in obj.args or ():
7889
- wr.write(' ')
7890
- wr.write(e)
7891
- if obj.block:
7892
- wr.write(' {\n')
7893
- with wr.indent():
7894
- render_nginx_config(wr, obj.block)
7895
- wr.write('}\n')
7896
- else:
7897
- wr.write(';\n')
7898
-
7899
- elif isinstance(obj, NginxConfigItems):
7900
- for e2 in obj.lst:
7901
- render_nginx_config(wr, e2)
7902
-
7903
- else:
7904
- raise TypeError(obj)
7905
-
7906
-
7907
- def render_nginx_config_str(obj: ta.Any) -> str:
7908
- iw = IndentWriter()
7909
- render_nginx_config(iw, obj)
7910
- return iw.getvalue()
7911
-
7912
-
7913
8330
  ########################################
7914
8331
  # ../../../omdev/interp/providers/base.py
7915
8332
  """
@@ -7977,113 +8394,15 @@ class LocalCommandExecutor(CommandExecutor):
7977
8394
  def __init__(
7978
8395
  self,
7979
8396
  *,
7980
- command_executors: CommandExecutorMap,
7981
- ) -> None:
7982
- super().__init__()
7983
-
7984
- self._command_executors = command_executors
7985
-
7986
- async def execute(self, cmd: Command) -> Command.Output:
7987
- ce: CommandExecutor = self._command_executors[type(cmd)]
7988
- return await ce.execute(cmd)
7989
-
7990
-
7991
- ########################################
7992
- # ../deploy/conf/specs.py
7993
-
7994
-
7995
- ##
7996
-
7997
-
7998
- class DeployAppConfContent(abc.ABC): # noqa
7999
- pass
8000
-
8001
-
8002
- #
8003
-
8004
-
8005
- @register_single_field_type_obj_marshaler('body')
8006
- @dc.dataclass(frozen=True)
8007
- class RawDeployAppConfContent(DeployAppConfContent):
8008
- body: str
8009
-
8010
-
8011
- #
8012
-
8013
-
8014
- @register_single_field_type_obj_marshaler('obj')
8015
- @dc.dataclass(frozen=True)
8016
- class JsonDeployAppConfContent(DeployAppConfContent):
8017
- obj: ta.Any
8018
-
8019
-
8020
- #
8021
-
8022
-
8023
- @register_single_field_type_obj_marshaler('sections')
8024
- @dc.dataclass(frozen=True)
8025
- class IniDeployAppConfContent(DeployAppConfContent):
8026
- sections: IniConfigSectionSettingsMap
8027
-
8028
-
8029
- #
8030
-
8031
-
8032
- @register_single_field_type_obj_marshaler('items')
8033
- @dc.dataclass(frozen=True)
8034
- class NginxDeployAppConfContent(DeployAppConfContent):
8035
- items: ta.Any
8036
-
8037
-
8038
- ##
8039
-
8040
-
8041
- @dc.dataclass(frozen=True)
8042
- class DeployAppConfFile:
8043
- path: str
8044
- content: DeployAppConfContent
8045
-
8046
- def __post_init__(self) -> None:
8047
- check_valid_deploy_spec_path(self.path)
8048
-
8049
-
8050
- ##
8051
-
8052
-
8053
- @dc.dataclass(frozen=True)
8054
- class DeployAppConfLink: # noqa
8055
- """
8056
- May be either:
8057
- - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
8058
- - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
8059
- - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
8060
- """
8061
-
8062
- src: str
8063
-
8064
- kind: ta.Literal['current_only', 'all_active'] = 'current_only'
8065
-
8066
- def __post_init__(self) -> None:
8067
- check_valid_deploy_spec_path(self.src)
8068
- if '/' in self.src:
8069
- check.equal(self.src.count('/'), 1)
8070
-
8071
-
8072
- ##
8073
-
8074
-
8075
- @dc.dataclass(frozen=True)
8076
- class DeployAppConfSpec:
8077
- files: ta.Optional[ta.Sequence[DeployAppConfFile]] = None
8397
+ command_executors: CommandExecutorMap,
8398
+ ) -> None:
8399
+ super().__init__()
8078
8400
 
8079
- links: ta.Optional[ta.Sequence[DeployAppConfLink]] = None
8401
+ self._command_executors = command_executors
8080
8402
 
8081
- def __post_init__(self) -> None:
8082
- if self.files:
8083
- seen: ta.Set[str] = set()
8084
- for f in self.files:
8085
- check.not_in(f.path, seen)
8086
- seen.add(f.path)
8403
+ async def execute(self, cmd: Command) -> Command.Output:
8404
+ ce: CommandExecutor = self._command_executors[type(cmd)]
8405
+ return await ce.execute(cmd)
8087
8406
 
8088
8407
 
8089
8408
  ########################################
@@ -8288,6 +8607,127 @@ class DeployPath(DeployPathRenderable):
8288
8607
  return cls(tuple(DeployPathPart.parse(p) for p in ps))
8289
8608
 
8290
8609
 
8610
+ ########################################
8611
+ # ../deploy/specs.py
8612
+
8613
+
8614
+ ##
8615
+
8616
+
8617
+ class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
8618
+ @cached_nullary
8619
+ def _key_str(self) -> str:
8620
+ return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
8621
+
8622
+ @abc.abstractmethod
8623
+ def key(self) -> KeyDeployTagT:
8624
+ raise NotImplementedError
8625
+
8626
+
8627
+ ##
8628
+
8629
+
8630
+ @dc.dataclass(frozen=True)
8631
+ class DeployGitRepo:
8632
+ host: ta.Optional[str] = None
8633
+ username: ta.Optional[str] = None
8634
+ path: ta.Optional[str] = None
8635
+
8636
+ def __post_init__(self) -> None:
8637
+ check.not_in('..', check.non_empty_str(self.host))
8638
+ check.not_in('.', check.non_empty_str(self.path))
8639
+
8640
+
8641
+ @dc.dataclass(frozen=True)
8642
+ class DeployGitSpec:
8643
+ repo: DeployGitRepo
8644
+ rev: DeployRev
8645
+
8646
+ subtrees: ta.Optional[ta.Sequence[str]] = None
8647
+
8648
+ def __post_init__(self) -> None:
8649
+ check.non_empty_str(self.rev)
8650
+ if self.subtrees is not None:
8651
+ for st in self.subtrees:
8652
+ check.non_empty_str(st)
8653
+
8654
+
8655
+ ##
8656
+
8657
+
8658
+ @dc.dataclass(frozen=True)
8659
+ class DeployVenvSpec:
8660
+ interp: ta.Optional[str] = None
8661
+
8662
+ requirements_files: ta.Optional[ta.Sequence[str]] = None
8663
+ extra_dependencies: ta.Optional[ta.Sequence[str]] = None
8664
+
8665
+ use_uv: bool = False
8666
+
8667
+
8668
+ ##
8669
+
8670
+
8671
+ @dc.dataclass(frozen=True)
8672
+ class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
8673
+ app: DeployApp
8674
+
8675
+ git: DeployGitSpec
8676
+
8677
+ venv: ta.Optional[DeployVenvSpec] = None
8678
+
8679
+ conf: ta.Optional[DeployAppConfSpec] = None
8680
+
8681
+ # @ta.override
8682
+ def key(self) -> DeployAppKey:
8683
+ return DeployAppKey(self._key_str())
8684
+
8685
+
8686
+ @dc.dataclass(frozen=True)
8687
+ class DeployAppLinksSpec:
8688
+ apps: ta.Sequence[DeployApp] = ()
8689
+
8690
+ removed_apps: ta.Sequence[DeployApp] = ()
8691
+
8692
+ exclude_unspecified: bool = False
8693
+
8694
+
8695
+ ##
8696
+
8697
+
8698
+ @dc.dataclass(frozen=True)
8699
+ class DeploySystemdSpec:
8700
+ # ~/.config/systemd/user/
8701
+ unit_dir: ta.Optional[str] = None
8702
+
8703
+
8704
+ ##
8705
+
8706
+
8707
+ @dc.dataclass(frozen=True)
8708
+ class DeploySpec(DeploySpecKeyed[DeployKey]):
8709
+ home: DeployHome
8710
+
8711
+ apps: ta.Sequence[DeployAppSpec] = ()
8712
+
8713
+ app_links: DeployAppLinksSpec = DeployAppLinksSpec()
8714
+
8715
+ systemd: ta.Optional[DeploySystemdSpec] = None
8716
+
8717
+ def __post_init__(self) -> None:
8718
+ check.non_empty_str(self.home)
8719
+
8720
+ seen: ta.Set[DeployApp] = set()
8721
+ for a in self.apps:
8722
+ if a.app in seen:
8723
+ raise KeyError(a.app)
8724
+ seen.add(a.app)
8725
+
8726
+ # @ta.override
8727
+ def key(self) -> DeployKey:
8728
+ return DeployKey(self._key_str())
8729
+
8730
+
8291
8731
  ########################################
8292
8732
  # ../remote/execution.py
8293
8733
  """
@@ -9320,7 +9760,7 @@ class DeployConfManager:
9320
9760
 
9321
9761
  elif isinstance(ac, IniDeployAppConfContent):
9322
9762
  ini_sections = pcc(ac.sections)
9323
- return strip_with_newline(render_ini_config(ini_sections))
9763
+ return strip_with_newline(render_ini_sections(ini_sections))
9324
9764
 
9325
9765
  elif isinstance(ac, NginxDeployAppConfContent):
9326
9766
  nginx_items = NginxConfigItems.of(pcc(ac.items))
@@ -9534,127 +9974,6 @@ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
9534
9974
  return self._owned_deploy_paths
9535
9975
 
9536
9976
 
9537
- ########################################
9538
- # ../deploy/specs.py
9539
-
9540
-
9541
- ##
9542
-
9543
-
9544
- class DeploySpecKeyed(ta.Generic[KeyDeployTagT]):
9545
- @cached_nullary
9546
- def _key_str(self) -> str:
9547
- return hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8]
9548
-
9549
- @abc.abstractmethod
9550
- def key(self) -> KeyDeployTagT:
9551
- raise NotImplementedError
9552
-
9553
-
9554
- ##
9555
-
9556
-
9557
- @dc.dataclass(frozen=True)
9558
- class DeployGitRepo:
9559
- host: ta.Optional[str] = None
9560
- username: ta.Optional[str] = None
9561
- path: ta.Optional[str] = None
9562
-
9563
- def __post_init__(self) -> None:
9564
- check.not_in('..', check.non_empty_str(self.host))
9565
- check.not_in('.', check.non_empty_str(self.path))
9566
-
9567
-
9568
- @dc.dataclass(frozen=True)
9569
- class DeployGitSpec:
9570
- repo: DeployGitRepo
9571
- rev: DeployRev
9572
-
9573
- subtrees: ta.Optional[ta.Sequence[str]] = None
9574
-
9575
- def __post_init__(self) -> None:
9576
- check.non_empty_str(self.rev)
9577
- if self.subtrees is not None:
9578
- for st in self.subtrees:
9579
- check.non_empty_str(st)
9580
-
9581
-
9582
- ##
9583
-
9584
-
9585
- @dc.dataclass(frozen=True)
9586
- class DeployVenvSpec:
9587
- interp: ta.Optional[str] = None
9588
-
9589
- requirements_files: ta.Optional[ta.Sequence[str]] = None
9590
- extra_dependencies: ta.Optional[ta.Sequence[str]] = None
9591
-
9592
- use_uv: bool = False
9593
-
9594
-
9595
- ##
9596
-
9597
-
9598
- @dc.dataclass(frozen=True)
9599
- class DeployAppSpec(DeploySpecKeyed[DeployAppKey]):
9600
- app: DeployApp
9601
-
9602
- git: DeployGitSpec
9603
-
9604
- venv: ta.Optional[DeployVenvSpec] = None
9605
-
9606
- conf: ta.Optional[DeployAppConfSpec] = None
9607
-
9608
- # @ta.override
9609
- def key(self) -> DeployAppKey:
9610
- return DeployAppKey(self._key_str())
9611
-
9612
-
9613
- @dc.dataclass(frozen=True)
9614
- class DeployAppLinksSpec:
9615
- apps: ta.Sequence[DeployApp] = ()
9616
-
9617
- removed_apps: ta.Sequence[DeployApp] = ()
9618
-
9619
- exclude_unspecified: bool = False
9620
-
9621
-
9622
- ##
9623
-
9624
-
9625
- @dc.dataclass(frozen=True)
9626
- class DeploySystemdSpec:
9627
- # ~/.config/systemd/user/
9628
- unit_dir: ta.Optional[str] = None
9629
-
9630
-
9631
- ##
9632
-
9633
-
9634
- @dc.dataclass(frozen=True)
9635
- class DeploySpec(DeploySpecKeyed[DeployKey]):
9636
- home: DeployHome
9637
-
9638
- apps: ta.Sequence[DeployAppSpec] = ()
9639
-
9640
- app_links: DeployAppLinksSpec = DeployAppLinksSpec()
9641
-
9642
- systemd: ta.Optional[DeploySystemdSpec] = None
9643
-
9644
- def __post_init__(self) -> None:
9645
- check.non_empty_str(self.home)
9646
-
9647
- seen: ta.Set[DeployApp] = set()
9648
- for a in self.apps:
9649
- if a.app in seen:
9650
- raise KeyError(a.app)
9651
- seen.add(a.app)
9652
-
9653
- # @ta.override
9654
- def key(self) -> DeployKey:
9655
- return DeployKey(self._key_str())
9656
-
9657
-
9658
9977
  ########################################
9659
9978
  # ../remote/_main.py
9660
9979
 
@@ -12294,7 +12613,7 @@ class MainCli(ArgparseCli):
12294
12613
  if cf is None:
12295
12614
  return ManageConfig()
12296
12615
  else:
12297
- return read_config_file(cf, ManageConfig)
12616
+ return load_config_file_obj(cf, ManageConfig)
12298
12617
 
12299
12618
  #
12300
12619
 
@@ -12365,7 +12684,7 @@ class MainCli(ArgparseCli):
12365
12684
  cmds.append(cmd)
12366
12685
 
12367
12686
  for cf in self.args.command_file or []:
12368
- cmd = read_config_file(cf, Command, msh=msh)
12687
+ cmd = load_config_file_obj(cf, Command, msh=msh)
12369
12688
  cmds.append(cmd)
12370
12689
 
12371
12690
  #