ominfra 0.0.0.dev120__py3-none-any.whl → 0.0.0.dev122__py3-none-any.whl

ominfra/scripts/journald2aws.py

@@ -25,12 +25,15 @@ import logging
 import os
 import os.path
 import queue
+import re
 import shlex
 import signal
+import string
 import subprocess
 import sys
 import threading
 import time
+import types
 import typing as ta
 import urllib.parse
 import urllib.request
@@ -49,16 +52,842 @@ if sys.version_info < (3, 8):
 ########################################
 
 
+# ../../../../../omdev/toml/parser.py
+TomlParseFloat = ta.Callable[[str], ta.Any]
+TomlKey = ta.Tuple[str, ...]
+TomlPos = int  # ta.TypeAlias
+
 # ../../../../../omlish/lite/cached.py
 T = ta.TypeVar('T')
 
 # ../../../../../omlish/lite/contextmanagers.py
 ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
 
+# ../../../../configs.py
+ConfigMapping = ta.Mapping[str, ta.Any]
+
 # ../../../../threadworkers.py
 ThreadWorkerT = ta.TypeVar('ThreadWorkerT', bound='ThreadWorker')
 
 
+########################################
+# ../../../../../omdev/toml/parser.py
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
+# Licensed to PSF under a Contributor Agreement.
+#
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
+# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
+# documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
+# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
+# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
+# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
+# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
+# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
+# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
+# any such work a brief summary of the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS" basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
+# EXPRESS OR IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
+# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
+# RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
+# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
+# ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
+# venture between PSF and Licensee.  This License Agreement does not grant permission to use PSF trademarks or trade
+# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
+# License Agreement.
+#
+# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
+
+
+##
+
+
+_TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
+
+TOML_RE_NUMBER = re.compile(
+    r"""
+0
+(?:
+    x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*   # hex
+    |
+    b[01](?:_?[01])*                 # bin
+    |
+    o[0-7](?:_?[0-7])*               # oct
+)
+|
+[+-]?(?:0|[1-9](?:_?[0-9])*)         # dec, integer part
+(?P<floatpart>
+    (?:\.[0-9](?:_?[0-9])*)?         # optional fractional part
+    (?:[eE][+-]?[0-9](?:_?[0-9])*)?  # optional exponent part
+)
+""",
+    flags=re.VERBOSE,
+)
+TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
+TOML_RE_DATETIME = re.compile(
+    rf"""
+([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])  # date, e.g. 1988-10-27
+(?:
+    [Tt ]
+    {_TOML_TIME_RE_STR}
+    (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))?  # optional time offset
+)?
+""",
+    flags=re.VERBOSE,
+)
+
+
+def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
+    """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
+
+    Raises ValueError if the match does not correspond to a valid date or datetime.
+    """
+    (
+        year_str,
+        month_str,
+        day_str,
+        hour_str,
+        minute_str,
+        sec_str,
+        micros_str,
+        zulu_time,
+        offset_sign_str,
+        offset_hour_str,
+        offset_minute_str,
+    ) = match.groups()
+    year, month, day = int(year_str), int(month_str), int(day_str)
+    if hour_str is None:
+        return datetime.date(year, month, day)
+    hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
+    micros = int(micros_str.ljust(6, '0')) if micros_str else 0
+    if offset_sign_str:
+        tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
+            offset_hour_str, offset_minute_str, offset_sign_str,
+        )
+    elif zulu_time:
+        tz = datetime.UTC
+    else:  # local date-time
+        tz = None
+    return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
+
+
+@functools.lru_cache()  # noqa
+def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
+    sign = 1 if sign_str == '+' else -1
+    return datetime.timezone(
+        datetime.timedelta(
+            hours=sign * int(hour_str),
+            minutes=sign * int(minute_str),
+        ),
+    )
+
+
+def toml_match_to_localtime(match: re.Match) -> datetime.time:
+    hour_str, minute_str, sec_str, micros_str = match.groups()
+    micros = int(micros_str.ljust(6, '0')) if micros_str else 0
+    return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
+
+
+def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
+    if match.group('floatpart'):
+        return parse_float(match.group())
+    return int(match.group(), 0)
+
+
+TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
+
+# Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
+# functions.
+TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
+TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
+
+TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
+TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
+
+TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
+
+TOML_WS = frozenset(' \t')
+TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
+TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
+TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
+TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
+
+TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
+    {
+        '\\b': '\u0008',  # backspace
+        '\\t': '\u0009',  # tab
+        '\\n': '\u000A',  # linefeed
+        '\\f': '\u000C',  # form feed
+        '\\r': '\u000D',  # carriage return
+        '\\"': '\u0022',  # quote
+        '\\\\': '\u005C',  # backslash
+    },
+)
+
+
+class TomlDecodeError(ValueError):
+    """An error raised if a document is not valid TOML."""
+
+
+def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
+    """Parse TOML from a binary file object."""
+    b = fp.read()
+    try:
+        s = b.decode()
+    except AttributeError:
+        raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
+    return toml_loads(s, parse_float=parse_float)
+
+
+def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:  # noqa: C901
+    """Parse TOML from a string."""
+
+    # The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
+    try:
+        src = s.replace('\r\n', '\n')
+    except (AttributeError, TypeError):
+        raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
+    pos = 0
+    out = TomlOutput(TomlNestedDict(), TomlFlags())
+    header: TomlKey = ()
+    parse_float = toml_make_safe_parse_float(parse_float)
+
+    # Parse one statement at a time (typically means one line in TOML source)
+    while True:
+        # 1. Skip line leading whitespace
+        pos = toml_skip_chars(src, pos, TOML_WS)
+
+        # 2. Parse rules. Expect one of the following:
+        #    - end of file
+        #    - end of line
+        #    - comment
+        #    - key/value pair
+        #    - append dict to list (and move to its namespace)
+        #    - create dict (and move to its namespace)
+        # Skip trailing whitespace when applicable.
+        try:
+            char = src[pos]
+        except IndexError:
+            break
+        if char == '\n':
+            pos += 1
+            continue
+        if char in TOML_KEY_INITIAL_CHARS:
+            pos = toml_key_value_rule(src, pos, out, header, parse_float)
+            pos = toml_skip_chars(src, pos, TOML_WS)
+        elif char == '[':
+            try:
+                second_char: ta.Optional[str] = src[pos + 1]
+            except IndexError:
+                second_char = None
+            out.flags.finalize_pending()
+            if second_char == '[':
+                pos, header = toml_create_list_rule(src, pos, out)
+            else:
+                pos, header = toml_create_dict_rule(src, pos, out)
+            pos = toml_skip_chars(src, pos, TOML_WS)
+        elif char != '#':
+            raise toml_suffixed_err(src, pos, 'Invalid statement')
+
+        # 3. Skip comment
+        pos = toml_skip_comment(src, pos)
+
+        # 4. Expect end of line or end of file
+        try:
+            char = src[pos]
+        except IndexError:
+            break
+        if char != '\n':
+            raise toml_suffixed_err(
+                src, pos, 'Expected newline or end of document after a statement',
+            )
+        pos += 1
+
+    return out.data.dict
+
+
+class TomlFlags:
+    """Flags that map to parsed keys/namespaces."""
+
+    # Marks an immutable namespace (inline array or inline table).
+    FROZEN = 0
+    # Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
+    EXPLICIT_NEST = 1
+
+    def __init__(self) -> None:
+        self._flags: ta.Dict[str, dict] = {}
+        self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
+
+    def add_pending(self, key: TomlKey, flag: int) -> None:
+        self._pending_flags.add((key, flag))
+
+    def finalize_pending(self) -> None:
+        for key, flag in self._pending_flags:
+            self.set(key, flag, recursive=False)
+        self._pending_flags.clear()
+
+    def unset_all(self, key: TomlKey) -> None:
+        cont = self._flags
+        for k in key[:-1]:
+            if k not in cont:
+                return
+            cont = cont[k]['nested']
+        cont.pop(key[-1], None)
+
+    def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None:  # noqa: A003
+        cont = self._flags
+        key_parent, key_stem = key[:-1], key[-1]
+        for k in key_parent:
+            if k not in cont:
+                cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
+            cont = cont[k]['nested']
+        if key_stem not in cont:
+            cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
+        cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
+
+    def is_(self, key: TomlKey, flag: int) -> bool:
+        if not key:
+            return False  # document root has no flags
+        cont = self._flags
+        for k in key[:-1]:
+            if k not in cont:
+                return False
+            inner_cont = cont[k]
+            if flag in inner_cont['recursive_flags']:
+                return True
+            cont = inner_cont['nested']
+        key_stem = key[-1]
+        if key_stem in cont:
+            cont = cont[key_stem]
+            return flag in cont['flags'] or flag in cont['recursive_flags']
+        return False
+
+
+class TomlNestedDict:
+    def __init__(self) -> None:
+        # The parsed content of the TOML document
+        self.dict: ta.Dict[str, ta.Any] = {}
+
+    def get_or_create_nest(
+            self,
+            key: TomlKey,
+            *,
+            access_lists: bool = True,
+    ) -> dict:
+        cont: ta.Any = self.dict
+        for k in key:
+            if k not in cont:
+                cont[k] = {}
+            cont = cont[k]
+            if access_lists and isinstance(cont, list):
+                cont = cont[-1]
+            if not isinstance(cont, dict):
+                raise KeyError('There is no nest behind this key')
+        return cont
+
+    def append_nest_to_list(self, key: TomlKey) -> None:
+        cont = self.get_or_create_nest(key[:-1])
+        last_key = key[-1]
+        if last_key in cont:
+            list_ = cont[last_key]
+            if not isinstance(list_, list):
+                raise KeyError('An object other than list found behind this key')
+            list_.append({})
+        else:
+            cont[last_key] = [{}]
+
+
+class TomlOutput(ta.NamedTuple):
+    data: TomlNestedDict
+    flags: TomlFlags
+
+
+def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
+    try:
+        while src[pos] in chars:
+            pos += 1
+    except IndexError:
+        pass
+    return pos
+
+
+def toml_skip_until(
+        src: str,
+        pos: TomlPos,
+        expect: str,
+        *,
+        error_on: ta.FrozenSet[str],
+        error_on_eof: bool,
+) -> TomlPos:
+    try:
+        new_pos = src.index(expect, pos)
+    except ValueError:
+        new_pos = len(src)
+        if error_on_eof:
+            raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
+
+    if not error_on.isdisjoint(src[pos:new_pos]):
+        while src[pos] not in error_on:
+            pos += 1
+        raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
+    return new_pos
+
+
+def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
+    try:
+        char: ta.Optional[str] = src[pos]
+    except IndexError:
+        char = None
+    if char == '#':
+        return toml_skip_until(
+            src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
+        )
+    return pos
+
+
+def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
+    while True:
+        pos_before_skip = pos
+        pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
+        pos = toml_skip_comment(src, pos)
+        if pos == pos_before_skip:
+            return pos
+
+
+def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
+    pos += 1  # Skip "["
+    pos = toml_skip_chars(src, pos, TOML_WS)
+    pos, key = toml_parse_key(src, pos)
+
+    if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
+        raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
+    out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
+    try:
+        out.data.get_or_create_nest(key)
+    except KeyError:
+        raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
+
+    if not src.startswith(']', pos):
+        raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
+    return pos + 1, key
+
+
+def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
+    pos += 2  # Skip "[["
+    pos = toml_skip_chars(src, pos, TOML_WS)
+    pos, key = toml_parse_key(src, pos)
+
+    if out.flags.is_(key, TomlFlags.FROZEN):
+        raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
+    # Free the namespace now that it points to another empty list item...
+    out.flags.unset_all(key)
+    # ...but this key precisely is still prohibited from table declaration
+    out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
+    try:
+        out.data.append_nest_to_list(key)
+    except KeyError:
+        raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
+
+    if not src.startswith(']]', pos):
+        raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
+    return pos + 2, key
+
+
+def toml_key_value_rule(
+        src: str,
+        pos: TomlPos,
+        out: TomlOutput,
+        header: TomlKey,
+        parse_float: TomlParseFloat,
+) -> TomlPos:
+    pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
+    key_parent, key_stem = key[:-1], key[-1]
+    abs_key_parent = header + key_parent
+
+    relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
+    for cont_key in relative_path_cont_keys:
+        # Check that dotted key syntax does not redefine an existing table
+        if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
+            raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
+        # Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
+        # table sections.
+        out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
+
+    if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
+        raise toml_suffixed_err(
+            src,
+            pos,
+            f'Cannot mutate immutable namespace {abs_key_parent}',
+        )
+
+    try:
+        nest = out.data.get_or_create_nest(abs_key_parent)
+    except KeyError:
+        raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
+    if key_stem in nest:
+        raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
+    # Mark inline table and array namespaces recursively immutable
+    if isinstance(value, (dict, list)):
+        out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
+    nest[key_stem] = value
+    return pos
+
+
+def toml_parse_key_value_pair(
+        src: str,
+        pos: TomlPos,
+        parse_float: TomlParseFloat,
+) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
+    pos, key = toml_parse_key(src, pos)
+    try:
+        char: ta.Optional[str] = src[pos]
+    except IndexError:
+        char = None
+    if char != '=':
+        raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
+    pos += 1
+    pos = toml_skip_chars(src, pos, TOML_WS)
+    pos, value = toml_parse_value(src, pos, parse_float)
+    return pos, key, value
+
+
+def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
+    pos, key_part = toml_parse_key_part(src, pos)
+    key: TomlKey = (key_part,)
+    pos = toml_skip_chars(src, pos, TOML_WS)
+    while True:
+        try:
+            char: ta.Optional[str] = src[pos]
+        except IndexError:
+            char = None
+        if char != '.':
+            return pos, key
+        pos += 1
+        pos = toml_skip_chars(src, pos, TOML_WS)
+        pos, key_part = toml_parse_key_part(src, pos)
+        key += (key_part,)
+        pos = toml_skip_chars(src, pos, TOML_WS)
+
+
+def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
+    try:
+        char: ta.Optional[str] = src[pos]
+    except IndexError:
+        char = None
+    if char in TOML_BARE_KEY_CHARS:
+        start_pos = pos
+        pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
+        return pos, src[start_pos:pos]
+    if char == "'":
+        return toml_parse_literal_str(src, pos)
+    if char == '"':
+        return toml_parse_one_line_basic_str(src, pos)
+    raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
+
+
+def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
+    pos += 1
+    return toml_parse_basic_str(src, pos, multiline=False)
+
+
+def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
+    pos += 1
+    array: list = []
+
+    pos = toml_skip_comments_and_array_ws(src, pos)
+    if src.startswith(']', pos):
+        return pos + 1, array
+    while True:
+        pos, val = toml_parse_value(src, pos, parse_float)
+        array.append(val)
+        pos = toml_skip_comments_and_array_ws(src, pos)
+
+        c = src[pos:pos + 1]
+        if c == ']':
+            return pos + 1, array
+        if c != ',':
+            raise toml_suffixed_err(src, pos, 'Unclosed array')
+        pos += 1
+
+        pos = toml_skip_comments_and_array_ws(src, pos)
+        if src.startswith(']', pos):
+            return pos + 1, array
+
+
+def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
+    pos += 1
+    nested_dict = TomlNestedDict()
+    flags = TomlFlags()
+
+    pos = toml_skip_chars(src, pos, TOML_WS)
+    if src.startswith('}', pos):
+        return pos + 1, nested_dict.dict
+    while True:
+        pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
+        key_parent, key_stem = key[:-1], key[-1]
+        if flags.is_(key, TomlFlags.FROZEN):
+            raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
+        try:
+            nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
+        except KeyError:
+            raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
+        if key_stem in nest:
+            raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
+        nest[key_stem] = value
+        pos = toml_skip_chars(src, pos, TOML_WS)
+        c = src[pos:pos + 1]
+        if c == '}':
+            return pos + 1, nested_dict.dict
+        if c != ',':
+            raise toml_suffixed_err(src, pos, 'Unclosed inline table')
+        if isinstance(value, (dict, list)):
+            flags.set(key, TomlFlags.FROZEN, recursive=True)
+        pos += 1
+        pos = toml_skip_chars(src, pos, TOML_WS)
+
+
+def toml_parse_basic_str_escape(
+        src: str,
+        pos: TomlPos,
+        *,
+        multiline: bool = False,
+) -> ta.Tuple[TomlPos, str]:
+    escape_id = src[pos:pos + 2]
+    pos += 2
+    if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
+        # Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
+        # newline.
+        if escape_id != '\\\n':
+            pos = toml_skip_chars(src, pos, TOML_WS)
+            try:
+                char = src[pos]
+            except IndexError:
+                return pos, ''
+            if char != '\n':
+                raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
+            pos += 1
+        pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
+        return pos, ''
+    if escape_id == '\\u':
+        return toml_parse_hex_char(src, pos, 4)
+    if escape_id == '\\U':
+        return toml_parse_hex_char(src, pos, 8)
+    try:
+        return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
+    except KeyError:
+        raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
+
+
+def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
+    return toml_parse_basic_str_escape(src, pos, multiline=True)
+
+
+def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
+    hex_str = src[pos:pos + hex_len]
+    if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
+        raise toml_suffixed_err(src, pos, 'Invalid hex value')
+    pos += hex_len
+    hex_int = int(hex_str, 16)
+    if not toml_is_unicode_scalar_value(hex_int):
+        raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
+    return pos, chr(hex_int)
+
+
+def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
+    pos += 1  # Skip starting apostrophe
+    start_pos = pos
+    pos = toml_skip_until(
+        src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
+    )
+    return pos + 1, src[start_pos:pos]  # Skip ending apostrophe
+
+
+def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
+    pos += 3
+    if src.startswith('\n', pos):
+        pos += 1
+
+    if literal:
+        delim = "'"
+        end_pos = toml_skip_until(
+            src,
+            pos,
+            "'''",
+            error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
+            error_on_eof=True,
+        )
+        result = src[pos:end_pos]
+        pos = end_pos + 3
+    else:
+        delim = '"'
+        pos, result = toml_parse_basic_str(src, pos, multiline=True)
+
+    # Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
+    if not src.startswith(delim, pos):
+        return pos, result
+    pos += 1
+    if not src.startswith(delim, pos):
+        return pos, result + delim
+    pos += 1
+    return pos, result + (delim * 2)
+
+
+def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
+    if multiline:
+        error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
+        parse_escapes = toml_parse_basic_str_escape_multiline
+    else:
+        error_on = TOML_ILLEGAL_BASIC_STR_CHARS
+        parse_escapes = toml_parse_basic_str_escape
+    result = ''
+    start_pos = pos
+    while True:
+        try:
+            char = src[pos]
+        except IndexError:
+            raise toml_suffixed_err(src, pos, 'Unterminated string') from None
+        if char == '"':
+            if not multiline:
+                return pos + 1, result + src[start_pos:pos]
+            if src.startswith('"""', pos):
+                return pos + 3, result + src[start_pos:pos]
+            pos += 1
+            continue
+        if char == '\\':
+            result += src[start_pos:pos]
+            pos, parsed_escape = parse_escapes(src, pos)
+            result += parsed_escape
+            start_pos = pos
+            continue
+        if char in error_on:
+            raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
+        pos += 1
+
+
+def toml_parse_value(  # noqa: C901
+        src: str,
+        pos: TomlPos,
+        parse_float: TomlParseFloat,
+) -> ta.Tuple[TomlPos, ta.Any]:
+    try:
+        char: ta.Optional[str] = src[pos]
+    except IndexError:
+        char = None
+
+    # IMPORTANT: order conditions based on speed of checking and likelihood
+
+    # Basic strings
+    if char == '"':
+        if src.startswith('"""', pos):
+            return toml_parse_multiline_str(src, pos, literal=False)
+        return toml_parse_one_line_basic_str(src, pos)
+
+    # Literal strings
+    if char == "'":
+        if src.startswith("'''", pos):
+            return toml_parse_multiline_str(src, pos, literal=True)
+        return toml_parse_literal_str(src, pos)
+
+    # Booleans
+    if char == 't':
+        if src.startswith('true', pos):
+            return pos + 4, True
+    if char == 'f':
+        if src.startswith('false', pos):
+            return pos + 5, False
+
+    # Arrays
+    if char == '[':
+        return toml_parse_array(src, pos, parse_float)
+
+    # Inline tables
+    if char == '{':
+        return toml_parse_inline_table(src, pos, parse_float)
+
+    # Dates and times
+    datetime_match = TOML_RE_DATETIME.match(src, pos)
+    if datetime_match:
+        try:
+            datetime_obj = toml_match_to_datetime(datetime_match)
+        except ValueError as e:
+            raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
+        return datetime_match.end(), datetime_obj
+    localtime_match = TOML_RE_LOCALTIME.match(src, pos)
+    if localtime_match:
+        return localtime_match.end(), toml_match_to_localtime(localtime_match)
+
+    # Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
+    # located after handling of dates and times.
+    number_match = TOML_RE_NUMBER.match(src, pos)
+    if number_match:
+        return number_match.end(), toml_match_to_number(number_match, parse_float)
+
+    # Special floats
+    first_three = src[pos:pos + 3]
+    if first_three in {'inf', 'nan'}:
+        return pos + 3, parse_float(first_three)
+    first_four = src[pos:pos + 4]
+    if first_four in {'-inf', '+inf', '-nan', '+nan'}:
+        return pos + 4, parse_float(first_four)
+
+    raise toml_suffixed_err(src, pos, 'Invalid value')
+
+
+def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
+    """Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
+
+    def coord_repr(src: str, pos: TomlPos) -> str:
+        if pos >= len(src):
+            return 'end of document'
+        line = src.count('\n', 0, pos) + 1
+        if line == 1:
+            column = pos + 1
+        else:
+            column = pos - src.rindex('\n', 0, pos)
+        return f'line {line}, column {column}'
+
+    return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
+
+
+def toml_is_unicode_scalar_value(codepoint: int) -> bool:
+    return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
+
+
+def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
+    """A decorator to make `parse_float` safe.
+
+    `parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
+    thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
+    """
+    # The default `float` callable never returns illegal types. Optimize it.
+    if parse_float is float:
+        return float
+
+    def safe_parse_float(float_str: str) -> ta.Any:
+        float_value = parse_float(float_str)
+        if isinstance(float_value, (dict, list)):
+            raise ValueError('parse_float must not return dicts or lists')  # noqa
+        return float_value
+
+    return safe_parse_float
+
+
 ########################################
 # ../../../../../omlish/lite/cached.py
 
@@ -80,7 +909,7 @@ class _cached_nullary:  # noqa
         return bound
 
 
-def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
+def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
     return _cached_nullary(fn)
 
 
@@ -273,6 +1102,14 @@ def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
     return it
 
 
+def is_new_type(spec: ta.Any) -> bool:
+    if isinstance(ta.NewType, type):
+        return isinstance(spec, ta.NewType)
+    else:
+        # Before https://github.com/python/cpython/commit/c2f33dfc83ab270412bf243fb21f724037effa1a
+        return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1]  # type: ignore  # noqa
+
+
 def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
     seen = set()
     todo = list(reversed(cls.__subclasses__()))
@@ -733,8 +1570,12 @@ class ExitStacked:
     def __exit__(self, exc_type, exc_val, exc_tb):
         if (es := self._exit_stack) is None:
             return None
+        self._exit_contexts()
         return es.__exit__(exc_type, exc_val, exc_tb)
 
+    def _exit_contexts(self) -> None:
+        pass
+
     def _enter_context(self, cm: ta.ContextManager[T]) -> T:
         es = check_not_none(self._exit_stack)
         return es.enter_context(cm)
@@ -1734,6 +2575,66 @@ class AwsLogMessageBuilder:
         return [post]
 
 
+########################################
+# ../../../../configs.py
+
+
+def read_config_file(
+        path: str,
+        cls: ta.Type[T],
+        *,
+        prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
+) -> T:
+    with open(path) as cf:
+        if path.endswith('.toml'):
+            config_dct = toml_loads(cf.read())
+        else:
+            config_dct = json.loads(cf.read())
+
+    if prepare is not None:
+        config_dct = prepare(config_dct)  # type: ignore
+
+    return unmarshal_obj(config_dct, cls)
+
+
+def build_config_named_children(
+        o: ta.Union[
+            ta.Sequence[ConfigMapping],
+            ta.Mapping[str, ConfigMapping],
+            None,
+        ],
+        *,
+        name_key: str = 'name',
+) -> ta.Optional[ta.Sequence[ConfigMapping]]:
+    if o is None:
+        return None
+
+    lst: ta.List[ConfigMapping] = []
+    if isinstance(o, ta.Mapping):
+        for k, v in o.items():
+            check_isinstance(v, ta.Mapping)
+            if name_key in v:
+                n = v[name_key]
+                if k != n:
+                    raise KeyError(f'Given names do not match: {n} != {k}')
+                lst.append(v)
+            else:
+                lst.append({name_key: k, **v})
+
+    else:
+        check_not_isinstance(o, str)
+        lst.extend(o)
+
+    seen = set()
+    for d in lst:
+        n = d['name']
+        if n in d:
+            raise KeyError(f'Duplicate name: {n}')
+        seen.add(n)
+
+    return lst
+
+
 ########################################
 # ../../../../journald/messages.py
 
@@ -1810,9 +2711,11 @@ class JournalctlMessageBuilder:
 ########################################
 # ../../../../threadworkers.py
 """
+FIXME:
+ - group is racy af - meditate on has_started, etc
+
 TODO:
- - implement stop lol
- - collective heartbeat monitoring - ThreadWorkerGroups
+ - overhaul stop lol
  - group -> 'context'? :|
   - shared stop_event?
 """
@@ -1826,6 +2729,7 @@ class ThreadWorker(ExitStacked, abc.ABC):
             self,
             *,
             stop_event: ta.Optional[threading.Event] = None,
+            worker_groups: ta.Optional[ta.Iterable['ThreadWorkerGroup']] = None,
     ) -> None:
         super().__init__()
 
@@ -1837,6 +2741,9 @@ class ThreadWorker(ExitStacked, abc.ABC):
         self._thread: ta.Optional[threading.Thread] = None
         self._last_heartbeat: ta.Optional[float] = None
 
+        for g in worker_groups or []:
+            g.add(self)
+
     #
 
     def __enter__(self: ThreadWorkerT) -> ThreadWorkerT:
@@ -1881,13 +2788,13 @@ class ThreadWorker(ExitStacked, abc.ABC):
             if self._thread is not None:
                 raise RuntimeError('Thread already started: %r', self)
 
-            thr = threading.Thread(target=self.__run)
+            thr = threading.Thread(target=self.__thread_main)
             self._thread = thr
             thr.start()
 
     #
 
-    def __run(self) -> None:
+    def __thread_main(self) -> None:
         try:
             self._run()
         except ThreadWorker.Stopping:
@@ -1905,10 +2812,17 @@ class ThreadWorker(ExitStacked, abc.ABC):
     def stop(self) -> None:
         self._stop_event.set()
 
-    def join(self, timeout: ta.Optional[float] = None) -> None:
+    def join(
+            self,
+            timeout: ta.Optional[float] = None,
+            *,
+            unless_not_started: bool = False,
+    ) -> None:
         with self._lock:
             if self._thread is None:
-                raise RuntimeError('Thread not started: %r', self)
+                if not unless_not_started:
+                    raise RuntimeError('Thread not started: %r', self)
+                return
             self._thread.join(timeout)
 
 
@@ -1917,24 +2831,68 @@ class ThreadWorker(ExitStacked, abc.ABC):
 
 class ThreadWorkerGroup:
     @dc.dataclass()
-    class State:
+    class _State:
         worker: ThreadWorker
 
+        last_heartbeat: ta.Optional[float] = None
+
     def __init__(self) -> None:
         super().__init__()
 
         self._lock = threading.RLock()
-        self._states: ta.Dict[ThreadWorker, ThreadWorkerGroup.State] = {}
+        self._states: ta.Dict[ThreadWorker, ThreadWorkerGroup._State] = {}
+        self._last_heartbeat_check: ta.Optional[float] = None
+
+    #
 
     def add(self, *workers: ThreadWorker) -> 'ThreadWorkerGroup':
         with self._lock:
             for w in workers:
                 if w in self._states:
                     raise KeyError(w)
-                self._states[w] = ThreadWorkerGroup.State(w)
+                self._states[w] = ThreadWorkerGroup._State(w)
 
         return self
 
+    #
+
+    def start_all(self) -> None:
+        thrs = list(self._states)
+        with self._lock:
+            for thr in thrs:
+                if not thr.has_started():
+                    thr.start()
+
+    def stop_all(self) -> None:
+        for w in reversed(list(self._states)):
+            if w.has_started():
+                w.stop()
+
+    def join_all(self, timeout: ta.Optional[float] = None) -> None:
+        for w in reversed(list(self._states)):
+            if w.has_started():
+                w.join(timeout, unless_not_started=True)
+
+    #
+
+    def get_dead(self) -> ta.List[ThreadWorker]:
+        with self._lock:
+            return [thr for thr in self._states if not thr.is_alive()]
+
+    def check_heartbeats(self) -> ta.Dict[ThreadWorker, float]:
+        with self._lock:
+            dct: ta.Dict[ThreadWorker, float] = {}
+            for thr, st in self._states.items():
+                if not thr.has_started():
+                    continue
+                hb = thr.last_heartbeat
+                if hb is None:
+                    hb = time.time()
+                st.last_heartbeat = hb
+                dct[st.worker] = time.time() - hb
+            self._last_heartbeat_check = time.time()
+        return dct
+
 
 ########################################
 # ../../../../../omlish/lite/subprocesses.py
@@ -2694,6 +3652,7 @@ class JournalctlToAwsDriver(ExitStacked):
         cursor_file: ta.Optional[str] = None
 
         runtime_limit: ta.Optional[float] = None
+        heartbeat_age_limit: ta.Optional[float] = 60.
 
         #
 
@@ -2770,6 +3729,12 @@ class JournalctlToAwsDriver(ExitStacked):
 
     #
 
+    @cached_nullary
+    def _worker_group(self) -> ThreadWorkerGroup:
+        return ThreadWorkerGroup()
+
+    #
+
     @cached_nullary
     def _journalctl_message_queue(self):  # type: () -> queue.Queue[ta.Sequence[JournalctlMessage]]
         return queue.Queue()
@@ -2796,6 +3761,8 @@ class JournalctlToAwsDriver(ExitStacked):
 
             cmd=self._config.journalctl_cmd,
             shell_wrap=is_debugger_attached(),
+
+            worker_groups=[self._worker_group()],
         )
 
     #
@@ -2809,26 +3776,38 @@ class JournalctlToAwsDriver(ExitStacked):
 
             ensure_locked=self._ensure_locked,
             dry_run=self._config.aws_dry_run,
+
+            worker_groups=[self._worker_group()],
         )
 
     #
 
-    def run(self) -> None:
-        pw: JournalctlToAwsPosterWorker = self._aws_poster_worker()
-        tw: JournalctlTailerWorker = self._journalctl_tailer_worker()
+    def _exit_contexts(self) -> None:
+        wg = self._worker_group()
+        wg.stop_all()
+        wg.join_all()
 
-        ws = [pw, tw]
+    def run(self) -> None:
+        self._aws_poster_worker()
+        self._journalctl_tailer_worker()
 
-        for w in ws:
-            w.start()
+        wg = self._worker_group()
+        wg.start_all()
 
         start = time.time()
 
         while True:
-            for w in ws:
-                if not w.is_alive():
-                    log.critical('Worker died: %r', w)
-                    break
+            for w in wg.get_dead():
+                log.critical('Worker died: %r', w)
+                break
+
+            if (al := self._config.heartbeat_age_limit) is not None:
+                hbs = wg.check_heartbeats()
+                log.debug('Worker heartbeats: %r', hbs)
+                for w, age in hbs.items():
+                    if age > al:
+                        log.critical('Worker heartbeat age limit exceeded: %r %f > %f', w, age, al)
+                        break
 
             if (rl := self._config.runtime_limit) is not None and time.time() - start >= rl:
                 log.warning('Runtime limit reached')
@@ -2836,9 +3815,8 @@ class JournalctlToAwsDriver(ExitStacked):
 
             time.sleep(1.)
 
-        for w in reversed(ws):
-            w.stop()
-            w.join()
+        wg.stop_all()
+        wg.join_all()
 
 
 ########################################
@@ -2870,9 +3848,7 @@ def _main() -> None:
 
     config: JournalctlToAwsDriver.Config
     if args.config_file:
-        with open(os.path.expanduser(args.config_file)) as cf:
-            config_dct = json.load(cf)
-        config = unmarshal_obj(config_dct, JournalctlToAwsDriver.Config)
+        config = read_config_file(os.path.expanduser(args.config_file), JournalctlToAwsDriver.Config)
     else:
         config = JournalctlToAwsDriver.Config()