ominfra 0.0.0.dev119__py3-none-any.whl → 0.0.0.dev121__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/__main__.py +4 -0
- ominfra/clouds/aws/journald2aws/driver.py +34 -13
- ominfra/clouds/aws/journald2aws/main.py +2 -5
- ominfra/configs.py +70 -0
- ominfra/deploy/_executor.py +1 -1
- ominfra/deploy/poly/_main.py +1 -1
- ominfra/pyremote/_runcommands.py +1 -1
- ominfra/scripts/journald2aws.py +994 -26
- ominfra/scripts/supervisor.py +1969 -262
- ominfra/supervisor/compat.py +13 -0
- ominfra/supervisor/configs.py +21 -0
- ominfra/supervisor/context.py +13 -2
- ominfra/supervisor/dispatchers.py +4 -4
- ominfra/supervisor/events.py +4 -7
- ominfra/supervisor/main.py +82 -11
- ominfra/supervisor/process.py +46 -10
- ominfra/supervisor/supervisor.py +118 -88
- ominfra/supervisor/types.py +5 -0
- ominfra/threadworkers.py +66 -9
- {ominfra-0.0.0.dev119.dist-info → ominfra-0.0.0.dev121.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev119.dist-info → ominfra-0.0.0.dev121.dist-info}/RECORD +25 -23
- {ominfra-0.0.0.dev119.dist-info → ominfra-0.0.0.dev121.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev119.dist-info → ominfra-0.0.0.dev121.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev119.dist-info → ominfra-0.0.0.dev121.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev119.dist-info → ominfra-0.0.0.dev121.dist-info}/top_level.txt +0 -0
ominfra/scripts/journald2aws.py
CHANGED
@@ -25,12 +25,15 @@ import logging
|
|
25
25
|
import os
|
26
26
|
import os.path
|
27
27
|
import queue
|
28
|
+
import re
|
28
29
|
import shlex
|
29
30
|
import signal
|
31
|
+
import string
|
30
32
|
import subprocess
|
31
33
|
import sys
|
32
34
|
import threading
|
33
35
|
import time
|
36
|
+
import types
|
34
37
|
import typing as ta
|
35
38
|
import urllib.parse
|
36
39
|
import urllib.request
|
@@ -49,16 +52,842 @@ if sys.version_info < (3, 8):
|
|
49
52
|
########################################
|
50
53
|
|
51
54
|
|
55
|
+
# ../../../../../omdev/toml/parser.py
|
56
|
+
TomlParseFloat = ta.Callable[[str], ta.Any]
|
57
|
+
TomlKey = ta.Tuple[str, ...]
|
58
|
+
TomlPos = int # ta.TypeAlias
|
59
|
+
|
52
60
|
# ../../../../../omlish/lite/cached.py
|
53
61
|
T = ta.TypeVar('T')
|
54
62
|
|
55
63
|
# ../../../../../omlish/lite/contextmanagers.py
|
56
64
|
ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
|
57
65
|
|
66
|
+
# ../../../../configs.py
|
67
|
+
ConfigMapping = ta.Mapping[str, ta.Any]
|
68
|
+
|
58
69
|
# ../../../../threadworkers.py
|
59
70
|
ThreadWorkerT = ta.TypeVar('ThreadWorkerT', bound='ThreadWorker')
|
60
71
|
|
61
72
|
|
73
|
+
########################################
|
74
|
+
# ../../../../../omdev/toml/parser.py
|
75
|
+
# SPDX-License-Identifier: MIT
|
76
|
+
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
77
|
+
# Licensed to PSF under a Contributor Agreement.
|
78
|
+
#
|
79
|
+
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
80
|
+
# --------------------------------------------
|
81
|
+
#
|
82
|
+
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
83
|
+
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
84
|
+
# documentation.
|
85
|
+
#
|
86
|
+
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
87
|
+
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
88
|
+
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
89
|
+
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
90
|
+
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
91
|
+
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
92
|
+
#
|
93
|
+
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
94
|
+
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
95
|
+
# any such work a brief summary of the changes made to Python.
|
96
|
+
#
|
97
|
+
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
98
|
+
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
99
|
+
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
100
|
+
# RIGHTS.
|
101
|
+
#
|
102
|
+
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
103
|
+
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
104
|
+
# ADVISED OF THE POSSIBILITY THEREOF.
|
105
|
+
#
|
106
|
+
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
107
|
+
#
|
108
|
+
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
109
|
+
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
110
|
+
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
111
|
+
#
|
112
|
+
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
113
|
+
# License Agreement.
|
114
|
+
#
|
115
|
+
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
116
|
+
|
117
|
+
|
118
|
+
##
|
119
|
+
|
120
|
+
|
121
|
+
_TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
|
122
|
+
|
123
|
+
TOML_RE_NUMBER = re.compile(
|
124
|
+
r"""
|
125
|
+
0
|
126
|
+
(?:
|
127
|
+
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
128
|
+
|
|
129
|
+
b[01](?:_?[01])* # bin
|
130
|
+
|
|
131
|
+
o[0-7](?:_?[0-7])* # oct
|
132
|
+
)
|
133
|
+
|
|
134
|
+
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
135
|
+
(?P<floatpart>
|
136
|
+
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
137
|
+
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
138
|
+
)
|
139
|
+
""",
|
140
|
+
flags=re.VERBOSE,
|
141
|
+
)
|
142
|
+
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
143
|
+
TOML_RE_DATETIME = re.compile(
|
144
|
+
rf"""
|
145
|
+
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
146
|
+
(?:
|
147
|
+
[Tt ]
|
148
|
+
{_TOML_TIME_RE_STR}
|
149
|
+
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
150
|
+
)?
|
151
|
+
""",
|
152
|
+
flags=re.VERBOSE,
|
153
|
+
)
|
154
|
+
|
155
|
+
|
156
|
+
def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
|
157
|
+
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
158
|
+
|
159
|
+
Raises ValueError if the match does not correspond to a valid date or datetime.
|
160
|
+
"""
|
161
|
+
(
|
162
|
+
year_str,
|
163
|
+
month_str,
|
164
|
+
day_str,
|
165
|
+
hour_str,
|
166
|
+
minute_str,
|
167
|
+
sec_str,
|
168
|
+
micros_str,
|
169
|
+
zulu_time,
|
170
|
+
offset_sign_str,
|
171
|
+
offset_hour_str,
|
172
|
+
offset_minute_str,
|
173
|
+
) = match.groups()
|
174
|
+
year, month, day = int(year_str), int(month_str), int(day_str)
|
175
|
+
if hour_str is None:
|
176
|
+
return datetime.date(year, month, day)
|
177
|
+
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
178
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
179
|
+
if offset_sign_str:
|
180
|
+
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
181
|
+
offset_hour_str, offset_minute_str, offset_sign_str,
|
182
|
+
)
|
183
|
+
elif zulu_time:
|
184
|
+
tz = datetime.UTC
|
185
|
+
else: # local date-time
|
186
|
+
tz = None
|
187
|
+
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
188
|
+
|
189
|
+
|
190
|
+
@functools.lru_cache() # noqa
|
191
|
+
def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
|
192
|
+
sign = 1 if sign_str == '+' else -1
|
193
|
+
return datetime.timezone(
|
194
|
+
datetime.timedelta(
|
195
|
+
hours=sign * int(hour_str),
|
196
|
+
minutes=sign * int(minute_str),
|
197
|
+
),
|
198
|
+
)
|
199
|
+
|
200
|
+
|
201
|
+
def toml_match_to_localtime(match: re.Match) -> datetime.time:
|
202
|
+
hour_str, minute_str, sec_str, micros_str = match.groups()
|
203
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
204
|
+
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
205
|
+
|
206
|
+
|
207
|
+
def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
|
208
|
+
if match.group('floatpart'):
|
209
|
+
return parse_float(match.group())
|
210
|
+
return int(match.group(), 0)
|
211
|
+
|
212
|
+
|
213
|
+
TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
|
214
|
+
|
215
|
+
# Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
|
216
|
+
# functions.
|
217
|
+
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
218
|
+
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
219
|
+
|
220
|
+
TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
221
|
+
TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
222
|
+
|
223
|
+
TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
224
|
+
|
225
|
+
TOML_WS = frozenset(' \t')
|
226
|
+
TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
|
227
|
+
TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
|
228
|
+
TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
|
229
|
+
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
230
|
+
|
231
|
+
TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
|
232
|
+
{
|
233
|
+
'\\b': '\u0008', # backspace
|
234
|
+
'\\t': '\u0009', # tab
|
235
|
+
'\\n': '\u000A', # linefeed
|
236
|
+
'\\f': '\u000C', # form feed
|
237
|
+
'\\r': '\u000D', # carriage return
|
238
|
+
'\\"': '\u0022', # quote
|
239
|
+
'\\\\': '\u005C', # backslash
|
240
|
+
},
|
241
|
+
)
|
242
|
+
|
243
|
+
|
244
|
+
class TomlDecodeError(ValueError):
|
245
|
+
"""An error raised if a document is not valid TOML."""
|
246
|
+
|
247
|
+
|
248
|
+
def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
|
249
|
+
"""Parse TOML from a binary file object."""
|
250
|
+
b = fp.read()
|
251
|
+
try:
|
252
|
+
s = b.decode()
|
253
|
+
except AttributeError:
|
254
|
+
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
255
|
+
return toml_loads(s, parse_float=parse_float)
|
256
|
+
|
257
|
+
|
258
|
+
def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
|
259
|
+
"""Parse TOML from a string."""
|
260
|
+
|
261
|
+
# The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
|
262
|
+
try:
|
263
|
+
src = s.replace('\r\n', '\n')
|
264
|
+
except (AttributeError, TypeError):
|
265
|
+
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
266
|
+
pos = 0
|
267
|
+
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
268
|
+
header: TomlKey = ()
|
269
|
+
parse_float = toml_make_safe_parse_float(parse_float)
|
270
|
+
|
271
|
+
# Parse one statement at a time (typically means one line in TOML source)
|
272
|
+
while True:
|
273
|
+
# 1. Skip line leading whitespace
|
274
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
275
|
+
|
276
|
+
# 2. Parse rules. Expect one of the following:
|
277
|
+
# - end of file
|
278
|
+
# - end of line
|
279
|
+
# - comment
|
280
|
+
# - key/value pair
|
281
|
+
# - append dict to list (and move to its namespace)
|
282
|
+
# - create dict (and move to its namespace)
|
283
|
+
# Skip trailing whitespace when applicable.
|
284
|
+
try:
|
285
|
+
char = src[pos]
|
286
|
+
except IndexError:
|
287
|
+
break
|
288
|
+
if char == '\n':
|
289
|
+
pos += 1
|
290
|
+
continue
|
291
|
+
if char in TOML_KEY_INITIAL_CHARS:
|
292
|
+
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
293
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
294
|
+
elif char == '[':
|
295
|
+
try:
|
296
|
+
second_char: ta.Optional[str] = src[pos + 1]
|
297
|
+
except IndexError:
|
298
|
+
second_char = None
|
299
|
+
out.flags.finalize_pending()
|
300
|
+
if second_char == '[':
|
301
|
+
pos, header = toml_create_list_rule(src, pos, out)
|
302
|
+
else:
|
303
|
+
pos, header = toml_create_dict_rule(src, pos, out)
|
304
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
305
|
+
elif char != '#':
|
306
|
+
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
307
|
+
|
308
|
+
# 3. Skip comment
|
309
|
+
pos = toml_skip_comment(src, pos)
|
310
|
+
|
311
|
+
# 4. Expect end of line or end of file
|
312
|
+
try:
|
313
|
+
char = src[pos]
|
314
|
+
except IndexError:
|
315
|
+
break
|
316
|
+
if char != '\n':
|
317
|
+
raise toml_suffixed_err(
|
318
|
+
src, pos, 'Expected newline or end of document after a statement',
|
319
|
+
)
|
320
|
+
pos += 1
|
321
|
+
|
322
|
+
return out.data.dict
|
323
|
+
|
324
|
+
|
325
|
+
class TomlFlags:
|
326
|
+
"""Flags that map to parsed keys/namespaces."""
|
327
|
+
|
328
|
+
# Marks an immutable namespace (inline array or inline table).
|
329
|
+
FROZEN = 0
|
330
|
+
# Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
|
331
|
+
EXPLICIT_NEST = 1
|
332
|
+
|
333
|
+
def __init__(self) -> None:
|
334
|
+
self._flags: ta.Dict[str, dict] = {}
|
335
|
+
self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
|
336
|
+
|
337
|
+
def add_pending(self, key: TomlKey, flag: int) -> None:
|
338
|
+
self._pending_flags.add((key, flag))
|
339
|
+
|
340
|
+
def finalize_pending(self) -> None:
|
341
|
+
for key, flag in self._pending_flags:
|
342
|
+
self.set(key, flag, recursive=False)
|
343
|
+
self._pending_flags.clear()
|
344
|
+
|
345
|
+
def unset_all(self, key: TomlKey) -> None:
|
346
|
+
cont = self._flags
|
347
|
+
for k in key[:-1]:
|
348
|
+
if k not in cont:
|
349
|
+
return
|
350
|
+
cont = cont[k]['nested']
|
351
|
+
cont.pop(key[-1], None)
|
352
|
+
|
353
|
+
def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
|
354
|
+
cont = self._flags
|
355
|
+
key_parent, key_stem = key[:-1], key[-1]
|
356
|
+
for k in key_parent:
|
357
|
+
if k not in cont:
|
358
|
+
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
359
|
+
cont = cont[k]['nested']
|
360
|
+
if key_stem not in cont:
|
361
|
+
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
362
|
+
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
363
|
+
|
364
|
+
def is_(self, key: TomlKey, flag: int) -> bool:
|
365
|
+
if not key:
|
366
|
+
return False # document root has no flags
|
367
|
+
cont = self._flags
|
368
|
+
for k in key[:-1]:
|
369
|
+
if k not in cont:
|
370
|
+
return False
|
371
|
+
inner_cont = cont[k]
|
372
|
+
if flag in inner_cont['recursive_flags']:
|
373
|
+
return True
|
374
|
+
cont = inner_cont['nested']
|
375
|
+
key_stem = key[-1]
|
376
|
+
if key_stem in cont:
|
377
|
+
cont = cont[key_stem]
|
378
|
+
return flag in cont['flags'] or flag in cont['recursive_flags']
|
379
|
+
return False
|
380
|
+
|
381
|
+
|
382
|
+
class TomlNestedDict:
|
383
|
+
def __init__(self) -> None:
|
384
|
+
# The parsed content of the TOML document
|
385
|
+
self.dict: ta.Dict[str, ta.Any] = {}
|
386
|
+
|
387
|
+
def get_or_create_nest(
|
388
|
+
self,
|
389
|
+
key: TomlKey,
|
390
|
+
*,
|
391
|
+
access_lists: bool = True,
|
392
|
+
) -> dict:
|
393
|
+
cont: ta.Any = self.dict
|
394
|
+
for k in key:
|
395
|
+
if k not in cont:
|
396
|
+
cont[k] = {}
|
397
|
+
cont = cont[k]
|
398
|
+
if access_lists and isinstance(cont, list):
|
399
|
+
cont = cont[-1]
|
400
|
+
if not isinstance(cont, dict):
|
401
|
+
raise KeyError('There is no nest behind this key')
|
402
|
+
return cont
|
403
|
+
|
404
|
+
def append_nest_to_list(self, key: TomlKey) -> None:
|
405
|
+
cont = self.get_or_create_nest(key[:-1])
|
406
|
+
last_key = key[-1]
|
407
|
+
if last_key in cont:
|
408
|
+
list_ = cont[last_key]
|
409
|
+
if not isinstance(list_, list):
|
410
|
+
raise KeyError('An object other than list found behind this key')
|
411
|
+
list_.append({})
|
412
|
+
else:
|
413
|
+
cont[last_key] = [{}]
|
414
|
+
|
415
|
+
|
416
|
+
class TomlOutput(ta.NamedTuple):
|
417
|
+
data: TomlNestedDict
|
418
|
+
flags: TomlFlags
|
419
|
+
|
420
|
+
|
421
|
+
def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
|
422
|
+
try:
|
423
|
+
while src[pos] in chars:
|
424
|
+
pos += 1
|
425
|
+
except IndexError:
|
426
|
+
pass
|
427
|
+
return pos
|
428
|
+
|
429
|
+
|
430
|
+
def toml_skip_until(
|
431
|
+
src: str,
|
432
|
+
pos: TomlPos,
|
433
|
+
expect: str,
|
434
|
+
*,
|
435
|
+
error_on: ta.FrozenSet[str],
|
436
|
+
error_on_eof: bool,
|
437
|
+
) -> TomlPos:
|
438
|
+
try:
|
439
|
+
new_pos = src.index(expect, pos)
|
440
|
+
except ValueError:
|
441
|
+
new_pos = len(src)
|
442
|
+
if error_on_eof:
|
443
|
+
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
444
|
+
|
445
|
+
if not error_on.isdisjoint(src[pos:new_pos]):
|
446
|
+
while src[pos] not in error_on:
|
447
|
+
pos += 1
|
448
|
+
raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
|
449
|
+
return new_pos
|
450
|
+
|
451
|
+
|
452
|
+
def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
|
453
|
+
try:
|
454
|
+
char: ta.Optional[str] = src[pos]
|
455
|
+
except IndexError:
|
456
|
+
char = None
|
457
|
+
if char == '#':
|
458
|
+
return toml_skip_until(
|
459
|
+
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
460
|
+
)
|
461
|
+
return pos
|
462
|
+
|
463
|
+
|
464
|
+
def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
|
465
|
+
while True:
|
466
|
+
pos_before_skip = pos
|
467
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
468
|
+
pos = toml_skip_comment(src, pos)
|
469
|
+
if pos == pos_before_skip:
|
470
|
+
return pos
|
471
|
+
|
472
|
+
|
473
|
+
def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
474
|
+
pos += 1 # Skip "["
|
475
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
476
|
+
pos, key = toml_parse_key(src, pos)
|
477
|
+
|
478
|
+
if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
|
479
|
+
raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
|
480
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
481
|
+
try:
|
482
|
+
out.data.get_or_create_nest(key)
|
483
|
+
except KeyError:
|
484
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
485
|
+
|
486
|
+
if not src.startswith(']', pos):
|
487
|
+
raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
|
488
|
+
return pos + 1, key
|
489
|
+
|
490
|
+
|
491
|
+
def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
492
|
+
pos += 2 # Skip "[["
|
493
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
494
|
+
pos, key = toml_parse_key(src, pos)
|
495
|
+
|
496
|
+
if out.flags.is_(key, TomlFlags.FROZEN):
|
497
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
498
|
+
# Free the namespace now that it points to another empty list item...
|
499
|
+
out.flags.unset_all(key)
|
500
|
+
# ...but this key precisely is still prohibited from table declaration
|
501
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
502
|
+
try:
|
503
|
+
out.data.append_nest_to_list(key)
|
504
|
+
except KeyError:
|
505
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
506
|
+
|
507
|
+
if not src.startswith(']]', pos):
|
508
|
+
raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
|
509
|
+
return pos + 2, key
|
510
|
+
|
511
|
+
|
512
|
+
def toml_key_value_rule(
|
513
|
+
src: str,
|
514
|
+
pos: TomlPos,
|
515
|
+
out: TomlOutput,
|
516
|
+
header: TomlKey,
|
517
|
+
parse_float: TomlParseFloat,
|
518
|
+
) -> TomlPos:
|
519
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
520
|
+
key_parent, key_stem = key[:-1], key[-1]
|
521
|
+
abs_key_parent = header + key_parent
|
522
|
+
|
523
|
+
relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
|
524
|
+
for cont_key in relative_path_cont_keys:
|
525
|
+
# Check that dotted key syntax does not redefine an existing table
|
526
|
+
if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
|
527
|
+
raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
|
528
|
+
# Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
|
529
|
+
# table sections.
|
530
|
+
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
531
|
+
|
532
|
+
if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
|
533
|
+
raise toml_suffixed_err(
|
534
|
+
src,
|
535
|
+
pos,
|
536
|
+
f'Cannot mutate immutable namespace {abs_key_parent}',
|
537
|
+
)
|
538
|
+
|
539
|
+
try:
|
540
|
+
nest = out.data.get_or_create_nest(abs_key_parent)
|
541
|
+
except KeyError:
|
542
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
543
|
+
if key_stem in nest:
|
544
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
545
|
+
# Mark inline table and array namespaces recursively immutable
|
546
|
+
if isinstance(value, (dict, list)):
|
547
|
+
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
548
|
+
nest[key_stem] = value
|
549
|
+
return pos
|
550
|
+
|
551
|
+
|
552
|
+
def toml_parse_key_value_pair(
|
553
|
+
src: str,
|
554
|
+
pos: TomlPos,
|
555
|
+
parse_float: TomlParseFloat,
|
556
|
+
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
557
|
+
pos, key = toml_parse_key(src, pos)
|
558
|
+
try:
|
559
|
+
char: ta.Optional[str] = src[pos]
|
560
|
+
except IndexError:
|
561
|
+
char = None
|
562
|
+
if char != '=':
|
563
|
+
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
564
|
+
pos += 1
|
565
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
566
|
+
pos, value = toml_parse_value(src, pos, parse_float)
|
567
|
+
return pos, key, value
|
568
|
+
|
569
|
+
|
570
|
+
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
571
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
572
|
+
key: TomlKey = (key_part,)
|
573
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
574
|
+
while True:
|
575
|
+
try:
|
576
|
+
char: ta.Optional[str] = src[pos]
|
577
|
+
except IndexError:
|
578
|
+
char = None
|
579
|
+
if char != '.':
|
580
|
+
return pos, key
|
581
|
+
pos += 1
|
582
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
583
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
584
|
+
key += (key_part,)
|
585
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
586
|
+
|
587
|
+
|
588
|
+
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
589
|
+
try:
|
590
|
+
char: ta.Optional[str] = src[pos]
|
591
|
+
except IndexError:
|
592
|
+
char = None
|
593
|
+
if char in TOML_BARE_KEY_CHARS:
|
594
|
+
start_pos = pos
|
595
|
+
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
596
|
+
return pos, src[start_pos:pos]
|
597
|
+
if char == "'":
|
598
|
+
return toml_parse_literal_str(src, pos)
|
599
|
+
if char == '"':
|
600
|
+
return toml_parse_one_line_basic_str(src, pos)
|
601
|
+
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
602
|
+
|
603
|
+
|
604
|
+
def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
605
|
+
pos += 1
|
606
|
+
return toml_parse_basic_str(src, pos, multiline=False)
|
607
|
+
|
608
|
+
|
609
|
+
def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
|
610
|
+
pos += 1
|
611
|
+
array: list = []
|
612
|
+
|
613
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
614
|
+
if src.startswith(']', pos):
|
615
|
+
return pos + 1, array
|
616
|
+
while True:
|
617
|
+
pos, val = toml_parse_value(src, pos, parse_float)
|
618
|
+
array.append(val)
|
619
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
620
|
+
|
621
|
+
c = src[pos:pos + 1]
|
622
|
+
if c == ']':
|
623
|
+
return pos + 1, array
|
624
|
+
if c != ',':
|
625
|
+
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
626
|
+
pos += 1
|
627
|
+
|
628
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
629
|
+
if src.startswith(']', pos):
|
630
|
+
return pos + 1, array
|
631
|
+
|
632
|
+
|
633
|
+
def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
|
634
|
+
pos += 1
|
635
|
+
nested_dict = TomlNestedDict()
|
636
|
+
flags = TomlFlags()
|
637
|
+
|
638
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
639
|
+
if src.startswith('}', pos):
|
640
|
+
return pos + 1, nested_dict.dict
|
641
|
+
while True:
|
642
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
643
|
+
key_parent, key_stem = key[:-1], key[-1]
|
644
|
+
if flags.is_(key, TomlFlags.FROZEN):
|
645
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
646
|
+
try:
|
647
|
+
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
648
|
+
except KeyError:
|
649
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
650
|
+
if key_stem in nest:
|
651
|
+
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
652
|
+
nest[key_stem] = value
|
653
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
654
|
+
c = src[pos:pos + 1]
|
655
|
+
if c == '}':
|
656
|
+
return pos + 1, nested_dict.dict
|
657
|
+
if c != ',':
|
658
|
+
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
659
|
+
if isinstance(value, (dict, list)):
|
660
|
+
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
661
|
+
pos += 1
|
662
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
663
|
+
|
664
|
+
|
665
|
+
def toml_parse_basic_str_escape(
|
666
|
+
src: str,
|
667
|
+
pos: TomlPos,
|
668
|
+
*,
|
669
|
+
multiline: bool = False,
|
670
|
+
) -> ta.Tuple[TomlPos, str]:
|
671
|
+
escape_id = src[pos:pos + 2]
|
672
|
+
pos += 2
|
673
|
+
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
674
|
+
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
675
|
+
# newline.
|
676
|
+
if escape_id != '\\\n':
|
677
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
678
|
+
try:
|
679
|
+
char = src[pos]
|
680
|
+
except IndexError:
|
681
|
+
return pos, ''
|
682
|
+
if char != '\n':
|
683
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
684
|
+
pos += 1
|
685
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
686
|
+
return pos, ''
|
687
|
+
if escape_id == '\\u':
|
688
|
+
return toml_parse_hex_char(src, pos, 4)
|
689
|
+
if escape_id == '\\U':
|
690
|
+
return toml_parse_hex_char(src, pos, 8)
|
691
|
+
try:
|
692
|
+
return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
|
693
|
+
except KeyError:
|
694
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
695
|
+
|
696
|
+
|
697
|
+
def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
698
|
+
return toml_parse_basic_str_escape(src, pos, multiline=True)
|
699
|
+
|
700
|
+
|
701
|
+
def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
|
702
|
+
hex_str = src[pos:pos + hex_len]
|
703
|
+
if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
|
704
|
+
raise toml_suffixed_err(src, pos, 'Invalid hex value')
|
705
|
+
pos += hex_len
|
706
|
+
hex_int = int(hex_str, 16)
|
707
|
+
if not toml_is_unicode_scalar_value(hex_int):
|
708
|
+
raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
|
709
|
+
return pos, chr(hex_int)
|
710
|
+
|
711
|
+
|
712
|
+
def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
713
|
+
pos += 1 # Skip starting apostrophe
|
714
|
+
start_pos = pos
|
715
|
+
pos = toml_skip_until(
|
716
|
+
src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
|
717
|
+
)
|
718
|
+
return pos + 1, src[start_pos:pos] # Skip ending apostrophe
|
719
|
+
|
720
|
+
|
721
|
+
def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
|
722
|
+
pos += 3
|
723
|
+
if src.startswith('\n', pos):
|
724
|
+
pos += 1
|
725
|
+
|
726
|
+
if literal:
|
727
|
+
delim = "'"
|
728
|
+
end_pos = toml_skip_until(
|
729
|
+
src,
|
730
|
+
pos,
|
731
|
+
"'''",
|
732
|
+
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
733
|
+
error_on_eof=True,
|
734
|
+
)
|
735
|
+
result = src[pos:end_pos]
|
736
|
+
pos = end_pos + 3
|
737
|
+
else:
|
738
|
+
delim = '"'
|
739
|
+
pos, result = toml_parse_basic_str(src, pos, multiline=True)
|
740
|
+
|
741
|
+
# Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
|
742
|
+
if not src.startswith(delim, pos):
|
743
|
+
return pos, result
|
744
|
+
pos += 1
|
745
|
+
if not src.startswith(delim, pos):
|
746
|
+
return pos, result + delim
|
747
|
+
pos += 1
|
748
|
+
return pos, result + (delim * 2)
|
749
|
+
|
750
|
+
|
751
|
+
def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
|
752
|
+
if multiline:
|
753
|
+
error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
754
|
+
parse_escapes = toml_parse_basic_str_escape_multiline
|
755
|
+
else:
|
756
|
+
error_on = TOML_ILLEGAL_BASIC_STR_CHARS
|
757
|
+
parse_escapes = toml_parse_basic_str_escape
|
758
|
+
result = ''
|
759
|
+
start_pos = pos
|
760
|
+
while True:
|
761
|
+
try:
|
762
|
+
char = src[pos]
|
763
|
+
except IndexError:
|
764
|
+
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
765
|
+
if char == '"':
|
766
|
+
if not multiline:
|
767
|
+
return pos + 1, result + src[start_pos:pos]
|
768
|
+
if src.startswith('"""', pos):
|
769
|
+
return pos + 3, result + src[start_pos:pos]
|
770
|
+
pos += 1
|
771
|
+
continue
|
772
|
+
if char == '\\':
|
773
|
+
result += src[start_pos:pos]
|
774
|
+
pos, parsed_escape = parse_escapes(src, pos)
|
775
|
+
result += parsed_escape
|
776
|
+
start_pos = pos
|
777
|
+
continue
|
778
|
+
if char in error_on:
|
779
|
+
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
780
|
+
pos += 1
|
781
|
+
|
782
|
+
|
783
|
+
def toml_parse_value( # noqa: C901
|
784
|
+
src: str,
|
785
|
+
pos: TomlPos,
|
786
|
+
parse_float: TomlParseFloat,
|
787
|
+
) -> ta.Tuple[TomlPos, ta.Any]:
|
788
|
+
try:
|
789
|
+
char: ta.Optional[str] = src[pos]
|
790
|
+
except IndexError:
|
791
|
+
char = None
|
792
|
+
|
793
|
+
# IMPORTANT: order conditions based on speed of checking and likelihood
|
794
|
+
|
795
|
+
# Basic strings
|
796
|
+
if char == '"':
|
797
|
+
if src.startswith('"""', pos):
|
798
|
+
return toml_parse_multiline_str(src, pos, literal=False)
|
799
|
+
return toml_parse_one_line_basic_str(src, pos)
|
800
|
+
|
801
|
+
# Literal strings
|
802
|
+
if char == "'":
|
803
|
+
if src.startswith("'''", pos):
|
804
|
+
return toml_parse_multiline_str(src, pos, literal=True)
|
805
|
+
return toml_parse_literal_str(src, pos)
|
806
|
+
|
807
|
+
# Booleans
|
808
|
+
if char == 't':
|
809
|
+
if src.startswith('true', pos):
|
810
|
+
return pos + 4, True
|
811
|
+
if char == 'f':
|
812
|
+
if src.startswith('false', pos):
|
813
|
+
return pos + 5, False
|
814
|
+
|
815
|
+
# Arrays
|
816
|
+
if char == '[':
|
817
|
+
return toml_parse_array(src, pos, parse_float)
|
818
|
+
|
819
|
+
# Inline tables
|
820
|
+
if char == '{':
|
821
|
+
return toml_parse_inline_table(src, pos, parse_float)
|
822
|
+
|
823
|
+
# Dates and times
|
824
|
+
datetime_match = TOML_RE_DATETIME.match(src, pos)
|
825
|
+
if datetime_match:
|
826
|
+
try:
|
827
|
+
datetime_obj = toml_match_to_datetime(datetime_match)
|
828
|
+
except ValueError as e:
|
829
|
+
raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
|
830
|
+
return datetime_match.end(), datetime_obj
|
831
|
+
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
832
|
+
if localtime_match:
|
833
|
+
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
834
|
+
|
835
|
+
# Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
|
836
|
+
# located after handling of dates and times.
|
837
|
+
number_match = TOML_RE_NUMBER.match(src, pos)
|
838
|
+
if number_match:
|
839
|
+
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
840
|
+
|
841
|
+
# Special floats
|
842
|
+
first_three = src[pos:pos + 3]
|
843
|
+
if first_three in {'inf', 'nan'}:
|
844
|
+
return pos + 3, parse_float(first_three)
|
845
|
+
first_four = src[pos:pos + 4]
|
846
|
+
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
847
|
+
return pos + 4, parse_float(first_four)
|
848
|
+
|
849
|
+
raise toml_suffixed_err(src, pos, 'Invalid value')
|
850
|
+
|
851
|
+
|
852
|
+
def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
|
853
|
+
"""Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
|
854
|
+
|
855
|
+
def coord_repr(src: str, pos: TomlPos) -> str:
|
856
|
+
if pos >= len(src):
|
857
|
+
return 'end of document'
|
858
|
+
line = src.count('\n', 0, pos) + 1
|
859
|
+
if line == 1:
|
860
|
+
column = pos + 1
|
861
|
+
else:
|
862
|
+
column = pos - src.rindex('\n', 0, pos)
|
863
|
+
return f'line {line}, column {column}'
|
864
|
+
|
865
|
+
return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
|
866
|
+
|
867
|
+
|
868
|
+
def toml_is_unicode_scalar_value(codepoint: int) -> bool:
|
869
|
+
return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
|
870
|
+
|
871
|
+
|
872
|
+
def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
|
873
|
+
"""A decorator to make `parse_float` safe.
|
874
|
+
|
875
|
+
`parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
|
876
|
+
thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
|
877
|
+
"""
|
878
|
+
# The default `float` callable never returns illegal types. Optimize it.
|
879
|
+
if parse_float is float:
|
880
|
+
return float
|
881
|
+
|
882
|
+
def safe_parse_float(float_str: str) -> ta.Any:
|
883
|
+
float_value = parse_float(float_str)
|
884
|
+
if isinstance(float_value, (dict, list)):
|
885
|
+
raise ValueError('parse_float must not return dicts or lists') # noqa
|
886
|
+
return float_value
|
887
|
+
|
888
|
+
return safe_parse_float
|
889
|
+
|
890
|
+
|
62
891
|
########################################
|
63
892
|
# ../../../../../omlish/lite/cached.py
|
64
893
|
|
@@ -80,7 +909,7 @@ class _cached_nullary: # noqa
|
|
80
909
|
return bound
|
81
910
|
|
82
911
|
|
83
|
-
def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
|
912
|
+
def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
84
913
|
return _cached_nullary(fn)
|
85
914
|
|
86
915
|
|
@@ -733,8 +1562,12 @@ class ExitStacked:
|
|
733
1562
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
734
1563
|
if (es := self._exit_stack) is None:
|
735
1564
|
return None
|
1565
|
+
self._exit_contexts()
|
736
1566
|
return es.__exit__(exc_type, exc_val, exc_tb)
|
737
1567
|
|
1568
|
+
def _exit_contexts(self) -> None:
|
1569
|
+
pass
|
1570
|
+
|
738
1571
|
def _enter_context(self, cm: ta.ContextManager[T]) -> T:
|
739
1572
|
es = check_not_none(self._exit_stack)
|
740
1573
|
return es.enter_context(cm)
|
@@ -1734,6 +2567,66 @@ class AwsLogMessageBuilder:
|
|
1734
2567
|
return [post]
|
1735
2568
|
|
1736
2569
|
|
2570
|
+
########################################
|
2571
|
+
# ../../../../configs.py
|
2572
|
+
|
2573
|
+
|
2574
|
+
def read_config_file(
|
2575
|
+
path: str,
|
2576
|
+
cls: ta.Type[T],
|
2577
|
+
*,
|
2578
|
+
prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
|
2579
|
+
) -> T:
|
2580
|
+
with open(path) as cf:
|
2581
|
+
if path.endswith('.toml'):
|
2582
|
+
config_dct = toml_loads(cf.read())
|
2583
|
+
else:
|
2584
|
+
config_dct = json.loads(cf.read())
|
2585
|
+
|
2586
|
+
if prepare is not None:
|
2587
|
+
config_dct = prepare(config_dct) # type: ignore
|
2588
|
+
|
2589
|
+
return unmarshal_obj(config_dct, cls)
|
2590
|
+
|
2591
|
+
|
2592
|
+
def build_config_named_children(
|
2593
|
+
o: ta.Union[
|
2594
|
+
ta.Sequence[ConfigMapping],
|
2595
|
+
ta.Mapping[str, ConfigMapping],
|
2596
|
+
None,
|
2597
|
+
],
|
2598
|
+
*,
|
2599
|
+
name_key: str = 'name',
|
2600
|
+
) -> ta.Optional[ta.Sequence[ConfigMapping]]:
|
2601
|
+
if o is None:
|
2602
|
+
return None
|
2603
|
+
|
2604
|
+
lst: ta.List[ConfigMapping] = []
|
2605
|
+
if isinstance(o, ta.Mapping):
|
2606
|
+
for k, v in o.items():
|
2607
|
+
check_isinstance(v, ta.Mapping)
|
2608
|
+
if name_key in v:
|
2609
|
+
n = v[name_key]
|
2610
|
+
if k != n:
|
2611
|
+
raise KeyError(f'Given names do not match: {n} != {k}')
|
2612
|
+
lst.append(v)
|
2613
|
+
else:
|
2614
|
+
lst.append({name_key: k, **v})
|
2615
|
+
|
2616
|
+
else:
|
2617
|
+
check_not_isinstance(o, str)
|
2618
|
+
lst.extend(o)
|
2619
|
+
|
2620
|
+
seen = set()
|
2621
|
+
for d in lst:
|
2622
|
+
n = d['name']
|
2623
|
+
if n in d:
|
2624
|
+
raise KeyError(f'Duplicate name: {n}')
|
2625
|
+
seen.add(n)
|
2626
|
+
|
2627
|
+
return lst
|
2628
|
+
|
2629
|
+
|
1737
2630
|
########################################
|
1738
2631
|
# ../../../../journald/messages.py
|
1739
2632
|
|
@@ -1810,9 +2703,11 @@ class JournalctlMessageBuilder:
|
|
1810
2703
|
########################################
|
1811
2704
|
# ../../../../threadworkers.py
|
1812
2705
|
"""
|
2706
|
+
FIXME:
|
2707
|
+
- group is racy af - meditate on has_started, etc
|
2708
|
+
|
1813
2709
|
TODO:
|
1814
|
-
-
|
1815
|
-
- collective heartbeat monitoring - ThreadWorkerGroups
|
2710
|
+
- overhaul stop lol
|
1816
2711
|
- group -> 'context'? :|
|
1817
2712
|
- shared stop_event?
|
1818
2713
|
"""
|
@@ -1826,6 +2721,7 @@ class ThreadWorker(ExitStacked, abc.ABC):
|
|
1826
2721
|
self,
|
1827
2722
|
*,
|
1828
2723
|
stop_event: ta.Optional[threading.Event] = None,
|
2724
|
+
worker_groups: ta.Optional[ta.Iterable['ThreadWorkerGroup']] = None,
|
1829
2725
|
) -> None:
|
1830
2726
|
super().__init__()
|
1831
2727
|
|
@@ -1837,6 +2733,9 @@ class ThreadWorker(ExitStacked, abc.ABC):
|
|
1837
2733
|
self._thread: ta.Optional[threading.Thread] = None
|
1838
2734
|
self._last_heartbeat: ta.Optional[float] = None
|
1839
2735
|
|
2736
|
+
for g in worker_groups or []:
|
2737
|
+
g.add(self)
|
2738
|
+
|
1840
2739
|
#
|
1841
2740
|
|
1842
2741
|
def __enter__(self: ThreadWorkerT) -> ThreadWorkerT:
|
@@ -1881,13 +2780,13 @@ class ThreadWorker(ExitStacked, abc.ABC):
|
|
1881
2780
|
if self._thread is not None:
|
1882
2781
|
raise RuntimeError('Thread already started: %r', self)
|
1883
2782
|
|
1884
|
-
thr = threading.Thread(target=self.
|
2783
|
+
thr = threading.Thread(target=self.__thread_main)
|
1885
2784
|
self._thread = thr
|
1886
2785
|
thr.start()
|
1887
2786
|
|
1888
2787
|
#
|
1889
2788
|
|
1890
|
-
def
|
2789
|
+
def __thread_main(self) -> None:
|
1891
2790
|
try:
|
1892
2791
|
self._run()
|
1893
2792
|
except ThreadWorker.Stopping:
|
@@ -1905,10 +2804,17 @@ class ThreadWorker(ExitStacked, abc.ABC):
|
|
1905
2804
|
def stop(self) -> None:
|
1906
2805
|
self._stop_event.set()
|
1907
2806
|
|
1908
|
-
def join(
|
2807
|
+
def join(
|
2808
|
+
self,
|
2809
|
+
timeout: ta.Optional[float] = None,
|
2810
|
+
*,
|
2811
|
+
unless_not_started: bool = False,
|
2812
|
+
) -> None:
|
1909
2813
|
with self._lock:
|
1910
2814
|
if self._thread is None:
|
1911
|
-
|
2815
|
+
if not unless_not_started:
|
2816
|
+
raise RuntimeError('Thread not started: %r', self)
|
2817
|
+
return
|
1912
2818
|
self._thread.join(timeout)
|
1913
2819
|
|
1914
2820
|
|
@@ -1917,24 +2823,68 @@ class ThreadWorker(ExitStacked, abc.ABC):
|
|
1917
2823
|
|
1918
2824
|
class ThreadWorkerGroup:
|
1919
2825
|
@dc.dataclass()
|
1920
|
-
class
|
2826
|
+
class _State:
|
1921
2827
|
worker: ThreadWorker
|
1922
2828
|
|
2829
|
+
last_heartbeat: ta.Optional[float] = None
|
2830
|
+
|
1923
2831
|
def __init__(self) -> None:
|
1924
2832
|
super().__init__()
|
1925
2833
|
|
1926
2834
|
self._lock = threading.RLock()
|
1927
|
-
self._states: ta.Dict[ThreadWorker, ThreadWorkerGroup.
|
2835
|
+
self._states: ta.Dict[ThreadWorker, ThreadWorkerGroup._State] = {}
|
2836
|
+
self._last_heartbeat_check: ta.Optional[float] = None
|
2837
|
+
|
2838
|
+
#
|
1928
2839
|
|
1929
2840
|
def add(self, *workers: ThreadWorker) -> 'ThreadWorkerGroup':
|
1930
2841
|
with self._lock:
|
1931
2842
|
for w in workers:
|
1932
2843
|
if w in self._states:
|
1933
2844
|
raise KeyError(w)
|
1934
|
-
self._states[w] = ThreadWorkerGroup.
|
2845
|
+
self._states[w] = ThreadWorkerGroup._State(w)
|
1935
2846
|
|
1936
2847
|
return self
|
1937
2848
|
|
2849
|
+
#
|
2850
|
+
|
2851
|
+
def start_all(self) -> None:
|
2852
|
+
thrs = list(self._states)
|
2853
|
+
with self._lock:
|
2854
|
+
for thr in thrs:
|
2855
|
+
if not thr.has_started():
|
2856
|
+
thr.start()
|
2857
|
+
|
2858
|
+
def stop_all(self) -> None:
|
2859
|
+
for w in reversed(list(self._states)):
|
2860
|
+
if w.has_started():
|
2861
|
+
w.stop()
|
2862
|
+
|
2863
|
+
def join_all(self, timeout: ta.Optional[float] = None) -> None:
|
2864
|
+
for w in reversed(list(self._states)):
|
2865
|
+
if w.has_started():
|
2866
|
+
w.join(timeout, unless_not_started=True)
|
2867
|
+
|
2868
|
+
#
|
2869
|
+
|
2870
|
+
def get_dead(self) -> ta.List[ThreadWorker]:
|
2871
|
+
with self._lock:
|
2872
|
+
return [thr for thr in self._states if not thr.is_alive()]
|
2873
|
+
|
2874
|
+
def check_heartbeats(self) -> ta.Dict[ThreadWorker, float]:
|
2875
|
+
with self._lock:
|
2876
|
+
dct: ta.Dict[ThreadWorker, float] = {}
|
2877
|
+
for thr, st in self._states.items():
|
2878
|
+
if not thr.has_started():
|
2879
|
+
continue
|
2880
|
+
hb = thr.last_heartbeat
|
2881
|
+
if hb is None:
|
2882
|
+
hb = time.time()
|
2883
|
+
st.last_heartbeat = hb
|
2884
|
+
dct[st.worker] = time.time() - hb
|
2885
|
+
self._last_heartbeat_check = time.time()
|
2886
|
+
return dct
|
2887
|
+
|
1938
2888
|
|
1939
2889
|
########################################
|
1940
2890
|
# ../../../../../omlish/lite/subprocesses.py
|
@@ -2694,6 +3644,7 @@ class JournalctlToAwsDriver(ExitStacked):
|
|
2694
3644
|
cursor_file: ta.Optional[str] = None
|
2695
3645
|
|
2696
3646
|
runtime_limit: ta.Optional[float] = None
|
3647
|
+
heartbeat_age_limit: ta.Optional[float] = 60.
|
2697
3648
|
|
2698
3649
|
#
|
2699
3650
|
|
@@ -2770,6 +3721,12 @@ class JournalctlToAwsDriver(ExitStacked):
|
|
2770
3721
|
|
2771
3722
|
#
|
2772
3723
|
|
3724
|
+
@cached_nullary
|
3725
|
+
def _worker_group(self) -> ThreadWorkerGroup:
|
3726
|
+
return ThreadWorkerGroup()
|
3727
|
+
|
3728
|
+
#
|
3729
|
+
|
2773
3730
|
@cached_nullary
|
2774
3731
|
def _journalctl_message_queue(self): # type: () -> queue.Queue[ta.Sequence[JournalctlMessage]]
|
2775
3732
|
return queue.Queue()
|
@@ -2796,6 +3753,8 @@ class JournalctlToAwsDriver(ExitStacked):
|
|
2796
3753
|
|
2797
3754
|
cmd=self._config.journalctl_cmd,
|
2798
3755
|
shell_wrap=is_debugger_attached(),
|
3756
|
+
|
3757
|
+
worker_groups=[self._worker_group()],
|
2799
3758
|
)
|
2800
3759
|
|
2801
3760
|
#
|
@@ -2809,26 +3768,38 @@ class JournalctlToAwsDriver(ExitStacked):
|
|
2809
3768
|
|
2810
3769
|
ensure_locked=self._ensure_locked,
|
2811
3770
|
dry_run=self._config.aws_dry_run,
|
3771
|
+
|
3772
|
+
worker_groups=[self._worker_group()],
|
2812
3773
|
)
|
2813
3774
|
|
2814
3775
|
#
|
2815
3776
|
|
2816
|
-
def
|
2817
|
-
|
2818
|
-
|
3777
|
+
def _exit_contexts(self) -> None:
|
3778
|
+
wg = self._worker_group()
|
3779
|
+
wg.stop_all()
|
3780
|
+
wg.join_all()
|
2819
3781
|
|
2820
|
-
|
3782
|
+
def run(self) -> None:
|
3783
|
+
self._aws_poster_worker()
|
3784
|
+
self._journalctl_tailer_worker()
|
2821
3785
|
|
2822
|
-
|
2823
|
-
|
3786
|
+
wg = self._worker_group()
|
3787
|
+
wg.start_all()
|
2824
3788
|
|
2825
3789
|
start = time.time()
|
2826
3790
|
|
2827
3791
|
while True:
|
2828
|
-
for w in
|
2829
|
-
|
2830
|
-
|
2831
|
-
|
3792
|
+
for w in wg.get_dead():
|
3793
|
+
log.critical('Worker died: %r', w)
|
3794
|
+
break
|
3795
|
+
|
3796
|
+
if (al := self._config.heartbeat_age_limit) is not None:
|
3797
|
+
hbs = wg.check_heartbeats()
|
3798
|
+
log.debug('Worker heartbeats: %r', hbs)
|
3799
|
+
for w, age in hbs.items():
|
3800
|
+
if age > al:
|
3801
|
+
log.critical('Worker heartbeat age limit exceeded: %r %f > %f', w, age, al)
|
3802
|
+
break
|
2832
3803
|
|
2833
3804
|
if (rl := self._config.runtime_limit) is not None and time.time() - start >= rl:
|
2834
3805
|
log.warning('Runtime limit reached')
|
@@ -2836,9 +3807,8 @@ class JournalctlToAwsDriver(ExitStacked):
|
|
2836
3807
|
|
2837
3808
|
time.sleep(1.)
|
2838
3809
|
|
2839
|
-
|
2840
|
-
|
2841
|
-
w.join()
|
3810
|
+
wg.stop_all()
|
3811
|
+
wg.join_all()
|
2842
3812
|
|
2843
3813
|
|
2844
3814
|
########################################
|
@@ -2870,9 +3840,7 @@ def _main() -> None:
|
|
2870
3840
|
|
2871
3841
|
config: JournalctlToAwsDriver.Config
|
2872
3842
|
if args.config_file:
|
2873
|
-
|
2874
|
-
config_dct = json.load(cf)
|
2875
|
-
config = unmarshal_obj(config_dct, JournalctlToAwsDriver.Config)
|
3843
|
+
config = read_config_file(os.path.expanduser(args.config_file), JournalctlToAwsDriver.Config)
|
2876
3844
|
else:
|
2877
3845
|
config = JournalctlToAwsDriver.Config()
|
2878
3846
|
|