ominfra 0.0.0.dev120__py3-none-any.whl → 0.0.0.dev122__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/__main__.py +4 -0
- ominfra/clouds/aws/journald2aws/driver.py +34 -13
- ominfra/clouds/aws/journald2aws/main.py +2 -5
- ominfra/configs.py +70 -0
- ominfra/deploy/_executor.py +10 -1
- ominfra/deploy/poly/_main.py +1 -1
- ominfra/pyremote/_runcommands.py +10 -1
- ominfra/scripts/journald2aws.py +1002 -26
- ominfra/scripts/supervisor.py +1848 -138
- ominfra/supervisor/compat.py +13 -0
- ominfra/supervisor/configs.py +21 -0
- ominfra/supervisor/context.py +13 -2
- ominfra/supervisor/main.py +82 -11
- ominfra/supervisor/process.py +39 -4
- ominfra/supervisor/supervisor.py +23 -2
- ominfra/supervisor/types.py +5 -0
- ominfra/threadworkers.py +66 -9
- {ominfra-0.0.0.dev120.dist-info → ominfra-0.0.0.dev122.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev120.dist-info → ominfra-0.0.0.dev122.dist-info}/RECORD +23 -21
- {ominfra-0.0.0.dev120.dist-info → ominfra-0.0.0.dev122.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev120.dist-info → ominfra-0.0.0.dev122.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev120.dist-info → ominfra-0.0.0.dev122.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev120.dist-info → ominfra-0.0.0.dev122.dist-info}/top_level.txt +0 -0
ominfra/scripts/supervisor.py
CHANGED
@@ -18,6 +18,7 @@ import fcntl
|
|
18
18
|
import fractions
|
19
19
|
import functools
|
20
20
|
import grp
|
21
|
+
import inspect
|
21
22
|
import itertools
|
22
23
|
import json
|
23
24
|
import logging
|
@@ -30,6 +31,7 @@ import select
|
|
30
31
|
import shlex
|
31
32
|
import signal
|
32
33
|
import stat
|
34
|
+
import string
|
33
35
|
import sys
|
34
36
|
import syslog
|
35
37
|
import tempfile
|
@@ -54,6 +56,11 @@ if sys.version_info < (3, 8):
|
|
54
56
|
########################################
|
55
57
|
|
56
58
|
|
59
|
+
# ../../../omdev/toml/parser.py
|
60
|
+
TomlParseFloat = ta.Callable[[str], ta.Any]
|
61
|
+
TomlKey = ta.Tuple[str, ...]
|
62
|
+
TomlPos = int # ta.TypeAlias
|
63
|
+
|
57
64
|
# ../compat.py
|
58
65
|
T = ta.TypeVar('T')
|
59
66
|
|
@@ -61,6 +68,837 @@ T = ta.TypeVar('T')
|
|
61
68
|
ProcessState = int # ta.TypeAlias
|
62
69
|
SupervisorState = int # ta.TypeAlias
|
63
70
|
|
71
|
+
# ../../../omlish/lite/inject.py
|
72
|
+
InjectorKeyCls = ta.Union[type, ta.NewType]
|
73
|
+
InjectorProviderFn = ta.Callable[['Injector'], ta.Any]
|
74
|
+
InjectorProviderFnMap = ta.Mapping['InjectorKey', 'InjectorProviderFn']
|
75
|
+
InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
|
76
|
+
|
77
|
+
# ../../configs.py
|
78
|
+
ConfigMapping = ta.Mapping[str, ta.Any]
|
79
|
+
|
80
|
+
# ../context.py
|
81
|
+
ServerEpoch = ta.NewType('ServerEpoch', int)
|
82
|
+
InheritedFds = ta.NewType('InheritedFds', ta.FrozenSet[int])
|
83
|
+
|
84
|
+
|
85
|
+
########################################
|
86
|
+
# ../../../omdev/toml/parser.py
|
87
|
+
# SPDX-License-Identifier: MIT
|
88
|
+
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
89
|
+
# Licensed to PSF under a Contributor Agreement.
|
90
|
+
#
|
91
|
+
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
92
|
+
# --------------------------------------------
|
93
|
+
#
|
94
|
+
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
95
|
+
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
96
|
+
# documentation.
|
97
|
+
#
|
98
|
+
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
99
|
+
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
100
|
+
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
101
|
+
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
102
|
+
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
103
|
+
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
104
|
+
#
|
105
|
+
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
106
|
+
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
107
|
+
# any such work a brief summary of the changes made to Python.
|
108
|
+
#
|
109
|
+
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
110
|
+
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
111
|
+
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
112
|
+
# RIGHTS.
|
113
|
+
#
|
114
|
+
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
115
|
+
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
116
|
+
# ADVISED OF THE POSSIBILITY THEREOF.
|
117
|
+
#
|
118
|
+
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
119
|
+
#
|
120
|
+
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
121
|
+
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
122
|
+
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
123
|
+
#
|
124
|
+
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
125
|
+
# License Agreement.
|
126
|
+
#
|
127
|
+
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
128
|
+
|
129
|
+
|
130
|
+
##
|
131
|
+
|
132
|
+
|
133
|
+
_TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
|
134
|
+
|
135
|
+
TOML_RE_NUMBER = re.compile(
|
136
|
+
r"""
|
137
|
+
0
|
138
|
+
(?:
|
139
|
+
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
140
|
+
|
|
141
|
+
b[01](?:_?[01])* # bin
|
142
|
+
|
|
143
|
+
o[0-7](?:_?[0-7])* # oct
|
144
|
+
)
|
145
|
+
|
|
146
|
+
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
147
|
+
(?P<floatpart>
|
148
|
+
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
149
|
+
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
150
|
+
)
|
151
|
+
""",
|
152
|
+
flags=re.VERBOSE,
|
153
|
+
)
|
154
|
+
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
155
|
+
TOML_RE_DATETIME = re.compile(
|
156
|
+
rf"""
|
157
|
+
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
158
|
+
(?:
|
159
|
+
[Tt ]
|
160
|
+
{_TOML_TIME_RE_STR}
|
161
|
+
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
162
|
+
)?
|
163
|
+
""",
|
164
|
+
flags=re.VERBOSE,
|
165
|
+
)
|
166
|
+
|
167
|
+
|
168
|
+
def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
|
169
|
+
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
170
|
+
|
171
|
+
Raises ValueError if the match does not correspond to a valid date or datetime.
|
172
|
+
"""
|
173
|
+
(
|
174
|
+
year_str,
|
175
|
+
month_str,
|
176
|
+
day_str,
|
177
|
+
hour_str,
|
178
|
+
minute_str,
|
179
|
+
sec_str,
|
180
|
+
micros_str,
|
181
|
+
zulu_time,
|
182
|
+
offset_sign_str,
|
183
|
+
offset_hour_str,
|
184
|
+
offset_minute_str,
|
185
|
+
) = match.groups()
|
186
|
+
year, month, day = int(year_str), int(month_str), int(day_str)
|
187
|
+
if hour_str is None:
|
188
|
+
return datetime.date(year, month, day)
|
189
|
+
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
190
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
191
|
+
if offset_sign_str:
|
192
|
+
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
193
|
+
offset_hour_str, offset_minute_str, offset_sign_str,
|
194
|
+
)
|
195
|
+
elif zulu_time:
|
196
|
+
tz = datetime.UTC
|
197
|
+
else: # local date-time
|
198
|
+
tz = None
|
199
|
+
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
200
|
+
|
201
|
+
|
202
|
+
@functools.lru_cache() # noqa
|
203
|
+
def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
|
204
|
+
sign = 1 if sign_str == '+' else -1
|
205
|
+
return datetime.timezone(
|
206
|
+
datetime.timedelta(
|
207
|
+
hours=sign * int(hour_str),
|
208
|
+
minutes=sign * int(minute_str),
|
209
|
+
),
|
210
|
+
)
|
211
|
+
|
212
|
+
|
213
|
+
def toml_match_to_localtime(match: re.Match) -> datetime.time:
|
214
|
+
hour_str, minute_str, sec_str, micros_str = match.groups()
|
215
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
216
|
+
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
217
|
+
|
218
|
+
|
219
|
+
def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
|
220
|
+
if match.group('floatpart'):
|
221
|
+
return parse_float(match.group())
|
222
|
+
return int(match.group(), 0)
|
223
|
+
|
224
|
+
|
225
|
+
TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
|
226
|
+
|
227
|
+
# Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
|
228
|
+
# functions.
|
229
|
+
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
230
|
+
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
231
|
+
|
232
|
+
TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
233
|
+
TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
234
|
+
|
235
|
+
TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
236
|
+
|
237
|
+
TOML_WS = frozenset(' \t')
|
238
|
+
TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
|
239
|
+
TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
|
240
|
+
TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
|
241
|
+
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
242
|
+
|
243
|
+
TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
|
244
|
+
{
|
245
|
+
'\\b': '\u0008', # backspace
|
246
|
+
'\\t': '\u0009', # tab
|
247
|
+
'\\n': '\u000A', # linefeed
|
248
|
+
'\\f': '\u000C', # form feed
|
249
|
+
'\\r': '\u000D', # carriage return
|
250
|
+
'\\"': '\u0022', # quote
|
251
|
+
'\\\\': '\u005C', # backslash
|
252
|
+
},
|
253
|
+
)
|
254
|
+
|
255
|
+
|
256
|
+
class TomlDecodeError(ValueError):
|
257
|
+
"""An error raised if a document is not valid TOML."""
|
258
|
+
|
259
|
+
|
260
|
+
def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
|
261
|
+
"""Parse TOML from a binary file object."""
|
262
|
+
b = fp.read()
|
263
|
+
try:
|
264
|
+
s = b.decode()
|
265
|
+
except AttributeError:
|
266
|
+
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
267
|
+
return toml_loads(s, parse_float=parse_float)
|
268
|
+
|
269
|
+
|
270
|
+
def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
|
271
|
+
"""Parse TOML from a string."""
|
272
|
+
|
273
|
+
# The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
|
274
|
+
try:
|
275
|
+
src = s.replace('\r\n', '\n')
|
276
|
+
except (AttributeError, TypeError):
|
277
|
+
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
278
|
+
pos = 0
|
279
|
+
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
280
|
+
header: TomlKey = ()
|
281
|
+
parse_float = toml_make_safe_parse_float(parse_float)
|
282
|
+
|
283
|
+
# Parse one statement at a time (typically means one line in TOML source)
|
284
|
+
while True:
|
285
|
+
# 1. Skip line leading whitespace
|
286
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
287
|
+
|
288
|
+
# 2. Parse rules. Expect one of the following:
|
289
|
+
# - end of file
|
290
|
+
# - end of line
|
291
|
+
# - comment
|
292
|
+
# - key/value pair
|
293
|
+
# - append dict to list (and move to its namespace)
|
294
|
+
# - create dict (and move to its namespace)
|
295
|
+
# Skip trailing whitespace when applicable.
|
296
|
+
try:
|
297
|
+
char = src[pos]
|
298
|
+
except IndexError:
|
299
|
+
break
|
300
|
+
if char == '\n':
|
301
|
+
pos += 1
|
302
|
+
continue
|
303
|
+
if char in TOML_KEY_INITIAL_CHARS:
|
304
|
+
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
305
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
306
|
+
elif char == '[':
|
307
|
+
try:
|
308
|
+
second_char: ta.Optional[str] = src[pos + 1]
|
309
|
+
except IndexError:
|
310
|
+
second_char = None
|
311
|
+
out.flags.finalize_pending()
|
312
|
+
if second_char == '[':
|
313
|
+
pos, header = toml_create_list_rule(src, pos, out)
|
314
|
+
else:
|
315
|
+
pos, header = toml_create_dict_rule(src, pos, out)
|
316
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
317
|
+
elif char != '#':
|
318
|
+
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
319
|
+
|
320
|
+
# 3. Skip comment
|
321
|
+
pos = toml_skip_comment(src, pos)
|
322
|
+
|
323
|
+
# 4. Expect end of line or end of file
|
324
|
+
try:
|
325
|
+
char = src[pos]
|
326
|
+
except IndexError:
|
327
|
+
break
|
328
|
+
if char != '\n':
|
329
|
+
raise toml_suffixed_err(
|
330
|
+
src, pos, 'Expected newline or end of document after a statement',
|
331
|
+
)
|
332
|
+
pos += 1
|
333
|
+
|
334
|
+
return out.data.dict
|
335
|
+
|
336
|
+
|
337
|
+
class TomlFlags:
|
338
|
+
"""Flags that map to parsed keys/namespaces."""
|
339
|
+
|
340
|
+
# Marks an immutable namespace (inline array or inline table).
|
341
|
+
FROZEN = 0
|
342
|
+
# Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
|
343
|
+
EXPLICIT_NEST = 1
|
344
|
+
|
345
|
+
def __init__(self) -> None:
|
346
|
+
self._flags: ta.Dict[str, dict] = {}
|
347
|
+
self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
|
348
|
+
|
349
|
+
def add_pending(self, key: TomlKey, flag: int) -> None:
|
350
|
+
self._pending_flags.add((key, flag))
|
351
|
+
|
352
|
+
def finalize_pending(self) -> None:
|
353
|
+
for key, flag in self._pending_flags:
|
354
|
+
self.set(key, flag, recursive=False)
|
355
|
+
self._pending_flags.clear()
|
356
|
+
|
357
|
+
def unset_all(self, key: TomlKey) -> None:
|
358
|
+
cont = self._flags
|
359
|
+
for k in key[:-1]:
|
360
|
+
if k not in cont:
|
361
|
+
return
|
362
|
+
cont = cont[k]['nested']
|
363
|
+
cont.pop(key[-1], None)
|
364
|
+
|
365
|
+
def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
|
366
|
+
cont = self._flags
|
367
|
+
key_parent, key_stem = key[:-1], key[-1]
|
368
|
+
for k in key_parent:
|
369
|
+
if k not in cont:
|
370
|
+
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
371
|
+
cont = cont[k]['nested']
|
372
|
+
if key_stem not in cont:
|
373
|
+
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
374
|
+
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
375
|
+
|
376
|
+
def is_(self, key: TomlKey, flag: int) -> bool:
|
377
|
+
if not key:
|
378
|
+
return False # document root has no flags
|
379
|
+
cont = self._flags
|
380
|
+
for k in key[:-1]:
|
381
|
+
if k not in cont:
|
382
|
+
return False
|
383
|
+
inner_cont = cont[k]
|
384
|
+
if flag in inner_cont['recursive_flags']:
|
385
|
+
return True
|
386
|
+
cont = inner_cont['nested']
|
387
|
+
key_stem = key[-1]
|
388
|
+
if key_stem in cont:
|
389
|
+
cont = cont[key_stem]
|
390
|
+
return flag in cont['flags'] or flag in cont['recursive_flags']
|
391
|
+
return False
|
392
|
+
|
393
|
+
|
394
|
+
class TomlNestedDict:
|
395
|
+
def __init__(self) -> None:
|
396
|
+
# The parsed content of the TOML document
|
397
|
+
self.dict: ta.Dict[str, ta.Any] = {}
|
398
|
+
|
399
|
+
def get_or_create_nest(
|
400
|
+
self,
|
401
|
+
key: TomlKey,
|
402
|
+
*,
|
403
|
+
access_lists: bool = True,
|
404
|
+
) -> dict:
|
405
|
+
cont: ta.Any = self.dict
|
406
|
+
for k in key:
|
407
|
+
if k not in cont:
|
408
|
+
cont[k] = {}
|
409
|
+
cont = cont[k]
|
410
|
+
if access_lists and isinstance(cont, list):
|
411
|
+
cont = cont[-1]
|
412
|
+
if not isinstance(cont, dict):
|
413
|
+
raise KeyError('There is no nest behind this key')
|
414
|
+
return cont
|
415
|
+
|
416
|
+
def append_nest_to_list(self, key: TomlKey) -> None:
|
417
|
+
cont = self.get_or_create_nest(key[:-1])
|
418
|
+
last_key = key[-1]
|
419
|
+
if last_key in cont:
|
420
|
+
list_ = cont[last_key]
|
421
|
+
if not isinstance(list_, list):
|
422
|
+
raise KeyError('An object other than list found behind this key')
|
423
|
+
list_.append({})
|
424
|
+
else:
|
425
|
+
cont[last_key] = [{}]
|
426
|
+
|
427
|
+
|
428
|
+
class TomlOutput(ta.NamedTuple):
|
429
|
+
data: TomlNestedDict
|
430
|
+
flags: TomlFlags
|
431
|
+
|
432
|
+
|
433
|
+
def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
|
434
|
+
try:
|
435
|
+
while src[pos] in chars:
|
436
|
+
pos += 1
|
437
|
+
except IndexError:
|
438
|
+
pass
|
439
|
+
return pos
|
440
|
+
|
441
|
+
|
442
|
+
def toml_skip_until(
|
443
|
+
src: str,
|
444
|
+
pos: TomlPos,
|
445
|
+
expect: str,
|
446
|
+
*,
|
447
|
+
error_on: ta.FrozenSet[str],
|
448
|
+
error_on_eof: bool,
|
449
|
+
) -> TomlPos:
|
450
|
+
try:
|
451
|
+
new_pos = src.index(expect, pos)
|
452
|
+
except ValueError:
|
453
|
+
new_pos = len(src)
|
454
|
+
if error_on_eof:
|
455
|
+
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
456
|
+
|
457
|
+
if not error_on.isdisjoint(src[pos:new_pos]):
|
458
|
+
while src[pos] not in error_on:
|
459
|
+
pos += 1
|
460
|
+
raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
|
461
|
+
return new_pos
|
462
|
+
|
463
|
+
|
464
|
+
def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
|
465
|
+
try:
|
466
|
+
char: ta.Optional[str] = src[pos]
|
467
|
+
except IndexError:
|
468
|
+
char = None
|
469
|
+
if char == '#':
|
470
|
+
return toml_skip_until(
|
471
|
+
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
472
|
+
)
|
473
|
+
return pos
|
474
|
+
|
475
|
+
|
476
|
+
def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
|
477
|
+
while True:
|
478
|
+
pos_before_skip = pos
|
479
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
480
|
+
pos = toml_skip_comment(src, pos)
|
481
|
+
if pos == pos_before_skip:
|
482
|
+
return pos
|
483
|
+
|
484
|
+
|
485
|
+
def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
486
|
+
pos += 1 # Skip "["
|
487
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
488
|
+
pos, key = toml_parse_key(src, pos)
|
489
|
+
|
490
|
+
if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
|
491
|
+
raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
|
492
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
493
|
+
try:
|
494
|
+
out.data.get_or_create_nest(key)
|
495
|
+
except KeyError:
|
496
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
497
|
+
|
498
|
+
if not src.startswith(']', pos):
|
499
|
+
raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
|
500
|
+
return pos + 1, key
|
501
|
+
|
502
|
+
|
503
|
+
def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
504
|
+
pos += 2 # Skip "[["
|
505
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
506
|
+
pos, key = toml_parse_key(src, pos)
|
507
|
+
|
508
|
+
if out.flags.is_(key, TomlFlags.FROZEN):
|
509
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
510
|
+
# Free the namespace now that it points to another empty list item...
|
511
|
+
out.flags.unset_all(key)
|
512
|
+
# ...but this key precisely is still prohibited from table declaration
|
513
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
514
|
+
try:
|
515
|
+
out.data.append_nest_to_list(key)
|
516
|
+
except KeyError:
|
517
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
518
|
+
|
519
|
+
if not src.startswith(']]', pos):
|
520
|
+
raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
|
521
|
+
return pos + 2, key
|
522
|
+
|
523
|
+
|
524
|
+
def toml_key_value_rule(
|
525
|
+
src: str,
|
526
|
+
pos: TomlPos,
|
527
|
+
out: TomlOutput,
|
528
|
+
header: TomlKey,
|
529
|
+
parse_float: TomlParseFloat,
|
530
|
+
) -> TomlPos:
|
531
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
532
|
+
key_parent, key_stem = key[:-1], key[-1]
|
533
|
+
abs_key_parent = header + key_parent
|
534
|
+
|
535
|
+
relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
|
536
|
+
for cont_key in relative_path_cont_keys:
|
537
|
+
# Check that dotted key syntax does not redefine an existing table
|
538
|
+
if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
|
539
|
+
raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
|
540
|
+
# Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
|
541
|
+
# table sections.
|
542
|
+
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
543
|
+
|
544
|
+
if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
|
545
|
+
raise toml_suffixed_err(
|
546
|
+
src,
|
547
|
+
pos,
|
548
|
+
f'Cannot mutate immutable namespace {abs_key_parent}',
|
549
|
+
)
|
550
|
+
|
551
|
+
try:
|
552
|
+
nest = out.data.get_or_create_nest(abs_key_parent)
|
553
|
+
except KeyError:
|
554
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
555
|
+
if key_stem in nest:
|
556
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
557
|
+
# Mark inline table and array namespaces recursively immutable
|
558
|
+
if isinstance(value, (dict, list)):
|
559
|
+
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
560
|
+
nest[key_stem] = value
|
561
|
+
return pos
|
562
|
+
|
563
|
+
|
564
|
+
def toml_parse_key_value_pair(
|
565
|
+
src: str,
|
566
|
+
pos: TomlPos,
|
567
|
+
parse_float: TomlParseFloat,
|
568
|
+
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
569
|
+
pos, key = toml_parse_key(src, pos)
|
570
|
+
try:
|
571
|
+
char: ta.Optional[str] = src[pos]
|
572
|
+
except IndexError:
|
573
|
+
char = None
|
574
|
+
if char != '=':
|
575
|
+
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
576
|
+
pos += 1
|
577
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
578
|
+
pos, value = toml_parse_value(src, pos, parse_float)
|
579
|
+
return pos, key, value
|
580
|
+
|
581
|
+
|
582
|
+
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
583
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
584
|
+
key: TomlKey = (key_part,)
|
585
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
586
|
+
while True:
|
587
|
+
try:
|
588
|
+
char: ta.Optional[str] = src[pos]
|
589
|
+
except IndexError:
|
590
|
+
char = None
|
591
|
+
if char != '.':
|
592
|
+
return pos, key
|
593
|
+
pos += 1
|
594
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
595
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
596
|
+
key += (key_part,)
|
597
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
598
|
+
|
599
|
+
|
600
|
+
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
601
|
+
try:
|
602
|
+
char: ta.Optional[str] = src[pos]
|
603
|
+
except IndexError:
|
604
|
+
char = None
|
605
|
+
if char in TOML_BARE_KEY_CHARS:
|
606
|
+
start_pos = pos
|
607
|
+
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
608
|
+
return pos, src[start_pos:pos]
|
609
|
+
if char == "'":
|
610
|
+
return toml_parse_literal_str(src, pos)
|
611
|
+
if char == '"':
|
612
|
+
return toml_parse_one_line_basic_str(src, pos)
|
613
|
+
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
614
|
+
|
615
|
+
|
616
|
+
def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
617
|
+
pos += 1
|
618
|
+
return toml_parse_basic_str(src, pos, multiline=False)
|
619
|
+
|
620
|
+
|
621
|
+
def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
|
622
|
+
pos += 1
|
623
|
+
array: list = []
|
624
|
+
|
625
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
626
|
+
if src.startswith(']', pos):
|
627
|
+
return pos + 1, array
|
628
|
+
while True:
|
629
|
+
pos, val = toml_parse_value(src, pos, parse_float)
|
630
|
+
array.append(val)
|
631
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
632
|
+
|
633
|
+
c = src[pos:pos + 1]
|
634
|
+
if c == ']':
|
635
|
+
return pos + 1, array
|
636
|
+
if c != ',':
|
637
|
+
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
638
|
+
pos += 1
|
639
|
+
|
640
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
641
|
+
if src.startswith(']', pos):
|
642
|
+
return pos + 1, array
|
643
|
+
|
644
|
+
|
645
|
+
def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
|
646
|
+
pos += 1
|
647
|
+
nested_dict = TomlNestedDict()
|
648
|
+
flags = TomlFlags()
|
649
|
+
|
650
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
651
|
+
if src.startswith('}', pos):
|
652
|
+
return pos + 1, nested_dict.dict
|
653
|
+
while True:
|
654
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
655
|
+
key_parent, key_stem = key[:-1], key[-1]
|
656
|
+
if flags.is_(key, TomlFlags.FROZEN):
|
657
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
658
|
+
try:
|
659
|
+
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
660
|
+
except KeyError:
|
661
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
662
|
+
if key_stem in nest:
|
663
|
+
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
664
|
+
nest[key_stem] = value
|
665
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
666
|
+
c = src[pos:pos + 1]
|
667
|
+
if c == '}':
|
668
|
+
return pos + 1, nested_dict.dict
|
669
|
+
if c != ',':
|
670
|
+
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
671
|
+
if isinstance(value, (dict, list)):
|
672
|
+
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
673
|
+
pos += 1
|
674
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
675
|
+
|
676
|
+
|
677
|
+
def toml_parse_basic_str_escape(
|
678
|
+
src: str,
|
679
|
+
pos: TomlPos,
|
680
|
+
*,
|
681
|
+
multiline: bool = False,
|
682
|
+
) -> ta.Tuple[TomlPos, str]:
|
683
|
+
escape_id = src[pos:pos + 2]
|
684
|
+
pos += 2
|
685
|
+
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
686
|
+
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
687
|
+
# newline.
|
688
|
+
if escape_id != '\\\n':
|
689
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
690
|
+
try:
|
691
|
+
char = src[pos]
|
692
|
+
except IndexError:
|
693
|
+
return pos, ''
|
694
|
+
if char != '\n':
|
695
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
696
|
+
pos += 1
|
697
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
698
|
+
return pos, ''
|
699
|
+
if escape_id == '\\u':
|
700
|
+
return toml_parse_hex_char(src, pos, 4)
|
701
|
+
if escape_id == '\\U':
|
702
|
+
return toml_parse_hex_char(src, pos, 8)
|
703
|
+
try:
|
704
|
+
return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
|
705
|
+
except KeyError:
|
706
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
707
|
+
|
708
|
+
|
709
|
+
def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
710
|
+
return toml_parse_basic_str_escape(src, pos, multiline=True)
|
711
|
+
|
712
|
+
|
713
|
+
def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
|
714
|
+
hex_str = src[pos:pos + hex_len]
|
715
|
+
if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
|
716
|
+
raise toml_suffixed_err(src, pos, 'Invalid hex value')
|
717
|
+
pos += hex_len
|
718
|
+
hex_int = int(hex_str, 16)
|
719
|
+
if not toml_is_unicode_scalar_value(hex_int):
|
720
|
+
raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
|
721
|
+
return pos, chr(hex_int)
|
722
|
+
|
723
|
+
|
724
|
+
def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
725
|
+
pos += 1 # Skip starting apostrophe
|
726
|
+
start_pos = pos
|
727
|
+
pos = toml_skip_until(
|
728
|
+
src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
|
729
|
+
)
|
730
|
+
return pos + 1, src[start_pos:pos] # Skip ending apostrophe
|
731
|
+
|
732
|
+
|
733
|
+
def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
|
734
|
+
pos += 3
|
735
|
+
if src.startswith('\n', pos):
|
736
|
+
pos += 1
|
737
|
+
|
738
|
+
if literal:
|
739
|
+
delim = "'"
|
740
|
+
end_pos = toml_skip_until(
|
741
|
+
src,
|
742
|
+
pos,
|
743
|
+
"'''",
|
744
|
+
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
745
|
+
error_on_eof=True,
|
746
|
+
)
|
747
|
+
result = src[pos:end_pos]
|
748
|
+
pos = end_pos + 3
|
749
|
+
else:
|
750
|
+
delim = '"'
|
751
|
+
pos, result = toml_parse_basic_str(src, pos, multiline=True)
|
752
|
+
|
753
|
+
# Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
|
754
|
+
if not src.startswith(delim, pos):
|
755
|
+
return pos, result
|
756
|
+
pos += 1
|
757
|
+
if not src.startswith(delim, pos):
|
758
|
+
return pos, result + delim
|
759
|
+
pos += 1
|
760
|
+
return pos, result + (delim * 2)
|
761
|
+
|
762
|
+
|
763
|
+
def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
|
764
|
+
if multiline:
|
765
|
+
error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
766
|
+
parse_escapes = toml_parse_basic_str_escape_multiline
|
767
|
+
else:
|
768
|
+
error_on = TOML_ILLEGAL_BASIC_STR_CHARS
|
769
|
+
parse_escapes = toml_parse_basic_str_escape
|
770
|
+
result = ''
|
771
|
+
start_pos = pos
|
772
|
+
while True:
|
773
|
+
try:
|
774
|
+
char = src[pos]
|
775
|
+
except IndexError:
|
776
|
+
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
777
|
+
if char == '"':
|
778
|
+
if not multiline:
|
779
|
+
return pos + 1, result + src[start_pos:pos]
|
780
|
+
if src.startswith('"""', pos):
|
781
|
+
return pos + 3, result + src[start_pos:pos]
|
782
|
+
pos += 1
|
783
|
+
continue
|
784
|
+
if char == '\\':
|
785
|
+
result += src[start_pos:pos]
|
786
|
+
pos, parsed_escape = parse_escapes(src, pos)
|
787
|
+
result += parsed_escape
|
788
|
+
start_pos = pos
|
789
|
+
continue
|
790
|
+
if char in error_on:
|
791
|
+
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
792
|
+
pos += 1
|
793
|
+
|
794
|
+
|
795
|
+
def toml_parse_value( # noqa: C901
|
796
|
+
src: str,
|
797
|
+
pos: TomlPos,
|
798
|
+
parse_float: TomlParseFloat,
|
799
|
+
) -> ta.Tuple[TomlPos, ta.Any]:
|
800
|
+
try:
|
801
|
+
char: ta.Optional[str] = src[pos]
|
802
|
+
except IndexError:
|
803
|
+
char = None
|
804
|
+
|
805
|
+
# IMPORTANT: order conditions based on speed of checking and likelihood
|
806
|
+
|
807
|
+
# Basic strings
|
808
|
+
if char == '"':
|
809
|
+
if src.startswith('"""', pos):
|
810
|
+
return toml_parse_multiline_str(src, pos, literal=False)
|
811
|
+
return toml_parse_one_line_basic_str(src, pos)
|
812
|
+
|
813
|
+
# Literal strings
|
814
|
+
if char == "'":
|
815
|
+
if src.startswith("'''", pos):
|
816
|
+
return toml_parse_multiline_str(src, pos, literal=True)
|
817
|
+
return toml_parse_literal_str(src, pos)
|
818
|
+
|
819
|
+
# Booleans
|
820
|
+
if char == 't':
|
821
|
+
if src.startswith('true', pos):
|
822
|
+
return pos + 4, True
|
823
|
+
if char == 'f':
|
824
|
+
if src.startswith('false', pos):
|
825
|
+
return pos + 5, False
|
826
|
+
|
827
|
+
# Arrays
|
828
|
+
if char == '[':
|
829
|
+
return toml_parse_array(src, pos, parse_float)
|
830
|
+
|
831
|
+
# Inline tables
|
832
|
+
if char == '{':
|
833
|
+
return toml_parse_inline_table(src, pos, parse_float)
|
834
|
+
|
835
|
+
# Dates and times
|
836
|
+
datetime_match = TOML_RE_DATETIME.match(src, pos)
|
837
|
+
if datetime_match:
|
838
|
+
try:
|
839
|
+
datetime_obj = toml_match_to_datetime(datetime_match)
|
840
|
+
except ValueError as e:
|
841
|
+
raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
|
842
|
+
return datetime_match.end(), datetime_obj
|
843
|
+
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
844
|
+
if localtime_match:
|
845
|
+
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
846
|
+
|
847
|
+
# Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
|
848
|
+
# located after handling of dates and times.
|
849
|
+
number_match = TOML_RE_NUMBER.match(src, pos)
|
850
|
+
if number_match:
|
851
|
+
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
852
|
+
|
853
|
+
# Special floats
|
854
|
+
first_three = src[pos:pos + 3]
|
855
|
+
if first_three in {'inf', 'nan'}:
|
856
|
+
return pos + 3, parse_float(first_three)
|
857
|
+
first_four = src[pos:pos + 4]
|
858
|
+
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
859
|
+
return pos + 4, parse_float(first_four)
|
860
|
+
|
861
|
+
raise toml_suffixed_err(src, pos, 'Invalid value')
|
862
|
+
|
863
|
+
|
864
|
+
def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
|
865
|
+
"""Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
|
866
|
+
|
867
|
+
def coord_repr(src: str, pos: TomlPos) -> str:
|
868
|
+
if pos >= len(src):
|
869
|
+
return 'end of document'
|
870
|
+
line = src.count('\n', 0, pos) + 1
|
871
|
+
if line == 1:
|
872
|
+
column = pos + 1
|
873
|
+
else:
|
874
|
+
column = pos - src.rindex('\n', 0, pos)
|
875
|
+
return f'line {line}, column {column}'
|
876
|
+
|
877
|
+
return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
|
878
|
+
|
879
|
+
|
880
|
+
def toml_is_unicode_scalar_value(codepoint: int) -> bool:
|
881
|
+
return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
|
882
|
+
|
883
|
+
|
884
|
+
def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
|
885
|
+
"""A decorator to make `parse_float` safe.
|
886
|
+
|
887
|
+
`parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
|
888
|
+
thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
|
889
|
+
"""
|
890
|
+
# The default `float` callable never returns illegal types. Optimize it.
|
891
|
+
if parse_float is float:
|
892
|
+
return float
|
893
|
+
|
894
|
+
def safe_parse_float(float_str: str) -> ta.Any:
|
895
|
+
float_value = parse_float(float_str)
|
896
|
+
if isinstance(float_value, (dict, list)):
|
897
|
+
raise ValueError('parse_float must not return dicts or lists') # noqa
|
898
|
+
return float_value
|
899
|
+
|
900
|
+
return safe_parse_float
|
901
|
+
|
64
902
|
|
65
903
|
########################################
|
66
904
|
# ../compat.py
|
@@ -216,6 +1054,19 @@ def close_fd(fd: int) -> bool:
|
|
216
1054
|
return True
|
217
1055
|
|
218
1056
|
|
1057
|
+
def is_fd_open(fd: int) -> bool:
|
1058
|
+
try:
|
1059
|
+
n = os.dup(fd)
|
1060
|
+
except OSError:
|
1061
|
+
return False
|
1062
|
+
os.close(n)
|
1063
|
+
return True
|
1064
|
+
|
1065
|
+
|
1066
|
+
def get_open_fds(limit: int) -> ta.FrozenSet[int]:
|
1067
|
+
return frozenset(filter(is_fd_open, range(limit)))
|
1068
|
+
|
1069
|
+
|
219
1070
|
def mktempfile(suffix: str, prefix: str, dir: str) -> str: # noqa
|
220
1071
|
fd, filename = tempfile.mkstemp(suffix, prefix, dir)
|
221
1072
|
os.close(fd)
|
@@ -490,7 +1341,7 @@ class _cached_nullary: # noqa
|
|
490
1341
|
return bound
|
491
1342
|
|
492
1343
|
|
493
|
-
def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
|
1344
|
+
def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
494
1345
|
return _cached_nullary(fn)
|
495
1346
|
|
496
1347
|
|
@@ -581,6 +1432,50 @@ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON
|
|
581
1432
|
json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
|
582
1433
|
|
583
1434
|
|
1435
|
+
########################################
|
1436
|
+
# ../../../omlish/lite/maybes.py
|
1437
|
+
|
1438
|
+
|
1439
|
+
class Maybe(ta.Generic[T]):
|
1440
|
+
@property
|
1441
|
+
@abc.abstractmethod
|
1442
|
+
def present(self) -> bool:
|
1443
|
+
raise NotImplementedError
|
1444
|
+
|
1445
|
+
@abc.abstractmethod
|
1446
|
+
def must(self) -> T:
|
1447
|
+
raise NotImplementedError
|
1448
|
+
|
1449
|
+
@classmethod
|
1450
|
+
def just(cls, v: T) -> 'Maybe[T]':
|
1451
|
+
return tuple.__new__(_Maybe, (v,)) # noqa
|
1452
|
+
|
1453
|
+
_empty: ta.ClassVar['Maybe']
|
1454
|
+
|
1455
|
+
@classmethod
|
1456
|
+
def empty(cls) -> 'Maybe[T]':
|
1457
|
+
return Maybe._empty
|
1458
|
+
|
1459
|
+
|
1460
|
+
class _Maybe(Maybe[T], tuple):
|
1461
|
+
__slots__ = ()
|
1462
|
+
|
1463
|
+
def __init_subclass__(cls, **kwargs):
|
1464
|
+
raise TypeError
|
1465
|
+
|
1466
|
+
@property
|
1467
|
+
def present(self) -> bool:
|
1468
|
+
return bool(self)
|
1469
|
+
|
1470
|
+
def must(self) -> T:
|
1471
|
+
if not self:
|
1472
|
+
raise ValueError
|
1473
|
+
return self[0]
|
1474
|
+
|
1475
|
+
|
1476
|
+
Maybe._empty = tuple.__new__(_Maybe, ()) # noqa
|
1477
|
+
|
1478
|
+
|
584
1479
|
########################################
|
585
1480
|
# ../../../omlish/lite/reflect.py
|
586
1481
|
|
@@ -616,6 +1511,14 @@ def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
|
|
616
1511
|
return it
|
617
1512
|
|
618
1513
|
|
1514
|
+
def is_new_type(spec: ta.Any) -> bool:
|
1515
|
+
if isinstance(ta.NewType, type):
|
1516
|
+
return isinstance(spec, ta.NewType)
|
1517
|
+
else:
|
1518
|
+
# Before https://github.com/python/cpython/commit/c2f33dfc83ab270412bf243fb21f724037effa1a
|
1519
|
+
return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1] # type: ignore # noqa
|
1520
|
+
|
1521
|
+
|
619
1522
|
def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
|
620
1523
|
seen = set()
|
621
1524
|
todo = list(reversed(cls.__subclasses__()))
|
@@ -629,179 +1532,685 @@ def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
|
|
629
1532
|
|
630
1533
|
|
631
1534
|
########################################
|
632
|
-
# ../
|
1535
|
+
# ../states.py
|
1536
|
+
|
1537
|
+
|
1538
|
+
##
|
1539
|
+
|
1540
|
+
|
1541
|
+
def _names_by_code(states: ta.Any) -> ta.Dict[int, str]:
|
1542
|
+
d = {}
|
1543
|
+
for name in states.__dict__:
|
1544
|
+
if not name.startswith('__'):
|
1545
|
+
code = getattr(states, name)
|
1546
|
+
d[code] = name
|
1547
|
+
return d
|
1548
|
+
|
1549
|
+
|
1550
|
+
##
|
1551
|
+
|
1552
|
+
|
1553
|
+
class ProcessStates:
|
1554
|
+
STOPPED = 0
|
1555
|
+
STARTING = 10
|
1556
|
+
RUNNING = 20
|
1557
|
+
BACKOFF = 30
|
1558
|
+
STOPPING = 40
|
1559
|
+
EXITED = 100
|
1560
|
+
FATAL = 200
|
1561
|
+
UNKNOWN = 1000
|
1562
|
+
|
1563
|
+
|
1564
|
+
STOPPED_STATES = (
|
1565
|
+
ProcessStates.STOPPED,
|
1566
|
+
ProcessStates.EXITED,
|
1567
|
+
ProcessStates.FATAL,
|
1568
|
+
ProcessStates.UNKNOWN,
|
1569
|
+
)
|
1570
|
+
|
1571
|
+
RUNNING_STATES = (
|
1572
|
+
ProcessStates.RUNNING,
|
1573
|
+
ProcessStates.BACKOFF,
|
1574
|
+
ProcessStates.STARTING,
|
1575
|
+
)
|
1576
|
+
|
1577
|
+
SIGNALLABLE_STATES = (
|
1578
|
+
ProcessStates.RUNNING,
|
1579
|
+
ProcessStates.STARTING,
|
1580
|
+
ProcessStates.STOPPING,
|
1581
|
+
)
|
1582
|
+
|
1583
|
+
|
1584
|
+
_process_states_by_code = _names_by_code(ProcessStates)
|
1585
|
+
|
1586
|
+
|
1587
|
+
def get_process_state_description(code: ProcessState) -> str:
|
1588
|
+
return check_not_none(_process_states_by_code.get(code))
|
1589
|
+
|
1590
|
+
|
1591
|
+
##
|
1592
|
+
|
1593
|
+
|
1594
|
+
class SupervisorStates:
|
1595
|
+
FATAL = 2
|
1596
|
+
RUNNING = 1
|
1597
|
+
RESTARTING = 0
|
1598
|
+
SHUTDOWN = -1
|
1599
|
+
|
1600
|
+
|
1601
|
+
_supervisor_states_by_code = _names_by_code(SupervisorStates)
|
1602
|
+
|
1603
|
+
|
1604
|
+
def get_supervisor_state_description(code: SupervisorState) -> str:
|
1605
|
+
return check_not_none(_supervisor_states_by_code.get(code))
|
1606
|
+
|
1607
|
+
|
1608
|
+
########################################
|
1609
|
+
# ../../../omlish/lite/inject.py
|
1610
|
+
|
1611
|
+
|
1612
|
+
###
|
1613
|
+
# types
|
633
1614
|
|
634
1615
|
|
635
1616
|
@dc.dataclass(frozen=True)
|
636
|
-
class
|
637
|
-
|
638
|
-
|
1617
|
+
class InjectorKey:
|
1618
|
+
cls: InjectorKeyCls
|
1619
|
+
tag: ta.Any = None
|
1620
|
+
array: bool = False
|
639
1621
|
|
640
|
-
uid: ta.Optional[int] = None
|
641
|
-
directory: ta.Optional[str] = None
|
642
|
-
umask: ta.Optional[int] = None
|
643
|
-
priority: int = 999
|
644
1622
|
|
645
|
-
|
646
|
-
autorestart: str = 'unexpected'
|
1623
|
+
##
|
647
1624
|
|
648
|
-
startsecs: int = 1
|
649
|
-
startretries: int = 3
|
650
1625
|
|
651
|
-
|
652
|
-
|
1626
|
+
class InjectorProvider(abc.ABC):
|
1627
|
+
@abc.abstractmethod
|
1628
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1629
|
+
raise NotImplementedError
|
653
1630
|
|
654
|
-
@dc.dataclass(frozen=True)
|
655
|
-
class Log:
|
656
|
-
file: ta.Optional[str] = None
|
657
|
-
capture_maxbytes: ta.Optional[int] = None
|
658
|
-
events_enabled: bool = False
|
659
|
-
syslog: bool = False
|
660
|
-
backups: ta.Optional[int] = None
|
661
|
-
maxbytes: ta.Optional[int] = None
|
662
1631
|
|
663
|
-
|
664
|
-
stderr: Log = Log()
|
1632
|
+
##
|
665
1633
|
|
666
|
-
stopsignal: int = signal.SIGTERM
|
667
|
-
stopwaitsecs: int = 10
|
668
|
-
stopasgroup: bool = False
|
669
1634
|
|
670
|
-
|
1635
|
+
@dc.dataclass(frozen=True)
|
1636
|
+
class InjectorBinding:
|
1637
|
+
key: InjectorKey
|
1638
|
+
provider: InjectorProvider
|
671
1639
|
|
672
|
-
exitcodes: ta.Sequence[int] = (0,)
|
673
1640
|
|
674
|
-
|
1641
|
+
class InjectorBindings(abc.ABC):
|
1642
|
+
@abc.abstractmethod
|
1643
|
+
def bindings(self) -> ta.Iterator[InjectorBinding]:
|
1644
|
+
raise NotImplementedError
|
675
1645
|
|
676
|
-
|
1646
|
+
##
|
1647
|
+
|
1648
|
+
|
1649
|
+
class Injector(abc.ABC):
|
1650
|
+
@abc.abstractmethod
|
1651
|
+
def try_provide(self, key: ta.Any) -> Maybe[ta.Any]:
|
1652
|
+
raise NotImplementedError
|
1653
|
+
|
1654
|
+
@abc.abstractmethod
|
1655
|
+
def provide(self, key: ta.Any) -> ta.Any:
|
1656
|
+
raise NotImplementedError
|
1657
|
+
|
1658
|
+
@abc.abstractmethod
|
1659
|
+
def provide_kwargs(self, obj: ta.Any) -> ta.Mapping[str, ta.Any]:
|
1660
|
+
raise NotImplementedError
|
1661
|
+
|
1662
|
+
@abc.abstractmethod
|
1663
|
+
def inject(self, obj: ta.Any) -> ta.Any:
|
1664
|
+
raise NotImplementedError
|
1665
|
+
|
1666
|
+
|
1667
|
+
###
|
1668
|
+
# exceptions
|
677
1669
|
|
678
1670
|
|
679
1671
|
@dc.dataclass(frozen=True)
|
680
|
-
class
|
681
|
-
|
1672
|
+
class InjectorKeyError(Exception):
|
1673
|
+
key: InjectorKey
|
682
1674
|
|
683
|
-
|
1675
|
+
source: ta.Any = None
|
1676
|
+
name: ta.Optional[str] = None
|
684
1677
|
|
685
|
-
|
1678
|
+
|
1679
|
+
@dc.dataclass(frozen=True)
|
1680
|
+
class UnboundInjectorKeyError(InjectorKeyError):
|
1681
|
+
pass
|
686
1682
|
|
687
1683
|
|
688
1684
|
@dc.dataclass(frozen=True)
|
689
|
-
class
|
690
|
-
|
691
|
-
nodaemon: bool = False
|
692
|
-
umask: int = 0o22
|
693
|
-
directory: ta.Optional[str] = None
|
694
|
-
logfile: str = 'supervisord.log'
|
695
|
-
logfile_maxbytes: int = 50 * 1024 * 1024
|
696
|
-
logfile_backups: int = 10
|
697
|
-
loglevel: int = logging.INFO
|
698
|
-
pidfile: str = 'supervisord.pid'
|
699
|
-
identifier: str = 'supervisor'
|
700
|
-
child_logdir: str = '/dev/null'
|
701
|
-
minfds: int = 1024
|
702
|
-
minprocs: int = 200
|
703
|
-
nocleanup: bool = False
|
704
|
-
strip_ansi: bool = False
|
705
|
-
silent: bool = False
|
1685
|
+
class DuplicateInjectorKeyError(InjectorKeyError):
|
1686
|
+
pass
|
706
1687
|
|
707
|
-
|
1688
|
+
|
1689
|
+
###
|
1690
|
+
# keys
|
1691
|
+
|
1692
|
+
|
1693
|
+
def as_injector_key(o: ta.Any) -> InjectorKey:
|
1694
|
+
if o is inspect.Parameter.empty:
|
1695
|
+
raise TypeError(o)
|
1696
|
+
if isinstance(o, InjectorKey):
|
1697
|
+
return o
|
1698
|
+
if isinstance(o, type) or is_new_type(o):
|
1699
|
+
return InjectorKey(o)
|
1700
|
+
raise TypeError(o)
|
1701
|
+
|
1702
|
+
|
1703
|
+
###
|
1704
|
+
# providers
|
1705
|
+
|
1706
|
+
|
1707
|
+
@dc.dataclass(frozen=True)
|
1708
|
+
class FnInjectorProvider(InjectorProvider):
|
1709
|
+
fn: ta.Any
|
1710
|
+
|
1711
|
+
def __post_init__(self) -> None:
|
1712
|
+
check_not_isinstance(self.fn, type)
|
1713
|
+
|
1714
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1715
|
+
def pfn(i: Injector) -> ta.Any:
|
1716
|
+
return i.inject(self.fn)
|
1717
|
+
|
1718
|
+
return pfn
|
1719
|
+
|
1720
|
+
|
1721
|
+
@dc.dataclass(frozen=True)
|
1722
|
+
class CtorInjectorProvider(InjectorProvider):
|
1723
|
+
cls: type
|
1724
|
+
|
1725
|
+
def __post_init__(self) -> None:
|
1726
|
+
check_isinstance(self.cls, type)
|
1727
|
+
|
1728
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1729
|
+
def pfn(i: Injector) -> ta.Any:
|
1730
|
+
return i.inject(self.cls)
|
1731
|
+
|
1732
|
+
return pfn
|
1733
|
+
|
1734
|
+
|
1735
|
+
@dc.dataclass(frozen=True)
|
1736
|
+
class ConstInjectorProvider(InjectorProvider):
|
1737
|
+
v: ta.Any
|
1738
|
+
|
1739
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1740
|
+
return lambda _: self.v
|
1741
|
+
|
1742
|
+
|
1743
|
+
@dc.dataclass(frozen=True)
|
1744
|
+
class SingletonInjectorProvider(InjectorProvider):
|
1745
|
+
p: InjectorProvider
|
1746
|
+
|
1747
|
+
def __post_init__(self) -> None:
|
1748
|
+
check_isinstance(self.p, InjectorProvider)
|
1749
|
+
|
1750
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1751
|
+
v = not_set = object()
|
1752
|
+
|
1753
|
+
def pfn(i: Injector) -> ta.Any:
|
1754
|
+
nonlocal v
|
1755
|
+
if v is not_set:
|
1756
|
+
v = ufn(i)
|
1757
|
+
return v
|
1758
|
+
|
1759
|
+
ufn = self.p.provider_fn()
|
1760
|
+
return pfn
|
1761
|
+
|
1762
|
+
|
1763
|
+
@dc.dataclass(frozen=True)
|
1764
|
+
class LinkInjectorProvider(InjectorProvider):
|
1765
|
+
k: InjectorKey
|
1766
|
+
|
1767
|
+
def __post_init__(self) -> None:
|
1768
|
+
check_isinstance(self.k, InjectorKey)
|
1769
|
+
|
1770
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1771
|
+
def pfn(i: Injector) -> ta.Any:
|
1772
|
+
return i.provide(self.k)
|
1773
|
+
|
1774
|
+
return pfn
|
1775
|
+
|
1776
|
+
|
1777
|
+
@dc.dataclass(frozen=True)
|
1778
|
+
class ArrayInjectorProvider(InjectorProvider):
|
1779
|
+
ps: ta.Sequence[InjectorProvider]
|
1780
|
+
|
1781
|
+
def provider_fn(self) -> InjectorProviderFn:
|
1782
|
+
ps = [p.provider_fn() for p in self.ps]
|
1783
|
+
|
1784
|
+
def pfn(i: Injector) -> ta.Any:
|
1785
|
+
rv = []
|
1786
|
+
for ep in ps:
|
1787
|
+
o = ep(i)
|
1788
|
+
rv.append(o)
|
1789
|
+
return rv
|
1790
|
+
|
1791
|
+
return pfn
|
1792
|
+
|
1793
|
+
|
1794
|
+
###
|
1795
|
+
# bindings
|
1796
|
+
|
1797
|
+
|
1798
|
+
@dc.dataclass(frozen=True)
|
1799
|
+
class _InjectorBindings(InjectorBindings):
|
1800
|
+
bs: ta.Optional[ta.Sequence[InjectorBinding]] = None
|
1801
|
+
ps: ta.Optional[ta.Sequence[InjectorBindings]] = None
|
1802
|
+
|
1803
|
+
def bindings(self) -> ta.Iterator[InjectorBinding]:
|
1804
|
+
if self.bs is not None:
|
1805
|
+
yield from self.bs
|
1806
|
+
if self.ps is not None:
|
1807
|
+
for p in self.ps:
|
1808
|
+
yield from p.bindings()
|
1809
|
+
|
1810
|
+
|
1811
|
+
def as_injector_bindings(*args: InjectorBindingOrBindings) -> InjectorBindings:
|
1812
|
+
bs: ta.List[InjectorBinding] = []
|
1813
|
+
ps: ta.List[InjectorBindings] = []
|
1814
|
+
|
1815
|
+
for a in args:
|
1816
|
+
if isinstance(a, InjectorBindings):
|
1817
|
+
ps.append(a)
|
1818
|
+
elif isinstance(a, InjectorBinding):
|
1819
|
+
bs.append(a)
|
1820
|
+
else:
|
1821
|
+
raise TypeError(a)
|
1822
|
+
|
1823
|
+
return _InjectorBindings(
|
1824
|
+
bs or None,
|
1825
|
+
ps or None,
|
1826
|
+
)
|
1827
|
+
|
1828
|
+
|
1829
|
+
##
|
1830
|
+
|
1831
|
+
|
1832
|
+
@dc.dataclass(frozen=True)
|
1833
|
+
class OverridesInjectorBindings(InjectorBindings):
|
1834
|
+
p: InjectorBindings
|
1835
|
+
m: ta.Mapping[InjectorKey, InjectorBinding]
|
1836
|
+
|
1837
|
+
def bindings(self) -> ta.Iterator[InjectorBinding]:
|
1838
|
+
for b in self.p.bindings():
|
1839
|
+
yield self.m.get(b.key, b)
|
1840
|
+
|
1841
|
+
|
1842
|
+
def injector_override(p: InjectorBindings, *args: InjectorBindingOrBindings) -> InjectorBindings:
|
1843
|
+
m: ta.Dict[InjectorKey, InjectorBinding] = {}
|
1844
|
+
|
1845
|
+
for b in as_injector_bindings(*args).bindings():
|
1846
|
+
if b.key in m:
|
1847
|
+
raise DuplicateInjectorKeyError(b.key)
|
1848
|
+
m[b.key] = b
|
1849
|
+
|
1850
|
+
return OverridesInjectorBindings(p, m)
|
1851
|
+
|
1852
|
+
|
1853
|
+
##
|
1854
|
+
|
1855
|
+
|
1856
|
+
def build_injector_provider_map(bs: InjectorBindings) -> ta.Mapping[InjectorKey, InjectorProvider]:
|
1857
|
+
pm: ta.Dict[InjectorKey, InjectorProvider] = {}
|
1858
|
+
am: ta.Dict[InjectorKey, ta.List[InjectorProvider]] = {}
|
1859
|
+
|
1860
|
+
for b in bs.bindings():
|
1861
|
+
if b.key.array:
|
1862
|
+
am.setdefault(b.key, []).append(b.provider)
|
1863
|
+
else:
|
1864
|
+
if b.key in pm:
|
1865
|
+
raise KeyError(b.key)
|
1866
|
+
pm[b.key] = b.provider
|
1867
|
+
|
1868
|
+
if am:
|
1869
|
+
for k, aps in am.items():
|
1870
|
+
pm[k] = ArrayInjectorProvider(aps)
|
1871
|
+
|
1872
|
+
return pm
|
1873
|
+
|
1874
|
+
|
1875
|
+
###
|
1876
|
+
# inspection
|
1877
|
+
|
1878
|
+
|
1879
|
+
_INJECTION_SIGNATURE_CACHE: ta.MutableMapping[ta.Any, inspect.Signature] = weakref.WeakKeyDictionary()
|
1880
|
+
|
1881
|
+
|
1882
|
+
def _injection_signature(obj: ta.Any) -> inspect.Signature:
|
1883
|
+
try:
|
1884
|
+
return _INJECTION_SIGNATURE_CACHE[obj]
|
1885
|
+
except TypeError:
|
1886
|
+
return inspect.signature(obj)
|
1887
|
+
except KeyError:
|
1888
|
+
pass
|
1889
|
+
sig = inspect.signature(obj)
|
1890
|
+
_INJECTION_SIGNATURE_CACHE[obj] = sig
|
1891
|
+
return sig
|
1892
|
+
|
1893
|
+
|
1894
|
+
class InjectionKwarg(ta.NamedTuple):
|
1895
|
+
name: str
|
1896
|
+
key: InjectorKey
|
1897
|
+
has_default: bool
|
1898
|
+
|
1899
|
+
|
1900
|
+
class InjectionKwargsTarget(ta.NamedTuple):
|
1901
|
+
obj: ta.Any
|
1902
|
+
kwargs: ta.Sequence[InjectionKwarg]
|
1903
|
+
|
1904
|
+
|
1905
|
+
def build_injection_kwargs_target(
|
1906
|
+
obj: ta.Any,
|
1907
|
+
*,
|
1908
|
+
skip_args: int = 0,
|
1909
|
+
skip_kwargs: ta.Optional[ta.Iterable[ta.Any]] = None,
|
1910
|
+
raw_optional: bool = False,
|
1911
|
+
) -> InjectionKwargsTarget:
|
1912
|
+
sig = _injection_signature(obj)
|
1913
|
+
|
1914
|
+
seen: ta.Set[InjectorKey] = set(map(as_injector_key, skip_kwargs)) if skip_kwargs is not None else set()
|
1915
|
+
kws: ta.List[InjectionKwarg] = []
|
1916
|
+
for p in list(sig.parameters.values())[skip_args:]:
|
1917
|
+
if p.annotation is inspect.Signature.empty:
|
1918
|
+
if p.default is not inspect.Parameter.empty:
|
1919
|
+
raise KeyError(f'{obj}, {p.name}')
|
1920
|
+
continue
|
1921
|
+
|
1922
|
+
if p.kind not in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY):
|
1923
|
+
raise TypeError(sig)
|
1924
|
+
|
1925
|
+
ann = p.annotation
|
1926
|
+
if (
|
1927
|
+
not raw_optional and
|
1928
|
+
is_optional_alias(ann)
|
1929
|
+
):
|
1930
|
+
ann = get_optional_alias_arg(ann)
|
1931
|
+
|
1932
|
+
k = as_injector_key(ann)
|
1933
|
+
|
1934
|
+
if k in seen:
|
1935
|
+
raise DuplicateInjectorKeyError(k)
|
1936
|
+
seen.add(k)
|
1937
|
+
|
1938
|
+
kws.append(InjectionKwarg(
|
1939
|
+
p.name,
|
1940
|
+
k,
|
1941
|
+
p.default is not inspect.Parameter.empty,
|
1942
|
+
))
|
1943
|
+
|
1944
|
+
return InjectionKwargsTarget(
|
1945
|
+
obj,
|
1946
|
+
kws,
|
1947
|
+
)
|
1948
|
+
|
1949
|
+
|
1950
|
+
###
|
1951
|
+
# binder
|
1952
|
+
|
1953
|
+
|
1954
|
+
class InjectorBinder:
|
1955
|
+
def __new__(cls, *args, **kwargs): # noqa
|
1956
|
+
raise TypeError
|
1957
|
+
|
1958
|
+
_FN_TYPES: ta.Tuple[type, ...] = (
|
1959
|
+
types.FunctionType,
|
1960
|
+
types.MethodType,
|
1961
|
+
|
1962
|
+
classmethod,
|
1963
|
+
staticmethod,
|
1964
|
+
|
1965
|
+
functools.partial,
|
1966
|
+
functools.partialmethod,
|
1967
|
+
)
|
708
1968
|
|
709
1969
|
@classmethod
|
710
|
-
def
|
1970
|
+
def _is_fn(cls, obj: ta.Any) -> bool:
|
1971
|
+
return isinstance(obj, cls._FN_TYPES)
|
1972
|
+
|
1973
|
+
@classmethod
|
1974
|
+
def bind_as_fn(cls, icls: ta.Type[T]) -> ta.Type[T]:
|
1975
|
+
check_isinstance(icls, type)
|
1976
|
+
if icls not in cls._FN_TYPES:
|
1977
|
+
cls._FN_TYPES = (*cls._FN_TYPES, icls)
|
1978
|
+
return icls
|
1979
|
+
|
1980
|
+
_BANNED_BIND_TYPES: ta.Tuple[type, ...] = (
|
1981
|
+
InjectorProvider,
|
1982
|
+
)
|
1983
|
+
|
1984
|
+
@classmethod
|
1985
|
+
def bind(
|
711
1986
|
cls,
|
712
|
-
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
|
729
|
-
|
1987
|
+
obj: ta.Any,
|
1988
|
+
*,
|
1989
|
+
key: ta.Any = None,
|
1990
|
+
tag: ta.Any = None,
|
1991
|
+
array: ta.Optional[bool] = None, # noqa
|
1992
|
+
|
1993
|
+
to_fn: ta.Any = None,
|
1994
|
+
to_ctor: ta.Any = None,
|
1995
|
+
to_const: ta.Any = None,
|
1996
|
+
to_key: ta.Any = None,
|
1997
|
+
|
1998
|
+
singleton: bool = False,
|
1999
|
+
) -> InjectorBinding:
|
2000
|
+
if obj is None or obj is inspect.Parameter.empty:
|
2001
|
+
raise TypeError(obj)
|
2002
|
+
if isinstance(obj, cls._BANNED_BIND_TYPES):
|
2003
|
+
raise TypeError(obj)
|
2004
|
+
|
2005
|
+
##
|
2006
|
+
|
2007
|
+
if key is not None:
|
2008
|
+
key = as_injector_key(key)
|
2009
|
+
|
2010
|
+
##
|
2011
|
+
|
2012
|
+
has_to = (
|
2013
|
+
to_fn is not None or
|
2014
|
+
to_ctor is not None or
|
2015
|
+
to_const is not None or
|
2016
|
+
to_key is not None
|
730
2017
|
)
|
2018
|
+
if isinstance(obj, InjectorKey):
|
2019
|
+
if key is None:
|
2020
|
+
key = obj
|
2021
|
+
elif isinstance(obj, type):
|
2022
|
+
if not has_to:
|
2023
|
+
to_ctor = obj
|
2024
|
+
if key is None:
|
2025
|
+
key = InjectorKey(obj)
|
2026
|
+
elif cls._is_fn(obj) and not has_to:
|
2027
|
+
to_fn = obj
|
2028
|
+
if key is None:
|
2029
|
+
sig = _injection_signature(obj)
|
2030
|
+
ty = check_isinstance(sig.return_annotation, type)
|
2031
|
+
key = InjectorKey(ty)
|
2032
|
+
else:
|
2033
|
+
if to_const is not None:
|
2034
|
+
raise TypeError('Cannot bind instance with to_const')
|
2035
|
+
to_const = obj
|
2036
|
+
if key is None:
|
2037
|
+
key = InjectorKey(type(obj))
|
2038
|
+
del has_to
|
731
2039
|
|
2040
|
+
##
|
732
2041
|
|
733
|
-
|
734
|
-
|
2042
|
+
if tag is not None:
|
2043
|
+
if key.tag is not None:
|
2044
|
+
raise TypeError('Tag already set')
|
2045
|
+
key = dc.replace(key, tag=tag)
|
2046
|
+
|
2047
|
+
if array is not None:
|
2048
|
+
key = dc.replace(key, array=array)
|
2049
|
+
|
2050
|
+
##
|
2051
|
+
|
2052
|
+
providers: ta.List[InjectorProvider] = []
|
2053
|
+
if to_fn is not None:
|
2054
|
+
providers.append(FnInjectorProvider(to_fn))
|
2055
|
+
if to_ctor is not None:
|
2056
|
+
providers.append(CtorInjectorProvider(to_ctor))
|
2057
|
+
if to_const is not None:
|
2058
|
+
providers.append(ConstInjectorProvider(to_const))
|
2059
|
+
if to_key is not None:
|
2060
|
+
providers.append(LinkInjectorProvider(as_injector_key(to_key)))
|
2061
|
+
if not providers:
|
2062
|
+
raise TypeError('Must specify provider')
|
2063
|
+
if len(providers) > 1:
|
2064
|
+
raise TypeError('May not specify multiple providers')
|
2065
|
+
provider, = providers
|
2066
|
+
|
2067
|
+
##
|
2068
|
+
|
2069
|
+
if singleton:
|
2070
|
+
provider = SingletonInjectorProvider(provider)
|
2071
|
+
|
2072
|
+
##
|
2073
|
+
|
2074
|
+
binding = InjectorBinding(key, provider)
|
2075
|
+
|
2076
|
+
##
|
2077
|
+
|
2078
|
+
return binding
|
2079
|
+
|
2080
|
+
|
2081
|
+
###
|
2082
|
+
# injector
|
2083
|
+
|
2084
|
+
|
2085
|
+
_INJECTOR_INJECTOR_KEY = InjectorKey(Injector)
|
2086
|
+
|
2087
|
+
|
2088
|
+
class _Injector(Injector):
|
2089
|
+
def __init__(self, bs: InjectorBindings, p: ta.Optional[Injector] = None) -> None:
|
2090
|
+
super().__init__()
|
2091
|
+
|
2092
|
+
self._bs = check_isinstance(bs, InjectorBindings)
|
2093
|
+
self._p: ta.Optional[Injector] = check_isinstance(p, (Injector, type(None)))
|
2094
|
+
|
2095
|
+
self._pfm = {k: v.provider_fn() for k, v in build_injector_provider_map(bs).items()}
|
2096
|
+
|
2097
|
+
if _INJECTOR_INJECTOR_KEY in self._pfm:
|
2098
|
+
raise DuplicateInjectorKeyError(_INJECTOR_INJECTOR_KEY)
|
735
2099
|
|
2100
|
+
def try_provide(self, key: ta.Any) -> Maybe[ta.Any]:
|
2101
|
+
key = as_injector_key(key)
|
736
2102
|
|
737
|
-
|
2103
|
+
if key == _INJECTOR_INJECTOR_KEY:
|
2104
|
+
return Maybe.just(self)
|
738
2105
|
|
2106
|
+
fn = self._pfm.get(key)
|
2107
|
+
if fn is not None:
|
2108
|
+
return Maybe.just(fn(self))
|
739
2109
|
|
740
|
-
|
741
|
-
|
742
|
-
|
743
|
-
|
744
|
-
code = getattr(states, name)
|
745
|
-
d[code] = name
|
746
|
-
return d
|
2110
|
+
if self._p is not None:
|
2111
|
+
pv = self._p.try_provide(key)
|
2112
|
+
if pv is not None:
|
2113
|
+
return Maybe.empty()
|
747
2114
|
|
2115
|
+
return Maybe.empty()
|
748
2116
|
|
749
|
-
|
2117
|
+
def provide(self, key: ta.Any) -> ta.Any:
|
2118
|
+
v = self.try_provide(key)
|
2119
|
+
if v.present:
|
2120
|
+
return v.must()
|
2121
|
+
raise UnboundInjectorKeyError(key)
|
750
2122
|
|
2123
|
+
def provide_kwargs(self, obj: ta.Any) -> ta.Mapping[str, ta.Any]:
|
2124
|
+
kt = build_injection_kwargs_target(obj)
|
2125
|
+
ret: ta.Dict[str, ta.Any] = {}
|
2126
|
+
for kw in kt.kwargs:
|
2127
|
+
if kw.has_default:
|
2128
|
+
if not (mv := self.try_provide(kw.key)).present:
|
2129
|
+
continue
|
2130
|
+
v = mv.must()
|
2131
|
+
else:
|
2132
|
+
v = self.provide(kw.key)
|
2133
|
+
ret[kw.name] = v
|
2134
|
+
return ret
|
751
2135
|
|
752
|
-
|
753
|
-
|
754
|
-
|
755
|
-
RUNNING = 20
|
756
|
-
BACKOFF = 30
|
757
|
-
STOPPING = 40
|
758
|
-
EXITED = 100
|
759
|
-
FATAL = 200
|
760
|
-
UNKNOWN = 1000
|
2136
|
+
def inject(self, obj: ta.Any) -> ta.Any:
|
2137
|
+
kws = self.provide_kwargs(obj)
|
2138
|
+
return obj(**kws)
|
761
2139
|
|
762
2140
|
|
763
|
-
|
764
|
-
|
765
|
-
ProcessStates.EXITED,
|
766
|
-
ProcessStates.FATAL,
|
767
|
-
ProcessStates.UNKNOWN,
|
768
|
-
)
|
2141
|
+
###
|
2142
|
+
# injection helpers
|
769
2143
|
|
770
|
-
RUNNING_STATES = (
|
771
|
-
ProcessStates.RUNNING,
|
772
|
-
ProcessStates.BACKOFF,
|
773
|
-
ProcessStates.STARTING,
|
774
|
-
)
|
775
2144
|
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
ProcessStates.STOPPING,
|
780
|
-
)
|
2145
|
+
class Injection:
|
2146
|
+
def __new__(cls, *args, **kwargs): # noqa
|
2147
|
+
raise TypeError
|
781
2148
|
|
2149
|
+
# keys
|
782
2150
|
|
783
|
-
|
2151
|
+
@classmethod
|
2152
|
+
def as_key(cls, o: ta.Any) -> InjectorKey:
|
2153
|
+
return as_injector_key(o)
|
784
2154
|
|
2155
|
+
@classmethod
|
2156
|
+
def array(cls, o: ta.Any) -> InjectorKey:
|
2157
|
+
return dc.replace(as_injector_key(o), array=True)
|
785
2158
|
|
786
|
-
|
787
|
-
|
2159
|
+
@classmethod
|
2160
|
+
def tag(cls, o: ta.Any, t: ta.Any) -> InjectorKey:
|
2161
|
+
return dc.replace(as_injector_key(o), tag=t)
|
788
2162
|
|
2163
|
+
# bindings
|
789
2164
|
|
790
|
-
|
2165
|
+
@classmethod
|
2166
|
+
def as_bindings(cls, *args: InjectorBindingOrBindings) -> InjectorBindings:
|
2167
|
+
return as_injector_bindings(*args)
|
791
2168
|
|
2169
|
+
@classmethod
|
2170
|
+
def override(cls, p: InjectorBindings, *args: InjectorBindingOrBindings) -> InjectorBindings:
|
2171
|
+
return injector_override(p, *args)
|
792
2172
|
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
2173
|
+
# binder
|
2174
|
+
|
2175
|
+
@classmethod
|
2176
|
+
def bind(
|
2177
|
+
cls,
|
2178
|
+
obj: ta.Any,
|
2179
|
+
*,
|
2180
|
+
key: ta.Any = None,
|
2181
|
+
tag: ta.Any = None,
|
2182
|
+
array: ta.Optional[bool] = None, # noqa
|
2183
|
+
|
2184
|
+
to_fn: ta.Any = None,
|
2185
|
+
to_ctor: ta.Any = None,
|
2186
|
+
to_const: ta.Any = None,
|
2187
|
+
to_key: ta.Any = None,
|
2188
|
+
|
2189
|
+
singleton: bool = False,
|
2190
|
+
) -> InjectorBinding:
|
2191
|
+
return InjectorBinder.bind(
|
2192
|
+
obj,
|
2193
|
+
|
2194
|
+
key=key,
|
2195
|
+
tag=tag,
|
2196
|
+
array=array,
|
2197
|
+
|
2198
|
+
to_fn=to_fn,
|
2199
|
+
to_ctor=to_ctor,
|
2200
|
+
to_const=to_const,
|
2201
|
+
to_key=to_key,
|
2202
|
+
|
2203
|
+
singleton=singleton,
|
2204
|
+
)
|
798
2205
|
|
2206
|
+
# injector
|
799
2207
|
|
800
|
-
|
2208
|
+
@classmethod
|
2209
|
+
def create_injector(cls, *args: InjectorBindingOrBindings, p: ta.Optional[Injector] = None) -> Injector:
|
2210
|
+
return _Injector(as_injector_bindings(*args), p)
|
801
2211
|
|
802
2212
|
|
803
|
-
|
804
|
-
return check_not_none(_supervisor_states_by_code.get(code))
|
2213
|
+
inj = Injection
|
805
2214
|
|
806
2215
|
|
807
2216
|
########################################
|
@@ -1576,6 +2985,66 @@ def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
|
|
1576
2985
|
return get_obj_marshaler(ty).unmarshal(o)
|
1577
2986
|
|
1578
2987
|
|
2988
|
+
########################################
|
2989
|
+
# ../../configs.py
|
2990
|
+
|
2991
|
+
|
2992
|
+
def read_config_file(
|
2993
|
+
path: str,
|
2994
|
+
cls: ta.Type[T],
|
2995
|
+
*,
|
2996
|
+
prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
|
2997
|
+
) -> T:
|
2998
|
+
with open(path) as cf:
|
2999
|
+
if path.endswith('.toml'):
|
3000
|
+
config_dct = toml_loads(cf.read())
|
3001
|
+
else:
|
3002
|
+
config_dct = json.loads(cf.read())
|
3003
|
+
|
3004
|
+
if prepare is not None:
|
3005
|
+
config_dct = prepare(config_dct) # type: ignore
|
3006
|
+
|
3007
|
+
return unmarshal_obj(config_dct, cls)
|
3008
|
+
|
3009
|
+
|
3010
|
+
def build_config_named_children(
|
3011
|
+
o: ta.Union[
|
3012
|
+
ta.Sequence[ConfigMapping],
|
3013
|
+
ta.Mapping[str, ConfigMapping],
|
3014
|
+
None,
|
3015
|
+
],
|
3016
|
+
*,
|
3017
|
+
name_key: str = 'name',
|
3018
|
+
) -> ta.Optional[ta.Sequence[ConfigMapping]]:
|
3019
|
+
if o is None:
|
3020
|
+
return None
|
3021
|
+
|
3022
|
+
lst: ta.List[ConfigMapping] = []
|
3023
|
+
if isinstance(o, ta.Mapping):
|
3024
|
+
for k, v in o.items():
|
3025
|
+
check_isinstance(v, ta.Mapping)
|
3026
|
+
if name_key in v:
|
3027
|
+
n = v[name_key]
|
3028
|
+
if k != n:
|
3029
|
+
raise KeyError(f'Given names do not match: {n} != {k}')
|
3030
|
+
lst.append(v)
|
3031
|
+
else:
|
3032
|
+
lst.append({name_key: k, **v})
|
3033
|
+
|
3034
|
+
else:
|
3035
|
+
check_not_isinstance(o, str)
|
3036
|
+
lst.extend(o)
|
3037
|
+
|
3038
|
+
seen = set()
|
3039
|
+
for d in lst:
|
3040
|
+
n = d['name']
|
3041
|
+
if n in d:
|
3042
|
+
raise KeyError(f'Duplicate name: {n}')
|
3043
|
+
seen.add(n)
|
3044
|
+
|
3045
|
+
return lst
|
3046
|
+
|
3047
|
+
|
1579
3048
|
########################################
|
1580
3049
|
# ../events.py
|
1581
3050
|
|
@@ -2108,6 +3577,127 @@ else:
|
|
2108
3577
|
Poller = SelectPoller
|
2109
3578
|
|
2110
3579
|
|
3580
|
+
########################################
|
3581
|
+
# ../configs.py
|
3582
|
+
|
3583
|
+
|
3584
|
+
##
|
3585
|
+
|
3586
|
+
|
3587
|
+
@dc.dataclass(frozen=True)
|
3588
|
+
class ProcessConfig:
|
3589
|
+
name: str
|
3590
|
+
command: str
|
3591
|
+
|
3592
|
+
uid: ta.Optional[int] = None
|
3593
|
+
directory: ta.Optional[str] = None
|
3594
|
+
umask: ta.Optional[int] = None
|
3595
|
+
priority: int = 999
|
3596
|
+
|
3597
|
+
autostart: bool = True
|
3598
|
+
autorestart: str = 'unexpected'
|
3599
|
+
|
3600
|
+
startsecs: int = 1
|
3601
|
+
startretries: int = 3
|
3602
|
+
|
3603
|
+
numprocs: int = 1
|
3604
|
+
numprocs_start: int = 0
|
3605
|
+
|
3606
|
+
@dc.dataclass(frozen=True)
|
3607
|
+
class Log:
|
3608
|
+
file: ta.Optional[str] = None
|
3609
|
+
capture_maxbytes: ta.Optional[int] = None
|
3610
|
+
events_enabled: bool = False
|
3611
|
+
syslog: bool = False
|
3612
|
+
backups: ta.Optional[int] = None
|
3613
|
+
maxbytes: ta.Optional[int] = None
|
3614
|
+
|
3615
|
+
stdout: Log = Log()
|
3616
|
+
stderr: Log = Log()
|
3617
|
+
|
3618
|
+
stopsignal: int = signal.SIGTERM
|
3619
|
+
stopwaitsecs: int = 10
|
3620
|
+
stopasgroup: bool = False
|
3621
|
+
|
3622
|
+
killasgroup: bool = False
|
3623
|
+
|
3624
|
+
exitcodes: ta.Sequence[int] = (0,)
|
3625
|
+
|
3626
|
+
redirect_stderr: bool = False
|
3627
|
+
|
3628
|
+
environment: ta.Optional[ta.Mapping[str, str]] = None
|
3629
|
+
|
3630
|
+
|
3631
|
+
@dc.dataclass(frozen=True)
|
3632
|
+
class ProcessGroupConfig:
|
3633
|
+
name: str
|
3634
|
+
|
3635
|
+
priority: int = 999
|
3636
|
+
|
3637
|
+
processes: ta.Optional[ta.Sequence[ProcessConfig]] = None
|
3638
|
+
|
3639
|
+
|
3640
|
+
@dc.dataclass(frozen=True)
|
3641
|
+
class ServerConfig:
|
3642
|
+
user: ta.Optional[str] = None
|
3643
|
+
nodaemon: bool = False
|
3644
|
+
umask: int = 0o22
|
3645
|
+
directory: ta.Optional[str] = None
|
3646
|
+
logfile: str = 'supervisord.log'
|
3647
|
+
logfile_maxbytes: int = 50 * 1024 * 1024
|
3648
|
+
logfile_backups: int = 10
|
3649
|
+
loglevel: int = logging.INFO
|
3650
|
+
pidfile: str = 'supervisord.pid'
|
3651
|
+
identifier: str = 'supervisor'
|
3652
|
+
child_logdir: str = '/dev/null'
|
3653
|
+
minfds: int = 1024
|
3654
|
+
minprocs: int = 200
|
3655
|
+
nocleanup: bool = False
|
3656
|
+
strip_ansi: bool = False
|
3657
|
+
silent: bool = False
|
3658
|
+
|
3659
|
+
groups: ta.Optional[ta.Sequence[ProcessGroupConfig]] = None
|
3660
|
+
|
3661
|
+
@classmethod
|
3662
|
+
def new(
|
3663
|
+
cls,
|
3664
|
+
umask: ta.Union[int, str] = 0o22,
|
3665
|
+
directory: ta.Optional[str] = None,
|
3666
|
+
logfile: str = 'supervisord.log',
|
3667
|
+
logfile_maxbytes: ta.Union[int, str] = 50 * 1024 * 1024,
|
3668
|
+
loglevel: ta.Union[int, str] = logging.INFO,
|
3669
|
+
pidfile: str = 'supervisord.pid',
|
3670
|
+
child_logdir: ta.Optional[str] = None,
|
3671
|
+
**kwargs: ta.Any,
|
3672
|
+
) -> 'ServerConfig':
|
3673
|
+
return cls(
|
3674
|
+
umask=octal_type(umask),
|
3675
|
+
directory=existing_directory(directory) if directory is not None else None,
|
3676
|
+
logfile=existing_dirpath(logfile),
|
3677
|
+
logfile_maxbytes=byte_size(logfile_maxbytes),
|
3678
|
+
loglevel=logging_level(loglevel),
|
3679
|
+
pidfile=existing_dirpath(pidfile),
|
3680
|
+
child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
|
3681
|
+
**kwargs,
|
3682
|
+
)
|
3683
|
+
|
3684
|
+
|
3685
|
+
##
|
3686
|
+
|
3687
|
+
|
3688
|
+
def prepare_process_group_config(dct: ConfigMapping) -> ConfigMapping:
|
3689
|
+
out = dict(dct)
|
3690
|
+
out['processes'] = build_config_named_children(out.get('processes'))
|
3691
|
+
return out
|
3692
|
+
|
3693
|
+
|
3694
|
+
def prepare_server_config(dct: ta.Mapping[str, ta.Any]) -> ta.Mapping[str, ta.Any]:
|
3695
|
+
out = dict(dct)
|
3696
|
+
group_dcts = build_config_named_children(out.get('groups'))
|
3697
|
+
out['groups'] = [prepare_process_group_config(group_dct) for group_dct in group_dcts or []]
|
3698
|
+
return out
|
3699
|
+
|
3700
|
+
|
2111
3701
|
########################################
|
2112
3702
|
# ../types.py
|
2113
3703
|
|
@@ -2132,6 +3722,11 @@ class AbstractServerContext(abc.ABC):
|
|
2132
3722
|
def pid_history(self) -> ta.Dict[int, 'AbstractSubprocess']:
|
2133
3723
|
raise NotImplementedError
|
2134
3724
|
|
3725
|
+
@property
|
3726
|
+
@abc.abstractmethod
|
3727
|
+
def inherited_fds(self) -> ta.FrozenSet[int]:
|
3728
|
+
raise NotImplementedError
|
3729
|
+
|
2135
3730
|
|
2136
3731
|
class AbstractSubprocess(abc.ABC):
|
2137
3732
|
@property
|
@@ -2163,12 +3758,14 @@ class ServerContext(AbstractServerContext):
|
|
2163
3758
|
self,
|
2164
3759
|
config: ServerConfig,
|
2165
3760
|
*,
|
2166
|
-
epoch:
|
3761
|
+
epoch: ServerEpoch = ServerEpoch(0),
|
3762
|
+
inherited_fds: ta.Optional[InheritedFds] = None,
|
2167
3763
|
) -> None:
|
2168
3764
|
super().__init__()
|
2169
3765
|
|
2170
3766
|
self._config = config
|
2171
3767
|
self._epoch = epoch
|
3768
|
+
self._inherited_fds = InheritedFds(frozenset(inherited_fds or []))
|
2172
3769
|
|
2173
3770
|
self._pid_history: ta.Dict[int, AbstractSubprocess] = {}
|
2174
3771
|
self._state: SupervisorState = SupervisorStates.RUNNING
|
@@ -2192,7 +3789,7 @@ class ServerContext(AbstractServerContext):
|
|
2192
3789
|
return self._config
|
2193
3790
|
|
2194
3791
|
@property
|
2195
|
-
def epoch(self) ->
|
3792
|
+
def epoch(self) -> ServerEpoch:
|
2196
3793
|
return self._epoch
|
2197
3794
|
|
2198
3795
|
@property
|
@@ -2222,6 +3819,10 @@ class ServerContext(AbstractServerContext):
|
|
2222
3819
|
def gid(self) -> ta.Optional[int]:
|
2223
3820
|
return self._gid
|
2224
3821
|
|
3822
|
+
@property
|
3823
|
+
def inherited_fds(self) -> InheritedFds:
|
3824
|
+
return self._inherited_fds
|
3825
|
+
|
2225
3826
|
##
|
2226
3827
|
|
2227
3828
|
def set_signals(self) -> None:
|
@@ -2865,6 +4466,9 @@ class InputDispatcher(Dispatcher):
|
|
2865
4466
|
# ../process.py
|
2866
4467
|
|
2867
4468
|
|
4469
|
+
##
|
4470
|
+
|
4471
|
+
|
2868
4472
|
@functools.total_ordering
|
2869
4473
|
class Subprocess(AbstractSubprocess):
|
2870
4474
|
"""A class to manage a subprocess."""
|
@@ -2890,7 +4494,12 @@ class Subprocess(AbstractSubprocess):
|
|
2890
4494
|
spawn_err = None # error message attached by spawn() if any
|
2891
4495
|
group = None # ProcessGroup instance if process is in the group
|
2892
4496
|
|
2893
|
-
def __init__(
|
4497
|
+
def __init__(
|
4498
|
+
self,
|
4499
|
+
config: ProcessConfig,
|
4500
|
+
group: 'ProcessGroup',
|
4501
|
+
context: AbstractServerContext,
|
4502
|
+
) -> None:
|
2894
4503
|
super().__init__()
|
2895
4504
|
self._config = config
|
2896
4505
|
self.group = group
|
@@ -3134,8 +4743,10 @@ class Subprocess(AbstractSubprocess):
|
|
3134
4743
|
os.dup2(self._pipes['child_stdout'], 2)
|
3135
4744
|
else:
|
3136
4745
|
os.dup2(self._pipes['child_stderr'], 2)
|
3137
|
-
|
4746
|
+
|
3138
4747
|
for i in range(3, self.context.config.minfds):
|
4748
|
+
if i in self.context.inherited_fds:
|
4749
|
+
continue
|
3139
4750
|
close_fd(i)
|
3140
4751
|
|
3141
4752
|
def _spawn_as_child(self, filename: str, argv: ta.Sequence[str]) -> None:
|
@@ -3529,15 +5140,39 @@ class Subprocess(AbstractSubprocess):
|
|
3529
5140
|
pass
|
3530
5141
|
|
3531
5142
|
|
5143
|
+
##
|
5144
|
+
|
5145
|
+
|
5146
|
+
@dc.dataclass(frozen=True)
|
5147
|
+
class SubprocessFactory:
|
5148
|
+
fn: ta.Callable[[ProcessConfig, 'ProcessGroup'], Subprocess]
|
5149
|
+
|
5150
|
+
def __call__(self, config: ProcessConfig, group: 'ProcessGroup') -> Subprocess:
|
5151
|
+
return self.fn(config, group)
|
5152
|
+
|
5153
|
+
|
3532
5154
|
@functools.total_ordering
|
3533
5155
|
class ProcessGroup:
|
3534
|
-
def __init__(
|
5156
|
+
def __init__(
|
5157
|
+
self,
|
5158
|
+
config: ProcessGroupConfig,
|
5159
|
+
context: ServerContext,
|
5160
|
+
*,
|
5161
|
+
subprocess_factory: ta.Optional[SubprocessFactory] = None,
|
5162
|
+
):
|
3535
5163
|
super().__init__()
|
3536
5164
|
self.config = config
|
3537
5165
|
self.context = context
|
5166
|
+
|
5167
|
+
if subprocess_factory is None:
|
5168
|
+
def make_subprocess(config: ProcessConfig, group: ProcessGroup) -> Subprocess:
|
5169
|
+
return Subprocess(config, group, self.context)
|
5170
|
+
subprocess_factory = SubprocessFactory(make_subprocess)
|
5171
|
+
self._subprocess_factory = subprocess_factory
|
5172
|
+
|
3538
5173
|
self.processes = {}
|
3539
5174
|
for pconfig in self.config.processes or []:
|
3540
|
-
process =
|
5175
|
+
process = self._subprocess_factory(pconfig, self)
|
3541
5176
|
self.processes[pconfig.name] = process
|
3542
5177
|
|
3543
5178
|
def __lt__(self, other):
|
@@ -3607,12 +5242,32 @@ def timeslice(period: int, when: float) -> int:
|
|
3607
5242
|
return int(when - (when % period))
|
3608
5243
|
|
3609
5244
|
|
5245
|
+
@dc.dataclass(frozen=True)
|
5246
|
+
class ProcessGroupFactory:
|
5247
|
+
fn: ta.Callable[[ProcessGroupConfig], ProcessGroup]
|
5248
|
+
|
5249
|
+
def __call__(self, config: ProcessGroupConfig) -> ProcessGroup:
|
5250
|
+
return self.fn(config)
|
5251
|
+
|
5252
|
+
|
3610
5253
|
class Supervisor:
|
3611
5254
|
|
3612
|
-
def __init__(
|
5255
|
+
def __init__(
|
5256
|
+
self,
|
5257
|
+
context: ServerContext,
|
5258
|
+
*,
|
5259
|
+
process_group_factory: ta.Optional[ProcessGroupFactory] = None,
|
5260
|
+
) -> None:
|
3613
5261
|
super().__init__()
|
3614
5262
|
|
3615
5263
|
self._context = context
|
5264
|
+
|
5265
|
+
if process_group_factory is None:
|
5266
|
+
def make_process_group(config: ProcessGroupConfig) -> ProcessGroup:
|
5267
|
+
return ProcessGroup(config, self._context)
|
5268
|
+
process_group_factory = ProcessGroupFactory(make_process_group)
|
5269
|
+
self._process_group_factory = process_group_factory
|
5270
|
+
|
3616
5271
|
self._ticks: ta.Dict[int, float] = {}
|
3617
5272
|
self._process_groups: ta.Dict[str, ProcessGroup] = {} # map of process group name to process group object
|
3618
5273
|
self._stop_groups: ta.Optional[ta.List[ProcessGroup]] = None # list used for priority ordered shutdown
|
@@ -3654,7 +5309,7 @@ class Supervisor:
|
|
3654
5309
|
if name in self._process_groups:
|
3655
5310
|
return False
|
3656
5311
|
|
3657
|
-
group = self._process_groups[name] =
|
5312
|
+
group = self._process_groups[name] = self._process_group_factory(config)
|
3658
5313
|
group.after_setuid()
|
3659
5314
|
|
3660
5315
|
EVENT_CALLBACKS.notify(ProcessGroupAddedEvent(name))
|
@@ -3924,6 +5579,53 @@ class Supervisor:
|
|
3924
5579
|
# main.py
|
3925
5580
|
|
3926
5581
|
|
5582
|
+
##
|
5583
|
+
|
5584
|
+
|
5585
|
+
def build_server_bindings(
|
5586
|
+
config: ServerConfig,
|
5587
|
+
*,
|
5588
|
+
server_epoch: ta.Optional[ServerEpoch] = None,
|
5589
|
+
inherited_fds: ta.Optional[InheritedFds] = None,
|
5590
|
+
) -> InjectorBindings:
|
5591
|
+
lst: ta.List[InjectorBindingOrBindings] = [
|
5592
|
+
inj.bind(config),
|
5593
|
+
|
5594
|
+
inj.bind(ServerContext, singleton=True),
|
5595
|
+
inj.bind(AbstractServerContext, to_key=ServerContext),
|
5596
|
+
|
5597
|
+
inj.bind(Supervisor, singleton=True),
|
5598
|
+
]
|
5599
|
+
|
5600
|
+
#
|
5601
|
+
|
5602
|
+
def make_process_group_factory(injector: Injector) -> ProcessGroupFactory:
|
5603
|
+
def inner(group_config: ProcessGroupConfig) -> ProcessGroup:
|
5604
|
+
return injector.inject(functools.partial(ProcessGroup, group_config))
|
5605
|
+
return ProcessGroupFactory(inner)
|
5606
|
+
lst.append(inj.bind(make_process_group_factory))
|
5607
|
+
|
5608
|
+
def make_subprocess_factory(injector: Injector) -> SubprocessFactory:
|
5609
|
+
def inner(process_config: ProcessConfig, group: ProcessGroup) -> Subprocess:
|
5610
|
+
return injector.inject(functools.partial(Subprocess, process_config, group))
|
5611
|
+
return SubprocessFactory(inner)
|
5612
|
+
lst.append(inj.bind(make_subprocess_factory))
|
5613
|
+
|
5614
|
+
#
|
5615
|
+
|
5616
|
+
if server_epoch is not None:
|
5617
|
+
lst.append(inj.bind(server_epoch, key=ServerEpoch))
|
5618
|
+
if inherited_fds is not None:
|
5619
|
+
lst.append(inj.bind(inherited_fds, key=InheritedFds))
|
5620
|
+
|
5621
|
+
#
|
5622
|
+
|
5623
|
+
return inj.as_bindings(*lst)
|
5624
|
+
|
5625
|
+
|
5626
|
+
##
|
5627
|
+
|
5628
|
+
|
3927
5629
|
def main(
|
3928
5630
|
argv: ta.Optional[ta.Sequence[str]] = None,
|
3929
5631
|
*,
|
@@ -3934,6 +5636,7 @@ def main(
|
|
3934
5636
|
parser = argparse.ArgumentParser()
|
3935
5637
|
parser.add_argument('config_file', metavar='config-file')
|
3936
5638
|
parser.add_argument('--no-journald', action='store_true')
|
5639
|
+
parser.add_argument('--inherit-initial-fds', action='store_true')
|
3937
5640
|
args = parser.parse_args(argv)
|
3938
5641
|
|
3939
5642
|
#
|
@@ -3949,20 +5652,27 @@ def main(
|
|
3949
5652
|
|
3950
5653
|
#
|
3951
5654
|
|
5655
|
+
initial_fds: ta.Optional[InheritedFds] = None
|
5656
|
+
if args.inherit_initial_fds:
|
5657
|
+
initial_fds = InheritedFds(get_open_fds(0x10000))
|
5658
|
+
|
3952
5659
|
# if we hup, restart by making a new Supervisor()
|
3953
5660
|
for epoch in itertools.count():
|
3954
|
-
|
3955
|
-
|
3956
|
-
|
3957
|
-
|
3958
|
-
|
5661
|
+
config = read_config_file(
|
5662
|
+
os.path.expanduser(cf),
|
5663
|
+
ServerConfig,
|
5664
|
+
prepare=prepare_server_config,
|
5665
|
+
)
|
3959
5666
|
|
3960
|
-
|
5667
|
+
injector = inj.create_injector(build_server_bindings(
|
3961
5668
|
config,
|
3962
|
-
|
3963
|
-
|
5669
|
+
server_epoch=ServerEpoch(epoch),
|
5670
|
+
inherited_fds=initial_fds,
|
5671
|
+
))
|
5672
|
+
|
5673
|
+
context = injector.provide(ServerContext)
|
5674
|
+
supervisor = injector.provide(Supervisor)
|
3964
5675
|
|
3965
|
-
supervisor = Supervisor(context)
|
3966
5676
|
try:
|
3967
5677
|
supervisor.main()
|
3968
5678
|
except ExitNow:
|