ominfra 0.0.0.dev192__py3-none-any.whl → 0.0.0.dev194__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/main.py +5 -2
- ominfra/manage/deploy/conf/manager.py +4 -4
- ominfra/manage/deploy/conf/specs.py +2 -2
- ominfra/manage/main.py +3 -3
- ominfra/scripts/journald2aws.py +448 -122
- ominfra/scripts/manage.py +2019 -1700
- ominfra/scripts/supervisor.py +2113 -1742
- ominfra/supervisor/configs.py +4 -3
- ominfra/supervisor/main.py +2 -2
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/METADATA +4 -4
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/RECORD +15 -16
- ominfra/configs.py +0 -129
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/top_level.txt +0 -0
ominfra/scripts/supervisor.py
CHANGED
@@ -36,6 +36,7 @@ import abc
|
|
36
36
|
import base64
|
37
37
|
import collections
|
38
38
|
import collections.abc
|
39
|
+
import configparser
|
39
40
|
import contextlib
|
40
41
|
import contextvars
|
41
42
|
import ctypes as ct
|
@@ -79,7 +80,7 @@ import types
|
|
79
80
|
import typing as ta
|
80
81
|
import uuid
|
81
82
|
import warnings
|
82
|
-
import weakref
|
83
|
+
import weakref
|
83
84
|
|
84
85
|
|
85
86
|
########################################
|
@@ -92,15 +93,21 @@ if sys.version_info < (3, 8):
|
|
92
93
|
########################################
|
93
94
|
|
94
95
|
|
95
|
-
# ../../omdev/toml/parser.py
|
96
|
-
TomlParseFloat = ta.Callable[[str], ta.Any]
|
97
|
-
TomlKey = ta.Tuple[str, ...]
|
98
|
-
TomlPos = int # ta.TypeAlias
|
99
|
-
|
100
96
|
# utils/collections.py
|
101
97
|
K = ta.TypeVar('K')
|
102
98
|
V = ta.TypeVar('V')
|
103
99
|
|
100
|
+
# ../../omlish/configs/types.py
|
101
|
+
ConfigMap = ta.Mapping[str, ta.Any]
|
102
|
+
|
103
|
+
# ../../omlish/formats/ini/sections.py
|
104
|
+
IniSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
|
105
|
+
|
106
|
+
# ../../omlish/formats/toml/parser.py
|
107
|
+
TomlParseFloat = ta.Callable[[str], ta.Any]
|
108
|
+
TomlKey = ta.Tuple[str, ...]
|
109
|
+
TomlPos = int # ta.TypeAlias
|
110
|
+
|
104
111
|
# ../../omlish/lite/cached.py
|
105
112
|
T = ta.TypeVar('T')
|
106
113
|
CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
@@ -125,6 +132,9 @@ SocketAddress = ta.Any
|
|
125
132
|
EventCallback = ta.Callable[['Event'], None]
|
126
133
|
ProcessOutputChannel = ta.Literal['stdout', 'stderr'] # ta.TypeAlias
|
127
134
|
|
135
|
+
# ../../omlish/configs/formats.py
|
136
|
+
ConfigDataT = ta.TypeVar('ConfigDataT', bound='ConfigData')
|
137
|
+
|
128
138
|
# ../../omlish/http/parsing.py
|
129
139
|
HttpHeaders = http.client.HTTPMessage # ta.TypeAlias
|
130
140
|
|
@@ -141,10 +151,6 @@ InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
|
|
141
151
|
# ../../omlish/sockets/handlers.py
|
142
152
|
SocketHandlerFactory = ta.Callable[[SocketAddress, ta.BinaryIO, ta.BinaryIO], 'SocketHandler']
|
143
153
|
|
144
|
-
# ../configs.py
|
145
|
-
ConfigMapping = ta.Mapping[str, ta.Any]
|
146
|
-
IniConfigSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
|
147
|
-
|
148
154
|
# ../../omlish/http/handlers.py
|
149
155
|
HttpHandler = ta.Callable[['HttpHandlerRequest'], 'HttpHandlerResponse'] # ta.TypeAlias
|
150
156
|
|
@@ -153,1291 +159,1462 @@ CoroHttpServerFactory = ta.Callable[[SocketAddress], 'CoroHttpServer']
|
|
153
159
|
|
154
160
|
|
155
161
|
########################################
|
156
|
-
#
|
157
|
-
# SPDX-License-Identifier: MIT
|
158
|
-
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
159
|
-
# Licensed to PSF under a Contributor Agreement.
|
160
|
-
#
|
161
|
-
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
162
|
-
# --------------------------------------------
|
163
|
-
#
|
164
|
-
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
165
|
-
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
166
|
-
# documentation.
|
167
|
-
#
|
168
|
-
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
169
|
-
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
170
|
-
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
171
|
-
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
172
|
-
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
173
|
-
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
174
|
-
#
|
175
|
-
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
176
|
-
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
177
|
-
# any such work a brief summary of the changes made to Python.
|
178
|
-
#
|
179
|
-
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
180
|
-
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
181
|
-
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
182
|
-
# RIGHTS.
|
183
|
-
#
|
184
|
-
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
185
|
-
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
186
|
-
# ADVISED OF THE POSSIBILITY THEREOF.
|
187
|
-
#
|
188
|
-
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
189
|
-
#
|
190
|
-
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
191
|
-
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
192
|
-
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
193
|
-
#
|
194
|
-
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
195
|
-
# License Agreement.
|
196
|
-
#
|
197
|
-
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
162
|
+
# ../exceptions.py
|
198
163
|
|
199
164
|
|
200
|
-
|
165
|
+
class ProcessError(Exception):
|
166
|
+
"""Specialized exceptions used when attempting to start a process."""
|
201
167
|
|
202
168
|
|
203
|
-
|
169
|
+
class BadCommandError(ProcessError):
|
170
|
+
"""Indicates the command could not be parsed properly."""
|
204
171
|
|
205
|
-
TOML_RE_NUMBER = re.compile(
|
206
|
-
r"""
|
207
|
-
0
|
208
|
-
(?:
|
209
|
-
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
210
|
-
|
|
211
|
-
b[01](?:_?[01])* # bin
|
212
|
-
|
|
213
|
-
o[0-7](?:_?[0-7])* # oct
|
214
|
-
)
|
215
|
-
|
|
216
|
-
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
217
|
-
(?P<floatpart>
|
218
|
-
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
219
|
-
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
220
|
-
)
|
221
|
-
""",
|
222
|
-
flags=re.VERBOSE,
|
223
|
-
)
|
224
|
-
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
225
|
-
TOML_RE_DATETIME = re.compile(
|
226
|
-
rf"""
|
227
|
-
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
228
|
-
(?:
|
229
|
-
[Tt ]
|
230
|
-
{_TOML_TIME_RE_STR}
|
231
|
-
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
232
|
-
)?
|
233
|
-
""",
|
234
|
-
flags=re.VERBOSE,
|
235
|
-
)
|
236
172
|
|
173
|
+
class NotExecutableError(ProcessError):
|
174
|
+
"""
|
175
|
+
Indicates that the filespec cannot be executed because its path resolves to a file which is not executable, or which
|
176
|
+
is a directory.
|
177
|
+
"""
|
237
178
|
|
238
|
-
def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
|
239
|
-
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
240
179
|
|
241
|
-
|
242
|
-
"""
|
243
|
-
(
|
244
|
-
year_str,
|
245
|
-
month_str,
|
246
|
-
day_str,
|
247
|
-
hour_str,
|
248
|
-
minute_str,
|
249
|
-
sec_str,
|
250
|
-
micros_str,
|
251
|
-
zulu_time,
|
252
|
-
offset_sign_str,
|
253
|
-
offset_hour_str,
|
254
|
-
offset_minute_str,
|
255
|
-
) = match.groups()
|
256
|
-
year, month, day = int(year_str), int(month_str), int(day_str)
|
257
|
-
if hour_str is None:
|
258
|
-
return datetime.date(year, month, day)
|
259
|
-
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
260
|
-
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
261
|
-
if offset_sign_str:
|
262
|
-
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
263
|
-
offset_hour_str, offset_minute_str, offset_sign_str,
|
264
|
-
)
|
265
|
-
elif zulu_time:
|
266
|
-
tz = datetime.UTC
|
267
|
-
else: # local date-time
|
268
|
-
tz = None
|
269
|
-
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
180
|
+
class NotFoundError(ProcessError):
|
181
|
+
"""Indicates that the filespec cannot be executed because it could not be found."""
|
270
182
|
|
271
183
|
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
hours=sign * int(hour_str),
|
278
|
-
minutes=sign * int(minute_str),
|
279
|
-
),
|
280
|
-
)
|
184
|
+
class NoPermissionError(ProcessError):
|
185
|
+
"""
|
186
|
+
Indicates that the file cannot be executed because the supervisor process does not possess the appropriate UNIX
|
187
|
+
filesystem permission to execute the file.
|
188
|
+
"""
|
281
189
|
|
282
190
|
|
283
|
-
|
284
|
-
|
285
|
-
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
286
|
-
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
191
|
+
########################################
|
192
|
+
# ../privileges.py
|
287
193
|
|
288
194
|
|
289
|
-
def
|
290
|
-
|
291
|
-
|
292
|
-
|
195
|
+
def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
196
|
+
"""
|
197
|
+
Drop privileges to become the specified user, which may be a username or uid. Called for supervisord startup and
|
198
|
+
when spawning subprocesses. Returns None on success or a string error message if privileges could not be dropped.
|
199
|
+
"""
|
293
200
|
|
201
|
+
if user is None:
|
202
|
+
return 'No user specified to setuid to!'
|
294
203
|
|
295
|
-
|
204
|
+
# get uid for user, which can be a number or username
|
205
|
+
try:
|
206
|
+
uid = int(user)
|
207
|
+
except ValueError:
|
208
|
+
try:
|
209
|
+
pwrec = pwd.getpwnam(user) # type: ignore
|
210
|
+
except KeyError:
|
211
|
+
return f"Can't find username {user!r}"
|
212
|
+
uid = pwrec[2]
|
213
|
+
else:
|
214
|
+
try:
|
215
|
+
pwrec = pwd.getpwuid(uid)
|
216
|
+
except KeyError:
|
217
|
+
return f"Can't find uid {uid!r}"
|
296
218
|
|
297
|
-
|
298
|
-
# functions.
|
299
|
-
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
300
|
-
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
219
|
+
current_uid = os.getuid()
|
301
220
|
|
302
|
-
|
303
|
-
|
221
|
+
if current_uid == uid:
|
222
|
+
# do nothing and return successfully if the uid is already the current one. this allows a supervisord running as
|
223
|
+
# an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in it.
|
224
|
+
return None
|
304
225
|
|
305
|
-
|
226
|
+
if current_uid != 0:
|
227
|
+
return "Can't drop privilege as nonroot user"
|
306
228
|
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
229
|
+
gid = pwrec[3]
|
230
|
+
if hasattr(os, 'setgroups'):
|
231
|
+
user = pwrec[0]
|
232
|
+
groups = [grprec[2] for grprec in grp.getgrall() if user in grprec[3]]
|
312
233
|
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
'\\\\': '\u005C', # backslash
|
322
|
-
},
|
323
|
-
)
|
234
|
+
# always put our primary gid first in this list, otherwise we can lose group info since sometimes the first
|
235
|
+
# group in the setgroups list gets overwritten on the subsequent setgid call (at least on freebsd 9 with
|
236
|
+
# python 2.7 - this will be safe though for all unix /python version combos)
|
237
|
+
groups.insert(0, gid)
|
238
|
+
try:
|
239
|
+
os.setgroups(groups)
|
240
|
+
except OSError:
|
241
|
+
return 'Could not set groups of effective user'
|
324
242
|
|
243
|
+
try:
|
244
|
+
os.setgid(gid)
|
245
|
+
except OSError:
|
246
|
+
return 'Could not set group id of effective user'
|
325
247
|
|
326
|
-
|
327
|
-
"""An error raised if a document is not valid TOML."""
|
248
|
+
os.setuid(uid)
|
328
249
|
|
250
|
+
return None
|
329
251
|
|
330
|
-
def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
|
331
|
-
"""Parse TOML from a binary file object."""
|
332
|
-
b = fp.read()
|
333
|
-
try:
|
334
|
-
s = b.decode()
|
335
|
-
except AttributeError:
|
336
|
-
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
337
|
-
return toml_loads(s, parse_float=parse_float)
|
338
252
|
|
253
|
+
########################################
|
254
|
+
# ../states.py
|
339
255
|
|
340
|
-
def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
|
341
|
-
"""Parse TOML from a string."""
|
342
256
|
|
343
|
-
|
344
|
-
try:
|
345
|
-
src = s.replace('\r\n', '\n')
|
346
|
-
except (AttributeError, TypeError):
|
347
|
-
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
348
|
-
pos = 0
|
349
|
-
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
350
|
-
header: TomlKey = ()
|
351
|
-
parse_float = toml_make_safe_parse_float(parse_float)
|
257
|
+
##
|
352
258
|
|
353
|
-
# Parse one statement at a time (typically means one line in TOML source)
|
354
|
-
while True:
|
355
|
-
# 1. Skip line leading whitespace
|
356
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
357
259
|
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
char = src[pos]
|
368
|
-
except IndexError:
|
369
|
-
break
|
370
|
-
if char == '\n':
|
371
|
-
pos += 1
|
372
|
-
continue
|
373
|
-
if char in TOML_KEY_INITIAL_CHARS:
|
374
|
-
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
375
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
376
|
-
elif char == '[':
|
377
|
-
try:
|
378
|
-
second_char: ta.Optional[str] = src[pos + 1]
|
379
|
-
except IndexError:
|
380
|
-
second_char = None
|
381
|
-
out.flags.finalize_pending()
|
382
|
-
if second_char == '[':
|
383
|
-
pos, header = toml_create_list_rule(src, pos, out)
|
384
|
-
else:
|
385
|
-
pos, header = toml_create_dict_rule(src, pos, out)
|
386
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
387
|
-
elif char != '#':
|
388
|
-
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
260
|
+
class ProcessState(enum.IntEnum):
|
261
|
+
STOPPED = 0
|
262
|
+
STARTING = 10
|
263
|
+
RUNNING = 20
|
264
|
+
BACKOFF = 30
|
265
|
+
STOPPING = 40
|
266
|
+
EXITED = 100
|
267
|
+
FATAL = 200
|
268
|
+
UNKNOWN = 1000
|
389
269
|
|
390
|
-
|
391
|
-
|
270
|
+
@property
|
271
|
+
def stopped(self) -> bool:
|
272
|
+
return self in STOPPED_STATES
|
392
273
|
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
except IndexError:
|
397
|
-
break
|
398
|
-
if char != '\n':
|
399
|
-
raise toml_suffixed_err(
|
400
|
-
src, pos, 'Expected newline or end of document after a statement',
|
401
|
-
)
|
402
|
-
pos += 1
|
274
|
+
@property
|
275
|
+
def running(self) -> bool:
|
276
|
+
return self in RUNNING_STATES
|
403
277
|
|
404
|
-
|
278
|
+
@property
|
279
|
+
def signalable(self) -> bool:
|
280
|
+
return self in SIGNALABLE_STATES
|
405
281
|
|
406
282
|
|
407
|
-
|
408
|
-
|
283
|
+
# http://supervisord.org/subprocess.html
|
284
|
+
STATE_TRANSITIONS = {
|
285
|
+
ProcessState.STOPPED: (ProcessState.STARTING,),
|
286
|
+
ProcessState.STARTING: (ProcessState.RUNNING, ProcessState.BACKOFF, ProcessState.STOPPING),
|
287
|
+
ProcessState.RUNNING: (ProcessState.STOPPING, ProcessState.EXITED),
|
288
|
+
ProcessState.BACKOFF: (ProcessState.STARTING, ProcessState.FATAL),
|
289
|
+
ProcessState.STOPPING: (ProcessState.STOPPED,),
|
290
|
+
ProcessState.EXITED: (ProcessState.STARTING,),
|
291
|
+
ProcessState.FATAL: (ProcessState.STARTING,),
|
292
|
+
}
|
409
293
|
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
294
|
+
STOPPED_STATES = (
|
295
|
+
ProcessState.STOPPED,
|
296
|
+
ProcessState.EXITED,
|
297
|
+
ProcessState.FATAL,
|
298
|
+
ProcessState.UNKNOWN,
|
299
|
+
)
|
414
300
|
|
415
|
-
|
416
|
-
|
417
|
-
|
301
|
+
RUNNING_STATES = (
|
302
|
+
ProcessState.RUNNING,
|
303
|
+
ProcessState.BACKOFF,
|
304
|
+
ProcessState.STARTING,
|
305
|
+
)
|
418
306
|
|
419
|
-
|
420
|
-
|
307
|
+
SIGNALABLE_STATES = (
|
308
|
+
ProcessState.RUNNING,
|
309
|
+
ProcessState.STARTING,
|
310
|
+
ProcessState.STOPPING,
|
311
|
+
)
|
421
312
|
|
422
|
-
def finalize_pending(self) -> None:
|
423
|
-
for key, flag in self._pending_flags:
|
424
|
-
self.set(key, flag, recursive=False)
|
425
|
-
self._pending_flags.clear()
|
426
313
|
|
427
|
-
|
428
|
-
cont = self._flags
|
429
|
-
for k in key[:-1]:
|
430
|
-
if k not in cont:
|
431
|
-
return
|
432
|
-
cont = cont[k]['nested']
|
433
|
-
cont.pop(key[-1], None)
|
314
|
+
##
|
434
315
|
|
435
|
-
def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
|
436
|
-
cont = self._flags
|
437
|
-
key_parent, key_stem = key[:-1], key[-1]
|
438
|
-
for k in key_parent:
|
439
|
-
if k not in cont:
|
440
|
-
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
441
|
-
cont = cont[k]['nested']
|
442
|
-
if key_stem not in cont:
|
443
|
-
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
444
|
-
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
445
316
|
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
if k not in cont:
|
452
|
-
return False
|
453
|
-
inner_cont = cont[k]
|
454
|
-
if flag in inner_cont['recursive_flags']:
|
455
|
-
return True
|
456
|
-
cont = inner_cont['nested']
|
457
|
-
key_stem = key[-1]
|
458
|
-
if key_stem in cont:
|
459
|
-
cont = cont[key_stem]
|
460
|
-
return flag in cont['flags'] or flag in cont['recursive_flags']
|
461
|
-
return False
|
317
|
+
class SupervisorState(enum.IntEnum):
|
318
|
+
FATAL = 2
|
319
|
+
RUNNING = 1
|
320
|
+
RESTARTING = 0
|
321
|
+
SHUTDOWN = -1
|
462
322
|
|
463
323
|
|
464
|
-
|
465
|
-
|
466
|
-
# The parsed content of the TOML document
|
467
|
-
self.dict: ta.Dict[str, ta.Any] = {}
|
324
|
+
########################################
|
325
|
+
# ../utils/collections.py
|
468
326
|
|
469
|
-
def get_or_create_nest(
|
470
|
-
self,
|
471
|
-
key: TomlKey,
|
472
|
-
*,
|
473
|
-
access_lists: bool = True,
|
474
|
-
) -> dict:
|
475
|
-
cont: ta.Any = self.dict
|
476
|
-
for k in key:
|
477
|
-
if k not in cont:
|
478
|
-
cont[k] = {}
|
479
|
-
cont = cont[k]
|
480
|
-
if access_lists and isinstance(cont, list):
|
481
|
-
cont = cont[-1]
|
482
|
-
if not isinstance(cont, dict):
|
483
|
-
raise KeyError('There is no nest behind this key')
|
484
|
-
return cont
|
485
327
|
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
if not isinstance(list_, list):
|
492
|
-
raise KeyError('An object other than list found behind this key')
|
493
|
-
list_.append({})
|
494
|
-
else:
|
495
|
-
cont[last_key] = [{}]
|
328
|
+
class KeyedCollectionAccessors(abc.ABC, ta.Generic[K, V]):
|
329
|
+
@property
|
330
|
+
@abc.abstractmethod
|
331
|
+
def _by_key(self) -> ta.Mapping[K, V]:
|
332
|
+
raise NotImplementedError
|
496
333
|
|
334
|
+
def __iter__(self) -> ta.Iterator[V]:
|
335
|
+
return iter(self._by_key.values())
|
497
336
|
|
498
|
-
|
499
|
-
|
500
|
-
flags: TomlFlags
|
337
|
+
def __len__(self) -> int:
|
338
|
+
return len(self._by_key)
|
501
339
|
|
340
|
+
def __contains__(self, key: K) -> bool:
|
341
|
+
return key in self._by_key
|
502
342
|
|
503
|
-
def
|
504
|
-
|
505
|
-
while src[pos] in chars:
|
506
|
-
pos += 1
|
507
|
-
except IndexError:
|
508
|
-
pass
|
509
|
-
return pos
|
343
|
+
def __getitem__(self, key: K) -> V:
|
344
|
+
return self._by_key[key]
|
510
345
|
|
346
|
+
def get(self, key: K, default: ta.Optional[V] = None) -> ta.Optional[V]:
|
347
|
+
return self._by_key.get(key, default)
|
511
348
|
|
512
|
-
def
|
513
|
-
|
514
|
-
pos: TomlPos,
|
515
|
-
expect: str,
|
516
|
-
*,
|
517
|
-
error_on: ta.FrozenSet[str],
|
518
|
-
error_on_eof: bool,
|
519
|
-
) -> TomlPos:
|
520
|
-
try:
|
521
|
-
new_pos = src.index(expect, pos)
|
522
|
-
except ValueError:
|
523
|
-
new_pos = len(src)
|
524
|
-
if error_on_eof:
|
525
|
-
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
349
|
+
def items(self) -> ta.Iterator[ta.Tuple[K, V]]:
|
350
|
+
return iter(self._by_key.items())
|
526
351
|
|
527
|
-
if not error_on.isdisjoint(src[pos:new_pos]):
|
528
|
-
while src[pos] not in error_on:
|
529
|
-
pos += 1
|
530
|
-
raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
|
531
|
-
return new_pos
|
532
352
|
|
353
|
+
class KeyedCollection(KeyedCollectionAccessors[K, V]):
|
354
|
+
def __init__(self, items: ta.Iterable[V]) -> None:
|
355
|
+
super().__init__()
|
533
356
|
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
return toml_skip_until(
|
541
|
-
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
542
|
-
)
|
543
|
-
return pos
|
357
|
+
by_key: ta.Dict[K, V] = {}
|
358
|
+
for v in items:
|
359
|
+
if (k := self._key(v)) in by_key:
|
360
|
+
raise KeyError(f'key {k} of {v} already registered by {by_key[k]}')
|
361
|
+
by_key[k] = v
|
362
|
+
self.__by_key = by_key
|
544
363
|
|
364
|
+
@property
|
365
|
+
def _by_key(self) -> ta.Mapping[K, V]:
|
366
|
+
return self.__by_key
|
545
367
|
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
550
|
-
pos = toml_skip_comment(src, pos)
|
551
|
-
if pos == pos_before_skip:
|
552
|
-
return pos
|
368
|
+
@abc.abstractmethod
|
369
|
+
def _key(self, v: V) -> K:
|
370
|
+
raise NotImplementedError
|
553
371
|
|
554
372
|
|
555
|
-
|
556
|
-
|
557
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
558
|
-
pos, key = toml_parse_key(src, pos)
|
373
|
+
########################################
|
374
|
+
# ../utils/diag.py
|
559
375
|
|
560
|
-
if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
|
561
|
-
raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
|
562
|
-
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
563
|
-
try:
|
564
|
-
out.data.get_or_create_nest(key)
|
565
|
-
except KeyError:
|
566
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
567
376
|
|
568
|
-
|
569
|
-
|
570
|
-
|
377
|
+
def compact_traceback() -> ta.Tuple[
|
378
|
+
ta.Tuple[str, str, int],
|
379
|
+
ta.Type[BaseException],
|
380
|
+
BaseException,
|
381
|
+
types.TracebackType,
|
382
|
+
]:
|
383
|
+
t, v, tb = sys.exc_info()
|
384
|
+
if not tb:
|
385
|
+
raise RuntimeError('No traceback')
|
571
386
|
|
387
|
+
tbinfo = []
|
388
|
+
while tb:
|
389
|
+
tbinfo.append((
|
390
|
+
tb.tb_frame.f_code.co_filename,
|
391
|
+
tb.tb_frame.f_code.co_name,
|
392
|
+
str(tb.tb_lineno),
|
393
|
+
))
|
394
|
+
tb = tb.tb_next
|
572
395
|
|
573
|
-
|
574
|
-
|
575
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
576
|
-
pos, key = toml_parse_key(src, pos)
|
396
|
+
# just to be safe
|
397
|
+
del tb
|
577
398
|
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
out.flags.unset_all(key)
|
582
|
-
# ...but this key precisely is still prohibited from table declaration
|
583
|
-
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
584
|
-
try:
|
585
|
-
out.data.append_nest_to_list(key)
|
586
|
-
except KeyError:
|
587
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
399
|
+
file, function, line = tbinfo[-1]
|
400
|
+
info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo]) # noqa
|
401
|
+
return (file, function, line), t, v, info # type: ignore
|
588
402
|
|
589
|
-
if not src.startswith(']]', pos):
|
590
|
-
raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
|
591
|
-
return pos + 2, key
|
592
403
|
|
404
|
+
########################################
|
405
|
+
# ../utils/fs.py
|
593
406
|
|
594
|
-
def toml_key_value_rule(
|
595
|
-
src: str,
|
596
|
-
pos: TomlPos,
|
597
|
-
out: TomlOutput,
|
598
|
-
header: TomlKey,
|
599
|
-
parse_float: TomlParseFloat,
|
600
|
-
) -> TomlPos:
|
601
|
-
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
602
|
-
key_parent, key_stem = key[:-1], key[-1]
|
603
|
-
abs_key_parent = header + key_parent
|
604
407
|
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
# table sections.
|
612
|
-
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
408
|
+
def try_unlink(path: str) -> bool:
|
409
|
+
try:
|
410
|
+
os.unlink(path)
|
411
|
+
except OSError:
|
412
|
+
return False
|
413
|
+
return True
|
613
414
|
|
614
|
-
if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
|
615
|
-
raise toml_suffixed_err(
|
616
|
-
src,
|
617
|
-
pos,
|
618
|
-
f'Cannot mutate immutable namespace {abs_key_parent}',
|
619
|
-
)
|
620
415
|
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
if key_stem in nest:
|
626
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
627
|
-
# Mark inline table and array namespaces recursively immutable
|
628
|
-
if isinstance(value, (dict, list)):
|
629
|
-
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
630
|
-
nest[key_stem] = value
|
631
|
-
return pos
|
416
|
+
def mktempfile(suffix: str, prefix: str, dir: str) -> str: # noqa
|
417
|
+
fd, filename = tempfile.mkstemp(suffix, prefix, dir)
|
418
|
+
os.close(fd)
|
419
|
+
return filename
|
632
420
|
|
633
421
|
|
634
|
-
def
|
635
|
-
|
636
|
-
pos: TomlPos,
|
637
|
-
parse_float: TomlParseFloat,
|
638
|
-
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
639
|
-
pos, key = toml_parse_key(src, pos)
|
640
|
-
try:
|
641
|
-
char: ta.Optional[str] = src[pos]
|
642
|
-
except IndexError:
|
643
|
-
char = None
|
644
|
-
if char != '=':
|
645
|
-
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
646
|
-
pos += 1
|
647
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
648
|
-
pos, value = toml_parse_value(src, pos, parse_float)
|
649
|
-
return pos, key, value
|
422
|
+
def get_path() -> ta.Sequence[str]:
|
423
|
+
"""Return a list corresponding to $PATH, or a default."""
|
650
424
|
|
425
|
+
path = ['/bin', '/usr/bin', '/usr/local/bin']
|
426
|
+
if 'PATH' in os.environ:
|
427
|
+
p = os.environ['PATH']
|
428
|
+
if p:
|
429
|
+
path = p.split(os.pathsep)
|
430
|
+
return path
|
651
431
|
|
652
|
-
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
653
|
-
pos, key_part = toml_parse_key_part(src, pos)
|
654
|
-
key: TomlKey = (key_part,)
|
655
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
656
|
-
while True:
|
657
|
-
try:
|
658
|
-
char: ta.Optional[str] = src[pos]
|
659
|
-
except IndexError:
|
660
|
-
char = None
|
661
|
-
if char != '.':
|
662
|
-
return pos, key
|
663
|
-
pos += 1
|
664
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
665
|
-
pos, key_part = toml_parse_key_part(src, pos)
|
666
|
-
key += (key_part,)
|
667
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
668
432
|
|
433
|
+
def check_existing_dir(v: str) -> str:
|
434
|
+
nv = os.path.expanduser(v)
|
435
|
+
if os.path.isdir(nv):
|
436
|
+
return nv
|
437
|
+
raise ValueError(f'{v} is not an existing directory')
|
669
438
|
|
670
|
-
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
671
|
-
try:
|
672
|
-
char: ta.Optional[str] = src[pos]
|
673
|
-
except IndexError:
|
674
|
-
char = None
|
675
|
-
if char in TOML_BARE_KEY_CHARS:
|
676
|
-
start_pos = pos
|
677
|
-
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
678
|
-
return pos, src[start_pos:pos]
|
679
|
-
if char == "'":
|
680
|
-
return toml_parse_literal_str(src, pos)
|
681
|
-
if char == '"':
|
682
|
-
return toml_parse_one_line_basic_str(src, pos)
|
683
|
-
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
684
439
|
|
440
|
+
def check_path_with_existing_dir(v: str) -> str:
|
441
|
+
nv = os.path.expanduser(v)
|
442
|
+
dir = os.path.dirname(nv) # noqa
|
443
|
+
if not dir:
|
444
|
+
# relative pathname with no directory component
|
445
|
+
return nv
|
446
|
+
if os.path.isdir(dir):
|
447
|
+
return nv
|
448
|
+
raise ValueError(f'The directory named as part of the path {v} does not exist')
|
685
449
|
|
686
|
-
def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
687
|
-
pos += 1
|
688
|
-
return toml_parse_basic_str(src, pos, multiline=False)
|
689
450
|
|
451
|
+
########################################
|
452
|
+
# ../utils/ostypes.py
|
690
453
|
|
691
|
-
def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
|
692
|
-
pos += 1
|
693
|
-
array: list = []
|
694
454
|
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
while True:
|
699
|
-
pos, val = toml_parse_value(src, pos, parse_float)
|
700
|
-
array.append(val)
|
701
|
-
pos = toml_skip_comments_and_array_ws(src, pos)
|
455
|
+
Fd = ta.NewType('Fd', int)
|
456
|
+
Pid = ta.NewType('Pid', int)
|
457
|
+
Rc = ta.NewType('Rc', int)
|
702
458
|
|
703
|
-
|
704
|
-
|
705
|
-
return pos + 1, array
|
706
|
-
if c != ',':
|
707
|
-
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
708
|
-
pos += 1
|
459
|
+
Uid = ta.NewType('Uid', int)
|
460
|
+
Gid = ta.NewType('Gid', int)
|
709
461
|
|
710
|
-
pos = toml_skip_comments_and_array_ws(src, pos)
|
711
|
-
if src.startswith(']', pos):
|
712
|
-
return pos + 1, array
|
713
462
|
|
463
|
+
########################################
|
464
|
+
# ../utils/signals.py
|
714
465
|
|
715
|
-
def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
|
716
|
-
pos += 1
|
717
|
-
nested_dict = TomlNestedDict()
|
718
|
-
flags = TomlFlags()
|
719
466
|
|
720
|
-
|
721
|
-
if src.startswith('}', pos):
|
722
|
-
return pos + 1, nested_dict.dict
|
723
|
-
while True:
|
724
|
-
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
725
|
-
key_parent, key_stem = key[:-1], key[-1]
|
726
|
-
if flags.is_(key, TomlFlags.FROZEN):
|
727
|
-
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
728
|
-
try:
|
729
|
-
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
730
|
-
except KeyError:
|
731
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
732
|
-
if key_stem in nest:
|
733
|
-
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
734
|
-
nest[key_stem] = value
|
735
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
736
|
-
c = src[pos:pos + 1]
|
737
|
-
if c == '}':
|
738
|
-
return pos + 1, nested_dict.dict
|
739
|
-
if c != ',':
|
740
|
-
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
741
|
-
if isinstance(value, (dict, list)):
|
742
|
-
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
743
|
-
pos += 1
|
744
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
467
|
+
##
|
745
468
|
|
746
469
|
|
747
|
-
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
752
|
-
) -> ta.Tuple[TomlPos, str]:
|
753
|
-
escape_id = src[pos:pos + 2]
|
754
|
-
pos += 2
|
755
|
-
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
756
|
-
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
757
|
-
# newline.
|
758
|
-
if escape_id != '\\\n':
|
759
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
760
|
-
try:
|
761
|
-
char = src[pos]
|
762
|
-
except IndexError:
|
763
|
-
return pos, ''
|
764
|
-
if char != '\n':
|
765
|
-
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
766
|
-
pos += 1
|
767
|
-
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
768
|
-
return pos, ''
|
769
|
-
if escape_id == '\\u':
|
770
|
-
return toml_parse_hex_char(src, pos, 4)
|
771
|
-
if escape_id == '\\U':
|
772
|
-
return toml_parse_hex_char(src, pos, 8)
|
470
|
+
_SIGS_BY_NUM: ta.Mapping[int, signal.Signals] = {s.value: s for s in signal.Signals}
|
471
|
+
_SIGS_BY_NAME: ta.Mapping[str, signal.Signals] = {s.name: s for s in signal.Signals}
|
472
|
+
|
473
|
+
|
474
|
+
def sig_num(value: ta.Union[int, str]) -> int:
|
773
475
|
try:
|
774
|
-
|
775
|
-
except KeyError:
|
776
|
-
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
476
|
+
num = int(value)
|
777
477
|
|
478
|
+
except (ValueError, TypeError):
|
479
|
+
name = value.strip().upper() # type: ignore
|
480
|
+
if not name.startswith('SIG'):
|
481
|
+
name = f'SIG{name}'
|
778
482
|
|
779
|
-
|
780
|
-
|
483
|
+
if (sn := _SIGS_BY_NAME.get(name)) is None:
|
484
|
+
raise ValueError(f'value {value!r} is not a valid signal name') # noqa
|
485
|
+
num = sn
|
781
486
|
|
487
|
+
if num not in _SIGS_BY_NUM:
|
488
|
+
raise ValueError(f'value {value!r} is not a valid signal number')
|
782
489
|
|
783
|
-
|
784
|
-
hex_str = src[pos:pos + hex_len]
|
785
|
-
if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
|
786
|
-
raise toml_suffixed_err(src, pos, 'Invalid hex value')
|
787
|
-
pos += hex_len
|
788
|
-
hex_int = int(hex_str, 16)
|
789
|
-
if not toml_is_unicode_scalar_value(hex_int):
|
790
|
-
raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
|
791
|
-
return pos, chr(hex_int)
|
490
|
+
return num
|
792
491
|
|
793
492
|
|
794
|
-
def
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
|
799
|
-
)
|
800
|
-
return pos + 1, src[start_pos:pos] # Skip ending apostrophe
|
493
|
+
def sig_name(num: int) -> str:
|
494
|
+
if (sig := _SIGS_BY_NUM.get(num)) is not None:
|
495
|
+
return sig.name
|
496
|
+
return f'signal {sig}'
|
801
497
|
|
802
498
|
|
803
|
-
|
804
|
-
pos += 3
|
805
|
-
if src.startswith('\n', pos):
|
806
|
-
pos += 1
|
499
|
+
##
|
807
500
|
|
808
|
-
if literal:
|
809
|
-
delim = "'"
|
810
|
-
end_pos = toml_skip_until(
|
811
|
-
src,
|
812
|
-
pos,
|
813
|
-
"'''",
|
814
|
-
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
815
|
-
error_on_eof=True,
|
816
|
-
)
|
817
|
-
result = src[pos:end_pos]
|
818
|
-
pos = end_pos + 3
|
819
|
-
else:
|
820
|
-
delim = '"'
|
821
|
-
pos, result = toml_parse_basic_str(src, pos, multiline=True)
|
822
501
|
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
pos += 1
|
827
|
-
if not src.startswith(delim, pos):
|
828
|
-
return pos, result + delim
|
829
|
-
pos += 1
|
830
|
-
return pos, result + (delim * 2)
|
502
|
+
class SignalReceiver:
|
503
|
+
def __init__(self) -> None:
|
504
|
+
super().__init__()
|
831
505
|
|
506
|
+
self._signals_recvd: ta.List[int] = []
|
832
507
|
|
833
|
-
def
|
834
|
-
|
835
|
-
|
836
|
-
|
508
|
+
def receive(self, sig: int, frame: ta.Any = None) -> None:
|
509
|
+
if sig not in self._signals_recvd:
|
510
|
+
self._signals_recvd.append(sig)
|
511
|
+
|
512
|
+
def install(self, *sigs: int) -> None:
|
513
|
+
for sig in sigs:
|
514
|
+
signal.signal(sig, self.receive)
|
515
|
+
|
516
|
+
def get_signal(self) -> ta.Optional[int]:
|
517
|
+
if self._signals_recvd:
|
518
|
+
sig = self._signals_recvd.pop(0)
|
519
|
+
else:
|
520
|
+
sig = None
|
521
|
+
return sig
|
522
|
+
|
523
|
+
|
524
|
+
########################################
|
525
|
+
# ../utils/strings.py
|
526
|
+
|
527
|
+
|
528
|
+
##
|
529
|
+
|
530
|
+
|
531
|
+
def as_bytes(s: ta.Union[str, bytes], encoding: str = 'utf8') -> bytes:
|
532
|
+
if isinstance(s, bytes):
|
533
|
+
return s
|
837
534
|
else:
|
838
|
-
|
839
|
-
parse_escapes = toml_parse_basic_str_escape
|
840
|
-
result = ''
|
841
|
-
start_pos = pos
|
842
|
-
while True:
|
843
|
-
try:
|
844
|
-
char = src[pos]
|
845
|
-
except IndexError:
|
846
|
-
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
847
|
-
if char == '"':
|
848
|
-
if not multiline:
|
849
|
-
return pos + 1, result + src[start_pos:pos]
|
850
|
-
if src.startswith('"""', pos):
|
851
|
-
return pos + 3, result + src[start_pos:pos]
|
852
|
-
pos += 1
|
853
|
-
continue
|
854
|
-
if char == '\\':
|
855
|
-
result += src[start_pos:pos]
|
856
|
-
pos, parsed_escape = parse_escapes(src, pos)
|
857
|
-
result += parsed_escape
|
858
|
-
start_pos = pos
|
859
|
-
continue
|
860
|
-
if char in error_on:
|
861
|
-
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
862
|
-
pos += 1
|
535
|
+
return s.encode(encoding)
|
863
536
|
|
864
537
|
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
parse_float: TomlParseFloat,
|
869
|
-
) -> ta.Tuple[TomlPos, ta.Any]:
|
870
|
-
try:
|
871
|
-
char: ta.Optional[str] = src[pos]
|
872
|
-
except IndexError:
|
873
|
-
char = None
|
538
|
+
@ta.overload
|
539
|
+
def find_prefix_at_end(haystack: str, needle: str) -> int:
|
540
|
+
...
|
874
541
|
|
875
|
-
# IMPORTANT: order conditions based on speed of checking and likelihood
|
876
542
|
|
877
|
-
|
878
|
-
|
879
|
-
|
880
|
-
return toml_parse_multiline_str(src, pos, literal=False)
|
881
|
-
return toml_parse_one_line_basic_str(src, pos)
|
543
|
+
@ta.overload
|
544
|
+
def find_prefix_at_end(haystack: bytes, needle: bytes) -> int:
|
545
|
+
...
|
882
546
|
|
883
|
-
# Literal strings
|
884
|
-
if char == "'":
|
885
|
-
if src.startswith("'''", pos):
|
886
|
-
return toml_parse_multiline_str(src, pos, literal=True)
|
887
|
-
return toml_parse_literal_str(src, pos)
|
888
547
|
|
889
|
-
|
890
|
-
|
891
|
-
|
892
|
-
|
893
|
-
|
894
|
-
if src.startswith('false', pos):
|
895
|
-
return pos + 5, False
|
548
|
+
def find_prefix_at_end(haystack, needle):
|
549
|
+
l = len(needle) - 1
|
550
|
+
while l and not haystack.endswith(needle[:l]):
|
551
|
+
l -= 1
|
552
|
+
return l
|
896
553
|
|
897
|
-
# Arrays
|
898
|
-
if char == '[':
|
899
|
-
return toml_parse_array(src, pos, parse_float)
|
900
554
|
|
901
|
-
|
902
|
-
|
903
|
-
|
555
|
+
##
|
556
|
+
|
557
|
+
|
558
|
+
ANSI_ESCAPE_BEGIN = b'\x1b['
|
559
|
+
ANSI_TERMINATORS = (b'H', b'f', b'A', b'B', b'C', b'D', b'R', b's', b'u', b'J', b'K', b'h', b'l', b'p', b'm')
|
560
|
+
|
561
|
+
|
562
|
+
def strip_escapes(s: bytes) -> bytes:
|
563
|
+
"""Remove all ANSI color escapes from the given string."""
|
564
|
+
|
565
|
+
result = b''
|
566
|
+
show = 1
|
567
|
+
i = 0
|
568
|
+
l = len(s)
|
569
|
+
while i < l:
|
570
|
+
if show == 0 and s[i:i + 1] in ANSI_TERMINATORS:
|
571
|
+
show = 1
|
572
|
+
elif show:
|
573
|
+
n = s.find(ANSI_ESCAPE_BEGIN, i)
|
574
|
+
if n == -1:
|
575
|
+
return result + s[i:]
|
576
|
+
else:
|
577
|
+
result = result + s[i:n]
|
578
|
+
i = n
|
579
|
+
show = 0
|
580
|
+
i += 1
|
581
|
+
return result
|
582
|
+
|
583
|
+
|
584
|
+
##
|
585
|
+
|
586
|
+
|
587
|
+
class SuffixMultiplier:
|
588
|
+
# d is a dictionary of suffixes to integer multipliers. If no suffixes match, default is the multiplier. Matches are
|
589
|
+
# case insensitive. Return values are in the fundamental unit.
|
590
|
+
def __init__(self, d, default=1):
|
591
|
+
super().__init__()
|
592
|
+
self._d = d
|
593
|
+
self._default = default
|
594
|
+
# all keys must be the same size
|
595
|
+
self._keysz = None
|
596
|
+
for k in d:
|
597
|
+
if self._keysz is None:
|
598
|
+
self._keysz = len(k)
|
599
|
+
elif self._keysz != len(k): # type: ignore
|
600
|
+
raise ValueError(k)
|
601
|
+
|
602
|
+
def __call__(self, v: ta.Union[str, int]) -> int:
|
603
|
+
if isinstance(v, int):
|
604
|
+
return v
|
605
|
+
v = v.lower()
|
606
|
+
for s, m in self._d.items():
|
607
|
+
if v[-self._keysz:] == s: # type: ignore
|
608
|
+
return int(v[:-self._keysz]) * m # type: ignore
|
609
|
+
return int(v) * self._default
|
610
|
+
|
611
|
+
|
612
|
+
parse_bytes_size = SuffixMultiplier({
|
613
|
+
'kb': 1024,
|
614
|
+
'mb': 1024 * 1024,
|
615
|
+
'gb': 1024 * 1024 * 1024,
|
616
|
+
})
|
617
|
+
|
618
|
+
|
619
|
+
#
|
620
|
+
|
621
|
+
|
622
|
+
def parse_octal(arg: ta.Union[str, int]) -> int:
|
623
|
+
if isinstance(arg, int):
|
624
|
+
return arg
|
625
|
+
try:
|
626
|
+
return int(arg, 8)
|
627
|
+
except (TypeError, ValueError):
|
628
|
+
raise ValueError(f'{arg} can not be converted to an octal type') # noqa
|
629
|
+
|
630
|
+
|
631
|
+
########################################
|
632
|
+
# ../../../omlish/configs/types.py
|
904
633
|
|
905
|
-
# Dates and times
|
906
|
-
datetime_match = TOML_RE_DATETIME.match(src, pos)
|
907
|
-
if datetime_match:
|
908
|
-
try:
|
909
|
-
datetime_obj = toml_match_to_datetime(datetime_match)
|
910
|
-
except ValueError as e:
|
911
|
-
raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
|
912
|
-
return datetime_match.end(), datetime_obj
|
913
|
-
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
914
|
-
if localtime_match:
|
915
|
-
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
916
634
|
|
917
|
-
|
918
|
-
# located after handling of dates and times.
|
919
|
-
number_match = TOML_RE_NUMBER.match(src, pos)
|
920
|
-
if number_match:
|
921
|
-
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
635
|
+
#
|
922
636
|
|
923
|
-
# Special floats
|
924
|
-
first_three = src[pos:pos + 3]
|
925
|
-
if first_three in {'inf', 'nan'}:
|
926
|
-
return pos + 3, parse_float(first_three)
|
927
|
-
first_four = src[pos:pos + 4]
|
928
|
-
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
929
|
-
return pos + 4, parse_float(first_four)
|
930
637
|
|
931
|
-
|
638
|
+
########################################
|
639
|
+
# ../../../omlish/formats/ini/sections.py
|
932
640
|
|
933
641
|
|
934
|
-
|
935
|
-
"""Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
|
642
|
+
##
|
936
643
|
|
937
|
-
def coord_repr(src: str, pos: TomlPos) -> str:
|
938
|
-
if pos >= len(src):
|
939
|
-
return 'end of document'
|
940
|
-
line = src.count('\n', 0, pos) + 1
|
941
|
-
if line == 1:
|
942
|
-
column = pos + 1
|
943
|
-
else:
|
944
|
-
column = pos - src.rindex('\n', 0, pos)
|
945
|
-
return f'line {line}, column {column}'
|
946
644
|
|
947
|
-
|
645
|
+
def extract_ini_sections(cp: configparser.ConfigParser) -> IniSectionSettingsMap:
|
646
|
+
config_dct: ta.Dict[str, ta.Any] = {}
|
647
|
+
for sec in cp.sections():
|
648
|
+
cd = config_dct
|
649
|
+
for k in sec.split('.'):
|
650
|
+
cd = cd.setdefault(k, {})
|
651
|
+
cd.update(cp.items(sec))
|
652
|
+
return config_dct
|
948
653
|
|
949
654
|
|
950
|
-
|
951
|
-
return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
|
655
|
+
##
|
952
656
|
|
953
657
|
|
954
|
-
def
|
955
|
-
|
658
|
+
def render_ini_sections(
|
659
|
+
settings_by_section: IniSectionSettingsMap,
|
660
|
+
) -> str:
|
661
|
+
out = io.StringIO()
|
956
662
|
|
957
|
-
|
958
|
-
|
959
|
-
|
960
|
-
# The default `float` callable never returns illegal types. Optimize it.
|
961
|
-
if parse_float is float:
|
962
|
-
return float
|
663
|
+
for i, (section, settings) in enumerate(settings_by_section.items()):
|
664
|
+
if i:
|
665
|
+
out.write('\n')
|
963
666
|
|
964
|
-
|
965
|
-
float_value = parse_float(float_str)
|
966
|
-
if isinstance(float_value, (dict, list)):
|
967
|
-
raise ValueError('parse_float must not return dicts or lists') # noqa
|
968
|
-
return float_value
|
667
|
+
out.write(f'[{section}]\n')
|
969
668
|
|
970
|
-
|
669
|
+
for k, v in settings.items():
|
670
|
+
if isinstance(v, str):
|
671
|
+
out.write(f'{k}={v}\n')
|
672
|
+
else:
|
673
|
+
for vv in v:
|
674
|
+
out.write(f'{k}={vv}\n')
|
675
|
+
|
676
|
+
return out.getvalue()
|
971
677
|
|
972
678
|
|
973
679
|
########################################
|
974
|
-
#
|
680
|
+
# ../../../omlish/formats/toml/parser.py
|
681
|
+
# SPDX-License-Identifier: MIT
|
682
|
+
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
683
|
+
# Licensed to PSF under a Contributor Agreement.
|
684
|
+
#
|
685
|
+
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
686
|
+
# --------------------------------------------
|
687
|
+
#
|
688
|
+
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
689
|
+
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
690
|
+
# documentation.
|
691
|
+
#
|
692
|
+
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
693
|
+
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
694
|
+
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
695
|
+
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
696
|
+
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
697
|
+
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
698
|
+
#
|
699
|
+
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
700
|
+
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
701
|
+
# any such work a brief summary of the changes made to Python.
|
702
|
+
#
|
703
|
+
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
704
|
+
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
705
|
+
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
706
|
+
# RIGHTS.
|
707
|
+
#
|
708
|
+
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
709
|
+
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
710
|
+
# ADVISED OF THE POSSIBILITY THEREOF.
|
711
|
+
#
|
712
|
+
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
713
|
+
#
|
714
|
+
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
715
|
+
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
716
|
+
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
717
|
+
#
|
718
|
+
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
719
|
+
# License Agreement.
|
720
|
+
#
|
721
|
+
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
975
722
|
|
976
723
|
|
977
|
-
|
978
|
-
"""Specialized exceptions used when attempting to start a process."""
|
724
|
+
##
|
979
725
|
|
980
726
|
|
981
|
-
|
982
|
-
|
727
|
+
_TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
|
728
|
+
|
729
|
+
TOML_RE_NUMBER = re.compile(
|
730
|
+
r"""
|
731
|
+
0
|
732
|
+
(?:
|
733
|
+
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
734
|
+
|
|
735
|
+
b[01](?:_?[01])* # bin
|
736
|
+
|
|
737
|
+
o[0-7](?:_?[0-7])* # oct
|
738
|
+
)
|
739
|
+
|
|
740
|
+
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
741
|
+
(?P<floatpart>
|
742
|
+
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
743
|
+
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
744
|
+
)
|
745
|
+
""",
|
746
|
+
flags=re.VERBOSE,
|
747
|
+
)
|
748
|
+
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
749
|
+
TOML_RE_DATETIME = re.compile(
|
750
|
+
rf"""
|
751
|
+
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
752
|
+
(?:
|
753
|
+
[Tt ]
|
754
|
+
{_TOML_TIME_RE_STR}
|
755
|
+
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
756
|
+
)?
|
757
|
+
""",
|
758
|
+
flags=re.VERBOSE,
|
759
|
+
)
|
983
760
|
|
984
761
|
|
985
|
-
|
986
|
-
"""
|
987
|
-
|
988
|
-
|
762
|
+
def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
|
763
|
+
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
764
|
+
|
765
|
+
Raises ValueError if the match does not correspond to a valid date or datetime.
|
989
766
|
"""
|
767
|
+
(
|
768
|
+
year_str,
|
769
|
+
month_str,
|
770
|
+
day_str,
|
771
|
+
hour_str,
|
772
|
+
minute_str,
|
773
|
+
sec_str,
|
774
|
+
micros_str,
|
775
|
+
zulu_time,
|
776
|
+
offset_sign_str,
|
777
|
+
offset_hour_str,
|
778
|
+
offset_minute_str,
|
779
|
+
) = match.groups()
|
780
|
+
year, month, day = int(year_str), int(month_str), int(day_str)
|
781
|
+
if hour_str is None:
|
782
|
+
return datetime.date(year, month, day)
|
783
|
+
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
784
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
785
|
+
if offset_sign_str:
|
786
|
+
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
787
|
+
offset_hour_str, offset_minute_str, offset_sign_str,
|
788
|
+
)
|
789
|
+
elif zulu_time:
|
790
|
+
tz = datetime.UTC
|
791
|
+
else: # local date-time
|
792
|
+
tz = None
|
793
|
+
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
990
794
|
|
991
795
|
|
992
|
-
|
993
|
-
|
796
|
+
@functools.lru_cache() # noqa
|
797
|
+
def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
|
798
|
+
sign = 1 if sign_str == '+' else -1
|
799
|
+
return datetime.timezone(
|
800
|
+
datetime.timedelta(
|
801
|
+
hours=sign * int(hour_str),
|
802
|
+
minutes=sign * int(minute_str),
|
803
|
+
),
|
804
|
+
)
|
994
805
|
|
995
806
|
|
996
|
-
|
997
|
-
|
998
|
-
|
999
|
-
|
1000
|
-
"""
|
807
|
+
def toml_match_to_localtime(match: re.Match) -> datetime.time:
|
808
|
+
hour_str, minute_str, sec_str, micros_str = match.groups()
|
809
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
810
|
+
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
1001
811
|
|
1002
812
|
|
1003
|
-
|
1004
|
-
|
813
|
+
def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
|
814
|
+
if match.group('floatpart'):
|
815
|
+
return parse_float(match.group())
|
816
|
+
return int(match.group(), 0)
|
1005
817
|
|
1006
818
|
|
1007
|
-
|
1008
|
-
"""
|
1009
|
-
Drop privileges to become the specified user, which may be a username or uid. Called for supervisord startup and
|
1010
|
-
when spawning subprocesses. Returns None on success or a string error message if privileges could not be dropped.
|
1011
|
-
"""
|
819
|
+
TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
|
1012
820
|
|
1013
|
-
|
1014
|
-
|
821
|
+
# Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
|
822
|
+
# functions.
|
823
|
+
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
824
|
+
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
1015
825
|
|
1016
|
-
|
1017
|
-
|
1018
|
-
uid = int(user)
|
1019
|
-
except ValueError:
|
1020
|
-
try:
|
1021
|
-
pwrec = pwd.getpwnam(user) # type: ignore
|
1022
|
-
except KeyError:
|
1023
|
-
return f"Can't find username {user!r}"
|
1024
|
-
uid = pwrec[2]
|
1025
|
-
else:
|
1026
|
-
try:
|
1027
|
-
pwrec = pwd.getpwuid(uid)
|
1028
|
-
except KeyError:
|
1029
|
-
return f"Can't find uid {uid!r}"
|
826
|
+
TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
827
|
+
TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1030
828
|
|
1031
|
-
|
829
|
+
TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
1032
830
|
|
1033
|
-
|
1034
|
-
|
1035
|
-
|
1036
|
-
|
831
|
+
TOML_WS = frozenset(' \t')
|
832
|
+
TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
|
833
|
+
TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
|
834
|
+
TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
|
835
|
+
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
1037
836
|
|
1038
|
-
|
1039
|
-
|
837
|
+
TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
|
838
|
+
{
|
839
|
+
'\\b': '\u0008', # backspace
|
840
|
+
'\\t': '\u0009', # tab
|
841
|
+
'\\n': '\u000A', # linefeed
|
842
|
+
'\\f': '\u000C', # form feed
|
843
|
+
'\\r': '\u000D', # carriage return
|
844
|
+
'\\"': '\u0022', # quote
|
845
|
+
'\\\\': '\u005C', # backslash
|
846
|
+
},
|
847
|
+
)
|
1040
848
|
|
1041
|
-
gid = pwrec[3]
|
1042
|
-
if hasattr(os, 'setgroups'):
|
1043
|
-
user = pwrec[0]
|
1044
|
-
groups = [grprec[2] for grprec in grp.getgrall() if user in grprec[3]]
|
1045
849
|
|
1046
|
-
|
1047
|
-
|
1048
|
-
# python 2.7 - this will be safe though for all unix /python version combos)
|
1049
|
-
groups.insert(0, gid)
|
1050
|
-
try:
|
1051
|
-
os.setgroups(groups)
|
1052
|
-
except OSError:
|
1053
|
-
return 'Could not set groups of effective user'
|
850
|
+
class TomlDecodeError(ValueError):
|
851
|
+
"""An error raised if a document is not valid TOML."""
|
1054
852
|
|
853
|
+
|
854
|
+
def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
|
855
|
+
"""Parse TOML from a binary file object."""
|
856
|
+
b = fp.read()
|
1055
857
|
try:
|
1056
|
-
|
1057
|
-
except
|
1058
|
-
|
858
|
+
s = b.decode()
|
859
|
+
except AttributeError:
|
860
|
+
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
861
|
+
return toml_loads(s, parse_float=parse_float)
|
1059
862
|
|
1060
|
-
os.setuid(uid)
|
1061
863
|
|
1062
|
-
|
864
|
+
def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
|
865
|
+
"""Parse TOML from a string."""
|
1063
866
|
|
867
|
+
# The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
|
868
|
+
try:
|
869
|
+
src = s.replace('\r\n', '\n')
|
870
|
+
except (AttributeError, TypeError):
|
871
|
+
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
872
|
+
pos = 0
|
873
|
+
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
874
|
+
header: TomlKey = ()
|
875
|
+
parse_float = toml_make_safe_parse_float(parse_float)
|
1064
876
|
|
1065
|
-
|
1066
|
-
|
877
|
+
# Parse one statement at a time (typically means one line in TOML source)
|
878
|
+
while True:
|
879
|
+
# 1. Skip line leading whitespace
|
880
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1067
881
|
|
882
|
+
# 2. Parse rules. Expect one of the following:
|
883
|
+
# - end of file
|
884
|
+
# - end of line
|
885
|
+
# - comment
|
886
|
+
# - key/value pair
|
887
|
+
# - append dict to list (and move to its namespace)
|
888
|
+
# - create dict (and move to its namespace)
|
889
|
+
# Skip trailing whitespace when applicable.
|
890
|
+
try:
|
891
|
+
char = src[pos]
|
892
|
+
except IndexError:
|
893
|
+
break
|
894
|
+
if char == '\n':
|
895
|
+
pos += 1
|
896
|
+
continue
|
897
|
+
if char in TOML_KEY_INITIAL_CHARS:
|
898
|
+
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
899
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
900
|
+
elif char == '[':
|
901
|
+
try:
|
902
|
+
second_char: ta.Optional[str] = src[pos + 1]
|
903
|
+
except IndexError:
|
904
|
+
second_char = None
|
905
|
+
out.flags.finalize_pending()
|
906
|
+
if second_char == '[':
|
907
|
+
pos, header = toml_create_list_rule(src, pos, out)
|
908
|
+
else:
|
909
|
+
pos, header = toml_create_dict_rule(src, pos, out)
|
910
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
911
|
+
elif char != '#':
|
912
|
+
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
1068
913
|
|
1069
|
-
|
914
|
+
# 3. Skip comment
|
915
|
+
pos = toml_skip_comment(src, pos)
|
1070
916
|
|
917
|
+
# 4. Expect end of line or end of file
|
918
|
+
try:
|
919
|
+
char = src[pos]
|
920
|
+
except IndexError:
|
921
|
+
break
|
922
|
+
if char != '\n':
|
923
|
+
raise toml_suffixed_err(
|
924
|
+
src, pos, 'Expected newline or end of document after a statement',
|
925
|
+
)
|
926
|
+
pos += 1
|
1071
927
|
|
1072
|
-
|
1073
|
-
STOPPED = 0
|
1074
|
-
STARTING = 10
|
1075
|
-
RUNNING = 20
|
1076
|
-
BACKOFF = 30
|
1077
|
-
STOPPING = 40
|
1078
|
-
EXITED = 100
|
1079
|
-
FATAL = 200
|
1080
|
-
UNKNOWN = 1000
|
928
|
+
return out.data.dict
|
1081
929
|
|
1082
|
-
@property
|
1083
|
-
def stopped(self) -> bool:
|
1084
|
-
return self in STOPPED_STATES
|
1085
930
|
|
1086
|
-
|
1087
|
-
|
1088
|
-
return self in RUNNING_STATES
|
931
|
+
class TomlFlags:
|
932
|
+
"""Flags that map to parsed keys/namespaces."""
|
1089
933
|
|
1090
|
-
|
1091
|
-
|
1092
|
-
|
934
|
+
# Marks an immutable namespace (inline array or inline table).
|
935
|
+
FROZEN = 0
|
936
|
+
# Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
|
937
|
+
EXPLICIT_NEST = 1
|
1093
938
|
|
939
|
+
def __init__(self) -> None:
|
940
|
+
self._flags: ta.Dict[str, dict] = {}
|
941
|
+
self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
|
1094
942
|
|
1095
|
-
|
1096
|
-
|
1097
|
-
ProcessState.STOPPED: (ProcessState.STARTING,),
|
1098
|
-
ProcessState.STARTING: (ProcessState.RUNNING, ProcessState.BACKOFF, ProcessState.STOPPING),
|
1099
|
-
ProcessState.RUNNING: (ProcessState.STOPPING, ProcessState.EXITED),
|
1100
|
-
ProcessState.BACKOFF: (ProcessState.STARTING, ProcessState.FATAL),
|
1101
|
-
ProcessState.STOPPING: (ProcessState.STOPPED,),
|
1102
|
-
ProcessState.EXITED: (ProcessState.STARTING,),
|
1103
|
-
ProcessState.FATAL: (ProcessState.STARTING,),
|
1104
|
-
}
|
943
|
+
def add_pending(self, key: TomlKey, flag: int) -> None:
|
944
|
+
self._pending_flags.add((key, flag))
|
1105
945
|
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
1109
|
-
|
1110
|
-
ProcessState.UNKNOWN,
|
1111
|
-
)
|
946
|
+
def finalize_pending(self) -> None:
|
947
|
+
for key, flag in self._pending_flags:
|
948
|
+
self.set(key, flag, recursive=False)
|
949
|
+
self._pending_flags.clear()
|
1112
950
|
|
1113
|
-
|
1114
|
-
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
951
|
+
def unset_all(self, key: TomlKey) -> None:
|
952
|
+
cont = self._flags
|
953
|
+
for k in key[:-1]:
|
954
|
+
if k not in cont:
|
955
|
+
return
|
956
|
+
cont = cont[k]['nested']
|
957
|
+
cont.pop(key[-1], None)
|
1118
958
|
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
959
|
+
def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
|
960
|
+
cont = self._flags
|
961
|
+
key_parent, key_stem = key[:-1], key[-1]
|
962
|
+
for k in key_parent:
|
963
|
+
if k not in cont:
|
964
|
+
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
965
|
+
cont = cont[k]['nested']
|
966
|
+
if key_stem not in cont:
|
967
|
+
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
968
|
+
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
1124
969
|
|
970
|
+
def is_(self, key: TomlKey, flag: int) -> bool:
|
971
|
+
if not key:
|
972
|
+
return False # document root has no flags
|
973
|
+
cont = self._flags
|
974
|
+
for k in key[:-1]:
|
975
|
+
if k not in cont:
|
976
|
+
return False
|
977
|
+
inner_cont = cont[k]
|
978
|
+
if flag in inner_cont['recursive_flags']:
|
979
|
+
return True
|
980
|
+
cont = inner_cont['nested']
|
981
|
+
key_stem = key[-1]
|
982
|
+
if key_stem in cont:
|
983
|
+
cont = cont[key_stem]
|
984
|
+
return flag in cont['flags'] or flag in cont['recursive_flags']
|
985
|
+
return False
|
1125
986
|
|
1126
|
-
##
|
1127
987
|
|
988
|
+
class TomlNestedDict:
|
989
|
+
def __init__(self) -> None:
|
990
|
+
# The parsed content of the TOML document
|
991
|
+
self.dict: ta.Dict[str, ta.Any] = {}
|
1128
992
|
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
993
|
+
def get_or_create_nest(
|
994
|
+
self,
|
995
|
+
key: TomlKey,
|
996
|
+
*,
|
997
|
+
access_lists: bool = True,
|
998
|
+
) -> dict:
|
999
|
+
cont: ta.Any = self.dict
|
1000
|
+
for k in key:
|
1001
|
+
if k not in cont:
|
1002
|
+
cont[k] = {}
|
1003
|
+
cont = cont[k]
|
1004
|
+
if access_lists and isinstance(cont, list):
|
1005
|
+
cont = cont[-1]
|
1006
|
+
if not isinstance(cont, dict):
|
1007
|
+
raise KeyError('There is no nest behind this key')
|
1008
|
+
return cont
|
1134
1009
|
|
1010
|
+
def append_nest_to_list(self, key: TomlKey) -> None:
|
1011
|
+
cont = self.get_or_create_nest(key[:-1])
|
1012
|
+
last_key = key[-1]
|
1013
|
+
if last_key in cont:
|
1014
|
+
list_ = cont[last_key]
|
1015
|
+
if not isinstance(list_, list):
|
1016
|
+
raise KeyError('An object other than list found behind this key')
|
1017
|
+
list_.append({})
|
1018
|
+
else:
|
1019
|
+
cont[last_key] = [{}]
|
1135
1020
|
|
1136
|
-
########################################
|
1137
|
-
# ../utils/collections.py
|
1138
1021
|
|
1022
|
+
class TomlOutput(ta.NamedTuple):
|
1023
|
+
data: TomlNestedDict
|
1024
|
+
flags: TomlFlags
|
1139
1025
|
|
1140
|
-
class KeyedCollectionAccessors(abc.ABC, ta.Generic[K, V]):
|
1141
|
-
@property
|
1142
|
-
@abc.abstractmethod
|
1143
|
-
def _by_key(self) -> ta.Mapping[K, V]:
|
1144
|
-
raise NotImplementedError
|
1145
1026
|
|
1146
|
-
|
1147
|
-
|
1027
|
+
def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
|
1028
|
+
try:
|
1029
|
+
while src[pos] in chars:
|
1030
|
+
pos += 1
|
1031
|
+
except IndexError:
|
1032
|
+
pass
|
1033
|
+
return pos
|
1148
1034
|
|
1149
|
-
def __len__(self) -> int:
|
1150
|
-
return len(self._by_key)
|
1151
1035
|
|
1152
|
-
|
1153
|
-
|
1036
|
+
def toml_skip_until(
|
1037
|
+
src: str,
|
1038
|
+
pos: TomlPos,
|
1039
|
+
expect: str,
|
1040
|
+
*,
|
1041
|
+
error_on: ta.FrozenSet[str],
|
1042
|
+
error_on_eof: bool,
|
1043
|
+
) -> TomlPos:
|
1044
|
+
try:
|
1045
|
+
new_pos = src.index(expect, pos)
|
1046
|
+
except ValueError:
|
1047
|
+
new_pos = len(src)
|
1048
|
+
if error_on_eof:
|
1049
|
+
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
1050
|
+
|
1051
|
+
if not error_on.isdisjoint(src[pos:new_pos]):
|
1052
|
+
while src[pos] not in error_on:
|
1053
|
+
pos += 1
|
1054
|
+
raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
|
1055
|
+
return new_pos
|
1154
1056
|
|
1155
|
-
def __getitem__(self, key: K) -> V:
|
1156
|
-
return self._by_key[key]
|
1157
1057
|
|
1158
|
-
|
1159
|
-
|
1058
|
+
def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
|
1059
|
+
try:
|
1060
|
+
char: ta.Optional[str] = src[pos]
|
1061
|
+
except IndexError:
|
1062
|
+
char = None
|
1063
|
+
if char == '#':
|
1064
|
+
return toml_skip_until(
|
1065
|
+
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
1066
|
+
)
|
1067
|
+
return pos
|
1160
1068
|
|
1161
|
-
def items(self) -> ta.Iterator[ta.Tuple[K, V]]:
|
1162
|
-
return iter(self._by_key.items())
|
1163
1069
|
|
1070
|
+
def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
|
1071
|
+
while True:
|
1072
|
+
pos_before_skip = pos
|
1073
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1074
|
+
pos = toml_skip_comment(src, pos)
|
1075
|
+
if pos == pos_before_skip:
|
1076
|
+
return pos
|
1164
1077
|
|
1165
|
-
class KeyedCollection(KeyedCollectionAccessors[K, V]):
|
1166
|
-
def __init__(self, items: ta.Iterable[V]) -> None:
|
1167
|
-
super().__init__()
|
1168
1078
|
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1173
|
-
by_key[k] = v
|
1174
|
-
self.__by_key = by_key
|
1079
|
+
def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
1080
|
+
pos += 1 # Skip "["
|
1081
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1082
|
+
pos, key = toml_parse_key(src, pos)
|
1175
1083
|
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1084
|
+
if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
|
1085
|
+
raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
|
1086
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
1087
|
+
try:
|
1088
|
+
out.data.get_or_create_nest(key)
|
1089
|
+
except KeyError:
|
1090
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1179
1091
|
|
1180
|
-
|
1181
|
-
|
1182
|
-
|
1092
|
+
if not src.startswith(']', pos):
|
1093
|
+
raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
|
1094
|
+
return pos + 1, key
|
1183
1095
|
|
1184
1096
|
|
1185
|
-
|
1186
|
-
#
|
1097
|
+
def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
1098
|
+
pos += 2 # Skip "[["
|
1099
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1100
|
+
pos, key = toml_parse_key(src, pos)
|
1187
1101
|
|
1102
|
+
if out.flags.is_(key, TomlFlags.FROZEN):
|
1103
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1104
|
+
# Free the namespace now that it points to another empty list item...
|
1105
|
+
out.flags.unset_all(key)
|
1106
|
+
# ...but this key precisely is still prohibited from table declaration
|
1107
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
1108
|
+
try:
|
1109
|
+
out.data.append_nest_to_list(key)
|
1110
|
+
except KeyError:
|
1111
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1188
1112
|
|
1189
|
-
|
1190
|
-
|
1191
|
-
|
1192
|
-
BaseException,
|
1193
|
-
types.TracebackType,
|
1194
|
-
]:
|
1195
|
-
t, v, tb = sys.exc_info()
|
1196
|
-
if not tb:
|
1197
|
-
raise RuntimeError('No traceback')
|
1113
|
+
if not src.startswith(']]', pos):
|
1114
|
+
raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
|
1115
|
+
return pos + 2, key
|
1198
1116
|
|
1199
|
-
tbinfo = []
|
1200
|
-
while tb:
|
1201
|
-
tbinfo.append((
|
1202
|
-
tb.tb_frame.f_code.co_filename,
|
1203
|
-
tb.tb_frame.f_code.co_name,
|
1204
|
-
str(tb.tb_lineno),
|
1205
|
-
))
|
1206
|
-
tb = tb.tb_next
|
1207
1117
|
|
1208
|
-
|
1209
|
-
|
1118
|
+
def toml_key_value_rule(
|
1119
|
+
src: str,
|
1120
|
+
pos: TomlPos,
|
1121
|
+
out: TomlOutput,
|
1122
|
+
header: TomlKey,
|
1123
|
+
parse_float: TomlParseFloat,
|
1124
|
+
) -> TomlPos:
|
1125
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1126
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1127
|
+
abs_key_parent = header + key_parent
|
1210
1128
|
|
1211
|
-
|
1212
|
-
|
1213
|
-
|
1129
|
+
relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
|
1130
|
+
for cont_key in relative_path_cont_keys:
|
1131
|
+
# Check that dotted key syntax does not redefine an existing table
|
1132
|
+
if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
|
1133
|
+
raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
|
1134
|
+
# Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
|
1135
|
+
# table sections.
|
1136
|
+
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
1214
1137
|
|
1138
|
+
if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
|
1139
|
+
raise toml_suffixed_err(
|
1140
|
+
src,
|
1141
|
+
pos,
|
1142
|
+
f'Cannot mutate immutable namespace {abs_key_parent}',
|
1143
|
+
)
|
1215
1144
|
|
1216
|
-
|
1217
|
-
|
1145
|
+
try:
|
1146
|
+
nest = out.data.get_or_create_nest(abs_key_parent)
|
1147
|
+
except KeyError:
|
1148
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1149
|
+
if key_stem in nest:
|
1150
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
1151
|
+
# Mark inline table and array namespaces recursively immutable
|
1152
|
+
if isinstance(value, (dict, list)):
|
1153
|
+
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
1154
|
+
nest[key_stem] = value
|
1155
|
+
return pos
|
1218
1156
|
|
1219
1157
|
|
1220
|
-
def
|
1158
|
+
def toml_parse_key_value_pair(
|
1159
|
+
src: str,
|
1160
|
+
pos: TomlPos,
|
1161
|
+
parse_float: TomlParseFloat,
|
1162
|
+
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
1163
|
+
pos, key = toml_parse_key(src, pos)
|
1221
1164
|
try:
|
1222
|
-
|
1223
|
-
except
|
1224
|
-
|
1225
|
-
|
1165
|
+
char: ta.Optional[str] = src[pos]
|
1166
|
+
except IndexError:
|
1167
|
+
char = None
|
1168
|
+
if char != '=':
|
1169
|
+
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
1170
|
+
pos += 1
|
1171
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1172
|
+
pos, value = toml_parse_value(src, pos, parse_float)
|
1173
|
+
return pos, key, value
|
1226
1174
|
|
1227
1175
|
|
1228
|
-
def
|
1229
|
-
|
1230
|
-
|
1231
|
-
|
1176
|
+
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
1177
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
1178
|
+
key: TomlKey = (key_part,)
|
1179
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1180
|
+
while True:
|
1181
|
+
try:
|
1182
|
+
char: ta.Optional[str] = src[pos]
|
1183
|
+
except IndexError:
|
1184
|
+
char = None
|
1185
|
+
if char != '.':
|
1186
|
+
return pos, key
|
1187
|
+
pos += 1
|
1188
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1189
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
1190
|
+
key += (key_part,)
|
1191
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1232
1192
|
|
1233
1193
|
|
1234
|
-
def
|
1235
|
-
|
1194
|
+
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1195
|
+
try:
|
1196
|
+
char: ta.Optional[str] = src[pos]
|
1197
|
+
except IndexError:
|
1198
|
+
char = None
|
1199
|
+
if char in TOML_BARE_KEY_CHARS:
|
1200
|
+
start_pos = pos
|
1201
|
+
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
1202
|
+
return pos, src[start_pos:pos]
|
1203
|
+
if char == "'":
|
1204
|
+
return toml_parse_literal_str(src, pos)
|
1205
|
+
if char == '"':
|
1206
|
+
return toml_parse_one_line_basic_str(src, pos)
|
1207
|
+
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
1236
1208
|
|
1237
|
-
path = ['/bin', '/usr/bin', '/usr/local/bin']
|
1238
|
-
if 'PATH' in os.environ:
|
1239
|
-
p = os.environ['PATH']
|
1240
|
-
if p:
|
1241
|
-
path = p.split(os.pathsep)
|
1242
|
-
return path
|
1243
1209
|
|
1210
|
+
def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1211
|
+
pos += 1
|
1212
|
+
return toml_parse_basic_str(src, pos, multiline=False)
|
1244
1213
|
|
1245
|
-
def check_existing_dir(v: str) -> str:
|
1246
|
-
nv = os.path.expanduser(v)
|
1247
|
-
if os.path.isdir(nv):
|
1248
|
-
return nv
|
1249
|
-
raise ValueError(f'{v} is not an existing directory')
|
1250
1214
|
|
1215
|
+
def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
|
1216
|
+
pos += 1
|
1217
|
+
array: list = []
|
1251
1218
|
|
1252
|
-
|
1253
|
-
|
1254
|
-
|
1255
|
-
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1219
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1220
|
+
if src.startswith(']', pos):
|
1221
|
+
return pos + 1, array
|
1222
|
+
while True:
|
1223
|
+
pos, val = toml_parse_value(src, pos, parse_float)
|
1224
|
+
array.append(val)
|
1225
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1226
|
+
|
1227
|
+
c = src[pos:pos + 1]
|
1228
|
+
if c == ']':
|
1229
|
+
return pos + 1, array
|
1230
|
+
if c != ',':
|
1231
|
+
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
1232
|
+
pos += 1
|
1261
1233
|
|
1234
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1235
|
+
if src.startswith(']', pos):
|
1236
|
+
return pos + 1, array
|
1262
1237
|
|
1263
|
-
########################################
|
1264
|
-
# ../utils/ostypes.py
|
1265
1238
|
|
1239
|
+
def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
|
1240
|
+
pos += 1
|
1241
|
+
nested_dict = TomlNestedDict()
|
1242
|
+
flags = TomlFlags()
|
1266
1243
|
|
1267
|
-
|
1268
|
-
|
1269
|
-
|
1244
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1245
|
+
if src.startswith('}', pos):
|
1246
|
+
return pos + 1, nested_dict.dict
|
1247
|
+
while True:
|
1248
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1249
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1250
|
+
if flags.is_(key, TomlFlags.FROZEN):
|
1251
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1252
|
+
try:
|
1253
|
+
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
1254
|
+
except KeyError:
|
1255
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1256
|
+
if key_stem in nest:
|
1257
|
+
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
1258
|
+
nest[key_stem] = value
|
1259
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1260
|
+
c = src[pos:pos + 1]
|
1261
|
+
if c == '}':
|
1262
|
+
return pos + 1, nested_dict.dict
|
1263
|
+
if c != ',':
|
1264
|
+
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
1265
|
+
if isinstance(value, (dict, list)):
|
1266
|
+
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
1267
|
+
pos += 1
|
1268
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1270
1269
|
|
1271
|
-
Uid = ta.NewType('Uid', int)
|
1272
|
-
Gid = ta.NewType('Gid', int)
|
1273
1270
|
|
1271
|
+
def toml_parse_basic_str_escape(
|
1272
|
+
src: str,
|
1273
|
+
pos: TomlPos,
|
1274
|
+
*,
|
1275
|
+
multiline: bool = False,
|
1276
|
+
) -> ta.Tuple[TomlPos, str]:
|
1277
|
+
escape_id = src[pos:pos + 2]
|
1278
|
+
pos += 2
|
1279
|
+
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
1280
|
+
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
1281
|
+
# newline.
|
1282
|
+
if escape_id != '\\\n':
|
1283
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1284
|
+
try:
|
1285
|
+
char = src[pos]
|
1286
|
+
except IndexError:
|
1287
|
+
return pos, ''
|
1288
|
+
if char != '\n':
|
1289
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
1290
|
+
pos += 1
|
1291
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1292
|
+
return pos, ''
|
1293
|
+
if escape_id == '\\u':
|
1294
|
+
return toml_parse_hex_char(src, pos, 4)
|
1295
|
+
if escape_id == '\\U':
|
1296
|
+
return toml_parse_hex_char(src, pos, 8)
|
1297
|
+
try:
|
1298
|
+
return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
|
1299
|
+
except KeyError:
|
1300
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
1274
1301
|
|
1275
|
-
########################################
|
1276
|
-
# ../utils/signals.py
|
1277
1302
|
|
1303
|
+
def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1304
|
+
return toml_parse_basic_str_escape(src, pos, multiline=True)
|
1278
1305
|
|
1279
|
-
##
|
1280
1306
|
|
1307
|
+
def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
|
1308
|
+
hex_str = src[pos:pos + hex_len]
|
1309
|
+
if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
|
1310
|
+
raise toml_suffixed_err(src, pos, 'Invalid hex value')
|
1311
|
+
pos += hex_len
|
1312
|
+
hex_int = int(hex_str, 16)
|
1313
|
+
if not toml_is_unicode_scalar_value(hex_int):
|
1314
|
+
raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
|
1315
|
+
return pos, chr(hex_int)
|
1281
1316
|
|
1282
|
-
_SIGS_BY_NUM: ta.Mapping[int, signal.Signals] = {s.value: s for s in signal.Signals}
|
1283
|
-
_SIGS_BY_NAME: ta.Mapping[str, signal.Signals] = {s.name: s for s in signal.Signals}
|
1284
1317
|
|
1318
|
+
def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1319
|
+
pos += 1 # Skip starting apostrophe
|
1320
|
+
start_pos = pos
|
1321
|
+
pos = toml_skip_until(
|
1322
|
+
src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
|
1323
|
+
)
|
1324
|
+
return pos + 1, src[start_pos:pos] # Skip ending apostrophe
|
1285
1325
|
|
1286
|
-
def sig_num(value: ta.Union[int, str]) -> int:
|
1287
|
-
try:
|
1288
|
-
num = int(value)
|
1289
1326
|
|
1290
|
-
|
1291
|
-
|
1292
|
-
|
1293
|
-
|
1327
|
+
def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
|
1328
|
+
pos += 3
|
1329
|
+
if src.startswith('\n', pos):
|
1330
|
+
pos += 1
|
1294
1331
|
|
1295
|
-
|
1296
|
-
|
1297
|
-
|
1332
|
+
if literal:
|
1333
|
+
delim = "'"
|
1334
|
+
end_pos = toml_skip_until(
|
1335
|
+
src,
|
1336
|
+
pos,
|
1337
|
+
"'''",
|
1338
|
+
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
1339
|
+
error_on_eof=True,
|
1340
|
+
)
|
1341
|
+
result = src[pos:end_pos]
|
1342
|
+
pos = end_pos + 3
|
1343
|
+
else:
|
1344
|
+
delim = '"'
|
1345
|
+
pos, result = toml_parse_basic_str(src, pos, multiline=True)
|
1298
1346
|
|
1299
|
-
if
|
1300
|
-
|
1347
|
+
# Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
|
1348
|
+
if not src.startswith(delim, pos):
|
1349
|
+
return pos, result
|
1350
|
+
pos += 1
|
1351
|
+
if not src.startswith(delim, pos):
|
1352
|
+
return pos, result + delim
|
1353
|
+
pos += 1
|
1354
|
+
return pos, result + (delim * 2)
|
1301
1355
|
|
1302
|
-
return num
|
1303
1356
|
|
1357
|
+
def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
|
1358
|
+
if multiline:
|
1359
|
+
error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1360
|
+
parse_escapes = toml_parse_basic_str_escape_multiline
|
1361
|
+
else:
|
1362
|
+
error_on = TOML_ILLEGAL_BASIC_STR_CHARS
|
1363
|
+
parse_escapes = toml_parse_basic_str_escape
|
1364
|
+
result = ''
|
1365
|
+
start_pos = pos
|
1366
|
+
while True:
|
1367
|
+
try:
|
1368
|
+
char = src[pos]
|
1369
|
+
except IndexError:
|
1370
|
+
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
1371
|
+
if char == '"':
|
1372
|
+
if not multiline:
|
1373
|
+
return pos + 1, result + src[start_pos:pos]
|
1374
|
+
if src.startswith('"""', pos):
|
1375
|
+
return pos + 3, result + src[start_pos:pos]
|
1376
|
+
pos += 1
|
1377
|
+
continue
|
1378
|
+
if char == '\\':
|
1379
|
+
result += src[start_pos:pos]
|
1380
|
+
pos, parsed_escape = parse_escapes(src, pos)
|
1381
|
+
result += parsed_escape
|
1382
|
+
start_pos = pos
|
1383
|
+
continue
|
1384
|
+
if char in error_on:
|
1385
|
+
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
1386
|
+
pos += 1
|
1304
1387
|
|
1305
|
-
def sig_name(num: int) -> str:
|
1306
|
-
if (sig := _SIGS_BY_NUM.get(num)) is not None:
|
1307
|
-
return sig.name
|
1308
|
-
return f'signal {sig}'
|
1309
1388
|
|
1389
|
+
def toml_parse_value( # noqa: C901
|
1390
|
+
src: str,
|
1391
|
+
pos: TomlPos,
|
1392
|
+
parse_float: TomlParseFloat,
|
1393
|
+
) -> ta.Tuple[TomlPos, ta.Any]:
|
1394
|
+
try:
|
1395
|
+
char: ta.Optional[str] = src[pos]
|
1396
|
+
except IndexError:
|
1397
|
+
char = None
|
1310
1398
|
|
1311
|
-
|
1399
|
+
# IMPORTANT: order conditions based on speed of checking and likelihood
|
1312
1400
|
|
1401
|
+
# Basic strings
|
1402
|
+
if char == '"':
|
1403
|
+
if src.startswith('"""', pos):
|
1404
|
+
return toml_parse_multiline_str(src, pos, literal=False)
|
1405
|
+
return toml_parse_one_line_basic_str(src, pos)
|
1313
1406
|
|
1314
|
-
|
1315
|
-
|
1316
|
-
|
1407
|
+
# Literal strings
|
1408
|
+
if char == "'":
|
1409
|
+
if src.startswith("'''", pos):
|
1410
|
+
return toml_parse_multiline_str(src, pos, literal=True)
|
1411
|
+
return toml_parse_literal_str(src, pos)
|
1317
1412
|
|
1318
|
-
|
1413
|
+
# Booleans
|
1414
|
+
if char == 't':
|
1415
|
+
if src.startswith('true', pos):
|
1416
|
+
return pos + 4, True
|
1417
|
+
if char == 'f':
|
1418
|
+
if src.startswith('false', pos):
|
1419
|
+
return pos + 5, False
|
1319
1420
|
|
1320
|
-
|
1321
|
-
|
1322
|
-
|
1421
|
+
# Arrays
|
1422
|
+
if char == '[':
|
1423
|
+
return toml_parse_array(src, pos, parse_float)
|
1323
1424
|
|
1324
|
-
|
1325
|
-
|
1326
|
-
|
1425
|
+
# Inline tables
|
1426
|
+
if char == '{':
|
1427
|
+
return toml_parse_inline_table(src, pos, parse_float)
|
1327
1428
|
|
1328
|
-
|
1329
|
-
|
1330
|
-
|
1331
|
-
|
1332
|
-
|
1333
|
-
|
1429
|
+
# Dates and times
|
1430
|
+
datetime_match = TOML_RE_DATETIME.match(src, pos)
|
1431
|
+
if datetime_match:
|
1432
|
+
try:
|
1433
|
+
datetime_obj = toml_match_to_datetime(datetime_match)
|
1434
|
+
except ValueError as e:
|
1435
|
+
raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
|
1436
|
+
return datetime_match.end(), datetime_obj
|
1437
|
+
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
1438
|
+
if localtime_match:
|
1439
|
+
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
1334
1440
|
|
1441
|
+
# Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
|
1442
|
+
# located after handling of dates and times.
|
1443
|
+
number_match = TOML_RE_NUMBER.match(src, pos)
|
1444
|
+
if number_match:
|
1445
|
+
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
1335
1446
|
|
1336
|
-
|
1337
|
-
|
1447
|
+
# Special floats
|
1448
|
+
first_three = src[pos:pos + 3]
|
1449
|
+
if first_three in {'inf', 'nan'}:
|
1450
|
+
return pos + 3, parse_float(first_three)
|
1451
|
+
first_four = src[pos:pos + 4]
|
1452
|
+
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
1453
|
+
return pos + 4, parse_float(first_four)
|
1338
1454
|
|
1455
|
+
raise toml_suffixed_err(src, pos, 'Invalid value')
|
1339
1456
|
|
1340
|
-
##
|
1341
1457
|
|
1458
|
+
def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
|
1459
|
+
"""Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
|
1342
1460
|
|
1343
|
-
def
|
1344
|
-
|
1345
|
-
|
1346
|
-
|
1347
|
-
|
1461
|
+
def coord_repr(src: str, pos: TomlPos) -> str:
|
1462
|
+
if pos >= len(src):
|
1463
|
+
return 'end of document'
|
1464
|
+
line = src.count('\n', 0, pos) + 1
|
1465
|
+
if line == 1:
|
1466
|
+
column = pos + 1
|
1467
|
+
else:
|
1468
|
+
column = pos - src.rindex('\n', 0, pos)
|
1469
|
+
return f'line {line}, column {column}'
|
1348
1470
|
|
1471
|
+
return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
|
1349
1472
|
|
1350
|
-
@ta.overload
|
1351
|
-
def find_prefix_at_end(haystack: str, needle: str) -> int:
|
1352
|
-
...
|
1353
1473
|
|
1474
|
+
def toml_is_unicode_scalar_value(codepoint: int) -> bool:
|
1475
|
+
return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
|
1354
1476
|
|
1355
|
-
@ta.overload
|
1356
|
-
def find_prefix_at_end(haystack: bytes, needle: bytes) -> int:
|
1357
|
-
...
|
1358
1477
|
|
1478
|
+
def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
|
1479
|
+
"""A decorator to make `parse_float` safe.
|
1359
1480
|
|
1360
|
-
|
1361
|
-
|
1362
|
-
|
1363
|
-
|
1364
|
-
|
1481
|
+
`parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
|
1482
|
+
thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
|
1483
|
+
"""
|
1484
|
+
# The default `float` callable never returns illegal types. Optimize it.
|
1485
|
+
if parse_float is float:
|
1486
|
+
return float
|
1365
1487
|
|
1488
|
+
def safe_parse_float(float_str: str) -> ta.Any:
|
1489
|
+
float_value = parse_float(float_str)
|
1490
|
+
if isinstance(float_value, (dict, list)):
|
1491
|
+
raise ValueError('parse_float must not return dicts or lists') # noqa
|
1492
|
+
return float_value
|
1366
1493
|
|
1367
|
-
|
1494
|
+
return safe_parse_float
|
1368
1495
|
|
1369
1496
|
|
1370
|
-
|
1371
|
-
|
1497
|
+
########################################
|
1498
|
+
# ../../../omlish/formats/toml/writer.py
|
1372
1499
|
|
1373
1500
|
|
1374
|
-
|
1375
|
-
|
1501
|
+
class TomlWriter:
|
1502
|
+
@dc.dataclass(frozen=True)
|
1503
|
+
class Literal:
|
1504
|
+
s: str
|
1376
1505
|
|
1377
|
-
|
1378
|
-
|
1379
|
-
|
1380
|
-
l = len(s)
|
1381
|
-
while i < l:
|
1382
|
-
if show == 0 and s[i:i + 1] in ANSI_TERMINATORS:
|
1383
|
-
show = 1
|
1384
|
-
elif show:
|
1385
|
-
n = s.find(ANSI_ESCAPE_BEGIN, i)
|
1386
|
-
if n == -1:
|
1387
|
-
return result + s[i:]
|
1388
|
-
else:
|
1389
|
-
result = result + s[i:n]
|
1390
|
-
i = n
|
1391
|
-
show = 0
|
1392
|
-
i += 1
|
1393
|
-
return result
|
1506
|
+
def __init__(self, out: ta.TextIO) -> None:
|
1507
|
+
super().__init__()
|
1508
|
+
self._out = out
|
1394
1509
|
|
1510
|
+
self._indent = 0
|
1511
|
+
self._wrote_indent = False
|
1395
1512
|
|
1396
|
-
|
1513
|
+
#
|
1397
1514
|
|
1515
|
+
def _w(self, s: str) -> None:
|
1516
|
+
if not self._wrote_indent:
|
1517
|
+
self._out.write(' ' * self._indent)
|
1518
|
+
self._wrote_indent = True
|
1519
|
+
self._out.write(s)
|
1398
1520
|
|
1399
|
-
|
1400
|
-
|
1401
|
-
|
1402
|
-
def __init__(self, d, default=1):
|
1403
|
-
super().__init__()
|
1404
|
-
self._d = d
|
1405
|
-
self._default = default
|
1406
|
-
# all keys must be the same size
|
1407
|
-
self._keysz = None
|
1408
|
-
for k in d:
|
1409
|
-
if self._keysz is None:
|
1410
|
-
self._keysz = len(k)
|
1411
|
-
elif self._keysz != len(k): # type: ignore
|
1412
|
-
raise ValueError(k)
|
1521
|
+
def _nl(self) -> None:
|
1522
|
+
self._out.write('\n')
|
1523
|
+
self._wrote_indent = False
|
1413
1524
|
|
1414
|
-
def
|
1415
|
-
|
1416
|
-
|
1417
|
-
|
1418
|
-
|
1419
|
-
|
1420
|
-
return int(v[:-self._keysz]) * m # type: ignore
|
1421
|
-
return int(v) * self._default
|
1525
|
+
def _needs_quote(self, s: str) -> bool:
|
1526
|
+
return (
|
1527
|
+
not s or
|
1528
|
+
any(c in s for c in '\'"\n') or
|
1529
|
+
s[0] not in string.ascii_letters
|
1530
|
+
)
|
1422
1531
|
|
1532
|
+
def _maybe_quote(self, s: str) -> str:
|
1533
|
+
if self._needs_quote(s):
|
1534
|
+
return repr(s)
|
1535
|
+
else:
|
1536
|
+
return s
|
1423
1537
|
|
1424
|
-
|
1425
|
-
'kb': 1024,
|
1426
|
-
'mb': 1024 * 1024,
|
1427
|
-
'gb': 1024 * 1024 * 1024,
|
1428
|
-
})
|
1538
|
+
#
|
1429
1539
|
|
1540
|
+
def write_root(self, obj: ta.Mapping) -> None:
|
1541
|
+
for i, (k, v) in enumerate(obj.items()):
|
1542
|
+
if i:
|
1543
|
+
self._nl()
|
1544
|
+
self._w('[')
|
1545
|
+
self._w(self._maybe_quote(k))
|
1546
|
+
self._w(']')
|
1547
|
+
self._nl()
|
1548
|
+
self.write_table_contents(v)
|
1549
|
+
|
1550
|
+
def write_table_contents(self, obj: ta.Mapping) -> None:
|
1551
|
+
for k, v in obj.items():
|
1552
|
+
self.write_key(k)
|
1553
|
+
self._w(' = ')
|
1554
|
+
self.write_value(v)
|
1555
|
+
self._nl()
|
1556
|
+
|
1557
|
+
def write_array(self, obj: ta.Sequence) -> None:
|
1558
|
+
self._w('[')
|
1559
|
+
self._nl()
|
1560
|
+
self._indent += 1
|
1561
|
+
for e in obj:
|
1562
|
+
self.write_value(e)
|
1563
|
+
self._w(',')
|
1564
|
+
self._nl()
|
1565
|
+
self._indent -= 1
|
1566
|
+
self._w(']')
|
1567
|
+
|
1568
|
+
def write_inline_table(self, obj: ta.Mapping) -> None:
|
1569
|
+
self._w('{')
|
1570
|
+
for i, (k, v) in enumerate(obj.items()):
|
1571
|
+
if i:
|
1572
|
+
self._w(', ')
|
1573
|
+
self.write_key(k)
|
1574
|
+
self._w(' = ')
|
1575
|
+
self.write_value(v)
|
1576
|
+
self._w('}')
|
1577
|
+
|
1578
|
+
def write_inline_array(self, obj: ta.Sequence) -> None:
|
1579
|
+
self._w('[')
|
1580
|
+
for i, e in enumerate(obj):
|
1581
|
+
if i:
|
1582
|
+
self._w(', ')
|
1583
|
+
self.write_value(e)
|
1584
|
+
self._w(']')
|
1585
|
+
|
1586
|
+
def write_key(self, obj: ta.Any) -> None:
|
1587
|
+
if isinstance(obj, TomlWriter.Literal):
|
1588
|
+
self._w(obj.s)
|
1589
|
+
elif isinstance(obj, str):
|
1590
|
+
self._w(self._maybe_quote(obj.replace('_', '-')))
|
1591
|
+
elif isinstance(obj, int):
|
1592
|
+
self._w(repr(str(obj)))
|
1593
|
+
else:
|
1594
|
+
raise TypeError(obj)
|
1430
1595
|
|
1431
|
-
|
1596
|
+
def write_value(self, obj: ta.Any) -> None:
|
1597
|
+
if isinstance(obj, bool):
|
1598
|
+
self._w(str(obj).lower())
|
1599
|
+
elif isinstance(obj, (str, int, float)):
|
1600
|
+
self._w(repr(obj))
|
1601
|
+
elif isinstance(obj, ta.Mapping):
|
1602
|
+
self.write_inline_table(obj)
|
1603
|
+
elif isinstance(obj, ta.Sequence):
|
1604
|
+
if not obj:
|
1605
|
+
self.write_inline_array(obj)
|
1606
|
+
else:
|
1607
|
+
self.write_array(obj)
|
1608
|
+
else:
|
1609
|
+
raise TypeError(obj)
|
1432
1610
|
|
1611
|
+
#
|
1433
1612
|
|
1434
|
-
|
1435
|
-
|
1436
|
-
|
1437
|
-
|
1438
|
-
return
|
1439
|
-
except (TypeError, ValueError):
|
1440
|
-
raise ValueError(f'{arg} can not be converted to an octal type') # noqa
|
1613
|
+
@classmethod
|
1614
|
+
def write_str(cls, obj: ta.Any) -> str:
|
1615
|
+
out = io.StringIO()
|
1616
|
+
cls(out).write_value(obj)
|
1617
|
+
return out.getvalue()
|
1441
1618
|
|
1442
1619
|
|
1443
1620
|
########################################
|
@@ -3069,61 +3246,338 @@ def waitpid(
|
|
3069
3246
|
##
|
3070
3247
|
|
3071
3248
|
|
3072
|
-
def name_to_uid(name: str) -> Uid:
|
3073
|
-
try:
|
3074
|
-
uid = int(name)
|
3075
|
-
except ValueError:
|
3076
|
-
try:
|
3077
|
-
pwdrec = pwd.getpwnam(name)
|
3078
|
-
except KeyError:
|
3079
|
-
raise ValueError(f'Invalid user name {name}') # noqa
|
3080
|
-
uid = pwdrec[2]
|
3081
|
-
else:
|
3082
|
-
try:
|
3083
|
-
pwd.getpwuid(uid) # check if uid is valid
|
3084
|
-
except KeyError:
|
3085
|
-
raise ValueError(f'Invalid user id {name}') # noqa
|
3086
|
-
return Uid(uid)
|
3249
|
+
def name_to_uid(name: str) -> Uid:
|
3250
|
+
try:
|
3251
|
+
uid = int(name)
|
3252
|
+
except ValueError:
|
3253
|
+
try:
|
3254
|
+
pwdrec = pwd.getpwnam(name)
|
3255
|
+
except KeyError:
|
3256
|
+
raise ValueError(f'Invalid user name {name}') # noqa
|
3257
|
+
uid = pwdrec[2]
|
3258
|
+
else:
|
3259
|
+
try:
|
3260
|
+
pwd.getpwuid(uid) # check if uid is valid
|
3261
|
+
except KeyError:
|
3262
|
+
raise ValueError(f'Invalid user id {name}') # noqa
|
3263
|
+
return Uid(uid)
|
3264
|
+
|
3265
|
+
|
3266
|
+
def name_to_gid(name: str) -> Gid:
|
3267
|
+
try:
|
3268
|
+
gid = int(name)
|
3269
|
+
except ValueError:
|
3270
|
+
try:
|
3271
|
+
grprec = grp.getgrnam(name)
|
3272
|
+
except KeyError:
|
3273
|
+
raise ValueError(f'Invalid group name {name}') # noqa
|
3274
|
+
gid = grprec[2]
|
3275
|
+
else:
|
3276
|
+
try:
|
3277
|
+
grp.getgrgid(gid) # check if gid is valid
|
3278
|
+
except KeyError:
|
3279
|
+
raise ValueError(f'Invalid group id {name}') # noqa
|
3280
|
+
return Gid(gid)
|
3281
|
+
|
3282
|
+
|
3283
|
+
def gid_for_uid(uid: Uid) -> Gid:
|
3284
|
+
pwrec = pwd.getpwuid(uid)
|
3285
|
+
return Gid(pwrec[3])
|
3286
|
+
|
3287
|
+
|
3288
|
+
##
|
3289
|
+
|
3290
|
+
|
3291
|
+
@dc.dataclass(frozen=True)
|
3292
|
+
class User:
|
3293
|
+
name: str
|
3294
|
+
uid: Uid
|
3295
|
+
gid: Gid
|
3296
|
+
|
3297
|
+
|
3298
|
+
def get_user(name: str) -> User:
|
3299
|
+
return User(
|
3300
|
+
name=name,
|
3301
|
+
uid=(uid := name_to_uid(name)),
|
3302
|
+
gid=gid_for_uid(uid),
|
3303
|
+
)
|
3304
|
+
|
3305
|
+
|
3306
|
+
########################################
|
3307
|
+
# ../../../omlish/configs/formats.py
|
3308
|
+
"""
|
3309
|
+
Notes:
|
3310
|
+
- necessarily string-oriented
|
3311
|
+
- single file, as this is intended to be amalg'd and thus all included anyway
|
3312
|
+
|
3313
|
+
TODO:
|
3314
|
+
- ConfigDataMapper? to_map -> ConfigMap?
|
3315
|
+
- nginx ?
|
3316
|
+
- raw ?
|
3317
|
+
"""
|
3318
|
+
|
3319
|
+
|
3320
|
+
##
|
3321
|
+
|
3322
|
+
|
3323
|
+
@dc.dataclass(frozen=True)
|
3324
|
+
class ConfigData(abc.ABC): # noqa
|
3325
|
+
@abc.abstractmethod
|
3326
|
+
def as_map(self) -> ConfigMap:
|
3327
|
+
raise NotImplementedError
|
3328
|
+
|
3329
|
+
|
3330
|
+
#
|
3331
|
+
|
3332
|
+
|
3333
|
+
class ConfigLoader(abc.ABC, ta.Generic[ConfigDataT]):
|
3334
|
+
@property
|
3335
|
+
def file_exts(self) -> ta.Sequence[str]:
|
3336
|
+
return ()
|
3337
|
+
|
3338
|
+
def match_file(self, n: str) -> bool:
|
3339
|
+
return '.' in n and n.split('.')[-1] in check.not_isinstance(self.file_exts, str)
|
3340
|
+
|
3341
|
+
#
|
3342
|
+
|
3343
|
+
def load_file(self, p: str) -> ConfigDataT:
|
3344
|
+
with open(p) as f:
|
3345
|
+
return self.load_str(f.read())
|
3346
|
+
|
3347
|
+
@abc.abstractmethod
|
3348
|
+
def load_str(self, s: str) -> ConfigDataT:
|
3349
|
+
raise NotImplementedError
|
3350
|
+
|
3351
|
+
|
3352
|
+
#
|
3353
|
+
|
3354
|
+
|
3355
|
+
class ConfigRenderer(abc.ABC, ta.Generic[ConfigDataT]):
|
3356
|
+
@property
|
3357
|
+
@abc.abstractmethod
|
3358
|
+
def data_cls(self) -> ta.Type[ConfigDataT]:
|
3359
|
+
raise NotImplementedError
|
3360
|
+
|
3361
|
+
def match_data(self, d: ConfigDataT) -> bool:
|
3362
|
+
return isinstance(d, self.data_cls)
|
3363
|
+
|
3364
|
+
#
|
3365
|
+
|
3366
|
+
@abc.abstractmethod
|
3367
|
+
def render(self, d: ConfigDataT) -> str:
|
3368
|
+
raise NotImplementedError
|
3369
|
+
|
3370
|
+
|
3371
|
+
##
|
3372
|
+
|
3373
|
+
|
3374
|
+
@dc.dataclass(frozen=True)
|
3375
|
+
class ObjConfigData(ConfigData, abc.ABC):
|
3376
|
+
obj: ta.Any
|
3377
|
+
|
3378
|
+
def as_map(self) -> ConfigMap:
|
3379
|
+
return check.isinstance(self.obj, collections.abc.Mapping)
|
3380
|
+
|
3381
|
+
|
3382
|
+
##
|
3383
|
+
|
3384
|
+
|
3385
|
+
@dc.dataclass(frozen=True)
|
3386
|
+
class JsonConfigData(ObjConfigData):
|
3387
|
+
pass
|
3388
|
+
|
3389
|
+
|
3390
|
+
class JsonConfigLoader(ConfigLoader[JsonConfigData]):
|
3391
|
+
file_exts = ('json',)
|
3392
|
+
|
3393
|
+
def load_str(self, s: str) -> JsonConfigData:
|
3394
|
+
return JsonConfigData(json.loads(s))
|
3395
|
+
|
3396
|
+
|
3397
|
+
class JsonConfigRenderer(ConfigRenderer[JsonConfigData]):
|
3398
|
+
data_cls = JsonConfigData
|
3399
|
+
|
3400
|
+
def render(self, d: JsonConfigData) -> str:
|
3401
|
+
return json_dumps_pretty(d.obj)
|
3402
|
+
|
3403
|
+
|
3404
|
+
##
|
3405
|
+
|
3406
|
+
|
3407
|
+
@dc.dataclass(frozen=True)
|
3408
|
+
class TomlConfigData(ObjConfigData):
|
3409
|
+
pass
|
3410
|
+
|
3411
|
+
|
3412
|
+
class TomlConfigLoader(ConfigLoader[TomlConfigData]):
|
3413
|
+
file_exts = ('toml',)
|
3414
|
+
|
3415
|
+
def load_str(self, s: str) -> TomlConfigData:
|
3416
|
+
return TomlConfigData(toml_loads(s))
|
3417
|
+
|
3418
|
+
|
3419
|
+
class TomlConfigRenderer(ConfigRenderer[TomlConfigData]):
|
3420
|
+
data_cls = TomlConfigData
|
3421
|
+
|
3422
|
+
def render(self, d: TomlConfigData) -> str:
|
3423
|
+
return TomlWriter.write_str(d.obj)
|
3424
|
+
|
3425
|
+
|
3426
|
+
##
|
3427
|
+
|
3428
|
+
|
3429
|
+
@dc.dataclass(frozen=True)
|
3430
|
+
class YamlConfigData(ObjConfigData):
|
3431
|
+
pass
|
3432
|
+
|
3433
|
+
|
3434
|
+
class YamlConfigLoader(ConfigLoader[YamlConfigData]):
|
3435
|
+
file_exts = ('yaml', 'yml')
|
3436
|
+
|
3437
|
+
def load_str(self, s: str) -> YamlConfigData:
|
3438
|
+
return YamlConfigData(__import__('yaml').safe_load(s))
|
3439
|
+
|
3440
|
+
|
3441
|
+
class YamlConfigRenderer(ConfigRenderer[YamlConfigData]):
|
3442
|
+
data_cls = YamlConfigData
|
3443
|
+
|
3444
|
+
def render(self, d: YamlConfigData) -> str:
|
3445
|
+
return __import__('yaml').safe_dump(d.obj)
|
3446
|
+
|
3447
|
+
|
3448
|
+
##
|
3449
|
+
|
3450
|
+
|
3451
|
+
@dc.dataclass(frozen=True)
|
3452
|
+
class IniConfigData(ConfigData):
|
3453
|
+
sections: IniSectionSettingsMap
|
3454
|
+
|
3455
|
+
def as_map(self) -> ConfigMap:
|
3456
|
+
return self.sections
|
3457
|
+
|
3458
|
+
|
3459
|
+
class IniConfigLoader(ConfigLoader[IniConfigData]):
|
3460
|
+
file_exts = ('ini',)
|
3461
|
+
|
3462
|
+
def load_str(self, s: str) -> IniConfigData:
|
3463
|
+
cp = configparser.ConfigParser()
|
3464
|
+
cp.read_string(s)
|
3465
|
+
return IniConfigData(extract_ini_sections(cp))
|
3466
|
+
|
3467
|
+
|
3468
|
+
class IniConfigRenderer(ConfigRenderer[IniConfigData]):
|
3469
|
+
data_cls = IniConfigData
|
3470
|
+
|
3471
|
+
def render(self, d: IniConfigData) -> str:
|
3472
|
+
return render_ini_sections(d.sections)
|
3473
|
+
|
3474
|
+
|
3475
|
+
##
|
3476
|
+
|
3477
|
+
|
3478
|
+
@dc.dataclass(frozen=True)
|
3479
|
+
class SwitchedConfigFileLoader:
|
3480
|
+
loaders: ta.Sequence[ConfigLoader]
|
3481
|
+
default: ta.Optional[ConfigLoader] = None
|
3482
|
+
|
3483
|
+
def load_file(self, p: str) -> ConfigData:
|
3484
|
+
n = os.path.basename(p)
|
3485
|
+
|
3486
|
+
for l in self.loaders:
|
3487
|
+
if l.match_file(n):
|
3488
|
+
return l.load_file(p)
|
3489
|
+
|
3490
|
+
if (d := self.default) is not None:
|
3491
|
+
return d.load_file(p)
|
3492
|
+
|
3493
|
+
raise NameError(n)
|
3494
|
+
|
3495
|
+
|
3496
|
+
DEFAULT_CONFIG_LOADERS: ta.Sequence[ConfigLoader] = [
|
3497
|
+
JsonConfigLoader(),
|
3498
|
+
TomlConfigLoader(),
|
3499
|
+
YamlConfigLoader(),
|
3500
|
+
IniConfigLoader(),
|
3501
|
+
]
|
3502
|
+
|
3503
|
+
DEFAULT_CONFIG_LOADER: ConfigLoader = JsonConfigLoader()
|
3504
|
+
|
3505
|
+
DEFAULT_CONFIG_FILE_LOADER = SwitchedConfigFileLoader(
|
3506
|
+
loaders=DEFAULT_CONFIG_LOADERS,
|
3507
|
+
default=DEFAULT_CONFIG_LOADER,
|
3508
|
+
)
|
3509
|
+
|
3510
|
+
|
3511
|
+
##
|
3512
|
+
|
3513
|
+
|
3514
|
+
@dc.dataclass(frozen=True)
|
3515
|
+
class SwitchedConfigRenderer:
|
3516
|
+
renderers: ta.Sequence[ConfigRenderer]
|
3517
|
+
|
3518
|
+
def render(self, d: ConfigData) -> str:
|
3519
|
+
for r in self.renderers:
|
3520
|
+
if r.match_data(d):
|
3521
|
+
return r.render(d)
|
3522
|
+
raise TypeError(d)
|
3523
|
+
|
3087
3524
|
|
3525
|
+
DEFAULT_CONFIG_RENDERERS: ta.Sequence[ConfigRenderer] = [
|
3526
|
+
JsonConfigRenderer(),
|
3527
|
+
TomlConfigRenderer(),
|
3528
|
+
YamlConfigRenderer(),
|
3529
|
+
IniConfigRenderer(),
|
3530
|
+
]
|
3088
3531
|
|
3089
|
-
|
3090
|
-
try:
|
3091
|
-
gid = int(name)
|
3092
|
-
except ValueError:
|
3093
|
-
try:
|
3094
|
-
grprec = grp.getgrnam(name)
|
3095
|
-
except KeyError:
|
3096
|
-
raise ValueError(f'Invalid group name {name}') # noqa
|
3097
|
-
gid = grprec[2]
|
3098
|
-
else:
|
3099
|
-
try:
|
3100
|
-
grp.getgrgid(gid) # check if gid is valid
|
3101
|
-
except KeyError:
|
3102
|
-
raise ValueError(f'Invalid group id {name}') # noqa
|
3103
|
-
return Gid(gid)
|
3532
|
+
DEFAULT_CONFIG_RENDERER = SwitchedConfigRenderer(DEFAULT_CONFIG_RENDERERS)
|
3104
3533
|
|
3105
3534
|
|
3106
|
-
|
3107
|
-
|
3108
|
-
|
3535
|
+
########################################
|
3536
|
+
# ../../../omlish/configs/processing/names.py
|
3537
|
+
"""
|
3538
|
+
usecase: supervisor process groups
|
3539
|
+
"""
|
3109
3540
|
|
3110
3541
|
|
3111
3542
|
##
|
3112
3543
|
|
3113
3544
|
|
3114
|
-
|
3115
|
-
|
3116
|
-
|
3117
|
-
|
3118
|
-
|
3545
|
+
def build_config_named_children(
|
3546
|
+
o: ta.Union[
|
3547
|
+
ta.Sequence[ConfigMap],
|
3548
|
+
ta.Mapping[str, ConfigMap],
|
3549
|
+
None,
|
3550
|
+
],
|
3551
|
+
*,
|
3552
|
+
name_key: str = 'name',
|
3553
|
+
) -> ta.Optional[ta.Sequence[ConfigMap]]:
|
3554
|
+
if o is None:
|
3555
|
+
return None
|
3119
3556
|
|
3557
|
+
lst: ta.List[ConfigMap] = []
|
3558
|
+
if isinstance(o, ta.Mapping):
|
3559
|
+
for k, v in o.items():
|
3560
|
+
check.isinstance(v, ta.Mapping)
|
3561
|
+
if name_key in v:
|
3562
|
+
n = v[name_key]
|
3563
|
+
if k != n:
|
3564
|
+
raise KeyError(f'Given names do not match: {n} != {k}')
|
3565
|
+
lst.append(v)
|
3566
|
+
else:
|
3567
|
+
lst.append({name_key: k, **v})
|
3120
3568
|
|
3121
|
-
|
3122
|
-
|
3123
|
-
|
3124
|
-
|
3125
|
-
|
3126
|
-
|
3569
|
+
else:
|
3570
|
+
check.not_isinstance(o, str)
|
3571
|
+
lst.extend(o)
|
3572
|
+
|
3573
|
+
seen = set()
|
3574
|
+
for d in lst:
|
3575
|
+
n = d['name']
|
3576
|
+
if n in d:
|
3577
|
+
raise KeyError(f'Duplicate name: {n}')
|
3578
|
+
seen.add(n)
|
3579
|
+
|
3580
|
+
return lst
|
3127
3581
|
|
3128
3582
|
|
3129
3583
|
########################################
|
@@ -5858,119 +6312,303 @@ class SocketHandler(abc.ABC):
|
|
5858
6312
|
raise NotImplementedError
|
5859
6313
|
|
5860
6314
|
|
5861
|
-
########################################
|
5862
|
-
#
|
6315
|
+
########################################
|
6316
|
+
# ../configs.py
|
6317
|
+
|
6318
|
+
|
6319
|
+
##
|
6320
|
+
|
6321
|
+
|
6322
|
+
class RestartWhenExitUnexpected:
|
6323
|
+
pass
|
6324
|
+
|
6325
|
+
|
6326
|
+
class RestartUnconditionally:
|
6327
|
+
pass
|
6328
|
+
|
6329
|
+
|
6330
|
+
##
|
6331
|
+
|
6332
|
+
|
6333
|
+
@dc.dataclass(frozen=True)
|
6334
|
+
class ProcessConfig:
|
6335
|
+
# A Python string expression that is used to compose the supervisor process name for this process. You usually don't
|
6336
|
+
# need to worry about setting this unless you change numprocs. The string expression is evaluated against a
|
6337
|
+
# dictionary that includes group_name, host_node_name, process_num, program_name, and here (the directory of the
|
6338
|
+
# supervisord config file).
|
6339
|
+
name: str
|
6340
|
+
|
6341
|
+
# The command that will be run when this program is started. The command can be either absolute (e.g.
|
6342
|
+
# /path/to/programname) or relative (e.g. programname). If it is relative, the supervisord's environment $PATH will
|
6343
|
+
# be searched for the executable. Programs can accept arguments, e.g. /path/to/program foo bar. The command line can
|
6344
|
+
# use double quotes to group arguments with spaces in them to pass to the program, e.g. /path/to/program/name -p
|
6345
|
+
# "foo bar". Note that the value of command may include Python string expressions, e.g. /path/to/programname
|
6346
|
+
# --port=80%(process_num)02d might expand to /path/to/programname --port=8000 at runtime. String expressions are
|
6347
|
+
# evaluated against a dictionary containing the keys group_name, host_node_name, program_name, process_num,
|
6348
|
+
# numprocs, here (the directory of the supervisord config file), and all supervisord's environment variables
|
6349
|
+
# prefixed with ENV_. Controlled programs should themselves not be daemons, as supervisord assumes it is responsible
|
6350
|
+
# for daemonizing its subprocesses
|
6351
|
+
command: str
|
6352
|
+
|
6353
|
+
#
|
6354
|
+
|
6355
|
+
# Supervisor will start as many instances of this program as named by numprocs. Note that if numprocs > 1, the
|
6356
|
+
# process_name expression must include %(process_num)s (or any other valid Python string expression that includes
|
6357
|
+
# process_num) within it.
|
6358
|
+
num_procs: int = 1
|
6359
|
+
|
6360
|
+
# An integer offset that is used to compute the number at which process_num starts.
|
6361
|
+
num_procs_start: int = 0
|
6362
|
+
|
6363
|
+
#
|
6364
|
+
|
6365
|
+
# Instruct supervisord to use this UNIX user account as the account which runs the program. The user can only be
|
6366
|
+
# switched if supervisord is run as the root user. If supervisord can't switch to the specified user, the program
|
6367
|
+
# will not be started.
|
6368
|
+
#
|
6369
|
+
# Note: The user will be changed using setuid only. This does not start a login shell and does not change
|
6370
|
+
# environment variables like USER or HOME
|
6371
|
+
user: ta.Optional[str] = None
|
6372
|
+
uid: ta.Optional[int] = None
|
6373
|
+
|
6374
|
+
# An octal number (e.g. 002, 022) representing the umask of the process.
|
6375
|
+
umask: ta.Optional[int] = None
|
6376
|
+
|
6377
|
+
#
|
6378
|
+
|
6379
|
+
# A file path representing a directory to which supervisord should temporarily chdir before exec'ing the child.
|
6380
|
+
directory: ta.Optional[str] = None
|
6381
|
+
|
6382
|
+
# A list of key/value pairs in the form KEY="val",KEY2="val2" that will be placed in the child process' environment.
|
6383
|
+
# The environment string may contain Python string expressions that will be evaluated against a dictionary
|
6384
|
+
# containing group_name, host_node_name, process_num, program_name, and here (the directory of the supervisord
|
6385
|
+
# config file). Values containing non-alphanumeric characters should be quoted (e.g. KEY="val:123",KEY2="val,456").
|
6386
|
+
# Otherwise, quoting the values is optional but recommended. Note that the subprocess will inherit the environment
|
6387
|
+
# variables of the shell used to start “supervisord” except for the ones overridden here.
|
6388
|
+
environment: ta.Optional[ta.Mapping[str, str]] = None
|
6389
|
+
|
6390
|
+
#
|
6391
|
+
|
6392
|
+
# The relative priority of the program in the start and shutdown ordering. Lower priorities indicate programs that
|
6393
|
+
# start first and shut down last at startup and when aggregate commands are used in various clients (e.g. “start
|
6394
|
+
# all”/”stop all”). Higher priorities indicate programs that start last and shut down first.
|
6395
|
+
priority: int = 999
|
6396
|
+
|
6397
|
+
# If true, this program will start automatically when supervisord is started.
|
6398
|
+
auto_start: bool = True
|
6399
|
+
|
6400
|
+
# Specifies if supervisord should automatically restart a process if it exits when it is in the RUNNING state. May
|
6401
|
+
# be one of false, unexpected, or true. If false, the process will not be autorestarted. If unexpected, the process
|
6402
|
+
# will be restarted when the program exits with an exit code that is not one of the exit codes associated with this
|
6403
|
+
# process' configuration (see exitcodes). If true, the process will be unconditionally restarted when it exits,
|
6404
|
+
# without regard to its exit code.
|
6405
|
+
#
|
6406
|
+
# Note: autorestart controls whether supervisord will autorestart a program if it exits after it has successfully
|
6407
|
+
# started up (the process is in the RUNNING state). supervisord has a different restart mechanism for when the
|
6408
|
+
# process is starting up (the process is in the STARTING state). Retries during process startup are controlled by
|
6409
|
+
# startsecs and startretries.
|
6410
|
+
auto_restart: str = 'unexpected'
|
6411
|
+
|
6412
|
+
# The total number of seconds which the program needs to stay running after a startup to consider the start
|
6413
|
+
# successful (moving the process from the STARTING state to the RUNNING state). Set to 0 to indicate that the
|
6414
|
+
# program needn't stay running for any particular amount of time.
|
6415
|
+
#
|
6416
|
+
# Note: Even if a process exits with an “expected” exit code (see exitcodes), the start will still be considered a
|
6417
|
+
# failure if the process exits quicker than startsecs.
|
6418
|
+
start_secs: int = 1
|
6419
|
+
|
6420
|
+
# The number of serial failure attempts that supervisord will allow when attempting to start the program before
|
6421
|
+
# giving up and putting the process into an FATAL state.
|
6422
|
+
#
|
6423
|
+
# Note: After each failed restart, process will be put in BACKOFF state and each retry attempt will take
|
6424
|
+
# increasingly more time.
|
6425
|
+
start_retries: int = 3
|
6426
|
+
|
6427
|
+
# The signal used to kill the program when a stop is requested. This can be specified using the signal's name or its
|
6428
|
+
# number. It is normally one of: TERM, HUP, INT, QUIT, KILL, USR1, or USR2.
|
6429
|
+
stop_signal: int = signal.SIGTERM
|
6430
|
+
|
6431
|
+
# The number of seconds to wait for the OS to return a SIGCHLD to supervisord after the program has been sent a
|
6432
|
+
# stopsignal. If this number of seconds elapses before supervisord receives a SIGCHLD from the process, supervisord
|
6433
|
+
# will attempt to kill it with a final SIGKILL.
|
6434
|
+
stop_wait_secs: int = 10
|
6435
|
+
|
6436
|
+
# If true, the flag causes supervisor to send the stop signal to the whole process group and implies killasgroup is
|
6437
|
+
# true. This is useful for programs, such as Flask in debug mode, that do not propagate stop signals to their
|
6438
|
+
# children, leaving them orphaned.
|
6439
|
+
stop_as_group: bool = False
|
6440
|
+
|
6441
|
+
# If true, when resorting to send SIGKILL to the program to terminate it send it to its whole process group instead,
|
6442
|
+
# taking care of its children as well, useful e.g with Python programs using multiprocessing.
|
6443
|
+
kill_as_group: bool = False
|
6444
|
+
|
6445
|
+
# The list of “expected” exit codes for this program used with autorestart. If the autorestart parameter is set to
|
6446
|
+
# unexpected, and the process exits in any other way than as a result of a supervisor stop request, supervisord will
|
6447
|
+
# restart the process if it exits with an exit code that is not defined in this list.
|
6448
|
+
#
|
6449
|
+
# Note: In Supervisor versions prior to 4.0, the default was 0,2. In Supervisor 4.0, the default was changed to 0.
|
6450
|
+
exitcodes: ta.Sequence[int] = (0,)
|
6451
|
+
|
6452
|
+
#
|
6453
|
+
|
6454
|
+
@dc.dataclass(frozen=True)
|
6455
|
+
class Log:
|
6456
|
+
file: ta.Optional[str] = None
|
6457
|
+
capture_max_bytes: ta.Optional[int] = None
|
6458
|
+
events_enabled: bool = False
|
6459
|
+
syslog: bool = False
|
6460
|
+
backups: ta.Optional[int] = None
|
6461
|
+
max_bytes: ta.Optional[int] = None
|
6462
|
+
|
6463
|
+
stdout: Log = Log()
|
6464
|
+
stderr: Log = Log()
|
6465
|
+
|
6466
|
+
# If true, cause the process' stderr output to be sent back to supervisord on its stdout file descriptor (in UNIX
|
6467
|
+
# shell terms, this is the equivalent of executing /the/program 2>&1).
|
6468
|
+
#
|
6469
|
+
# Note: Do not set redirect_stderr=true in an [eventlistener:x] section. Eventlisteners use stdout and stdin to
|
6470
|
+
# communicate with supervisord. If stderr is redirected, output from stderr will interfere with the eventlistener
|
6471
|
+
# protocol.
|
6472
|
+
redirect_stderr: bool = False
|
6473
|
+
|
6474
|
+
|
6475
|
+
@dc.dataclass(frozen=True)
|
6476
|
+
class ProcessGroupConfig:
|
6477
|
+
name: str
|
6478
|
+
|
6479
|
+
priority: int = 999
|
6480
|
+
|
6481
|
+
processes: ta.Optional[ta.Sequence[ProcessConfig]] = None
|
6482
|
+
|
6483
|
+
|
6484
|
+
@dc.dataclass(frozen=True)
|
6485
|
+
class ServerConfig:
|
6486
|
+
# Instruct supervisord to switch users to this UNIX user account before doing any meaningful processing. The user
|
6487
|
+
# can only be switched if supervisord is started as the root user.
|
6488
|
+
user: ta.Optional[str] = None
|
6489
|
+
|
6490
|
+
# If true, supervisord will start in the foreground instead of daemonizing.
|
6491
|
+
nodaemon: bool = False
|
6492
|
+
|
6493
|
+
# The umask of the supervisord process.
|
6494
|
+
umask: int = 0o22
|
6495
|
+
|
6496
|
+
#
|
5863
6497
|
|
6498
|
+
# When supervisord daemonizes, switch to this directory. This option can include the value %(here)s, which expands
|
6499
|
+
# to the directory in which the supervisord configuration file was found.
|
6500
|
+
directory: ta.Optional[str] = None
|
5864
6501
|
|
5865
|
-
|
6502
|
+
# The location in which supervisord keeps its pid file. This option can include the value %(here)s, which expands to
|
6503
|
+
# the directory in which the supervisord configuration file was found.
|
6504
|
+
pidfile: str = 'supervisord.pid'
|
5866
6505
|
|
6506
|
+
# The identifier string for this supervisor process, used by the RPC interface.
|
6507
|
+
identifier: str = 'supervisor'
|
5867
6508
|
|
5868
|
-
|
5869
|
-
|
5870
|
-
|
5871
|
-
|
5872
|
-
if name.endswith('.toml'):
|
5873
|
-
return toml_loads(f.read())
|
6509
|
+
# The minimum number of file descriptors that must be available before supervisord will start successfully.
|
6510
|
+
min_fds: int = 1024
|
6511
|
+
# The minimum number of process descriptors that must be available before supervisord will start successfully.
|
6512
|
+
min_procs: int = 200
|
5874
6513
|
|
5875
|
-
|
5876
|
-
|
5877
|
-
return yaml.safe_load(f)
|
6514
|
+
# Prevent supervisord from clearing any existing AUTO child log files at startup time. Useful for debugging
|
6515
|
+
nocleanup: bool = False
|
5878
6516
|
|
5879
|
-
|
5880
|
-
|
5881
|
-
cp = configparser.ConfigParser()
|
5882
|
-
cp.read_file(f)
|
5883
|
-
config_dct: ta.Dict[str, ta.Any] = {}
|
5884
|
-
for sec in cp.sections():
|
5885
|
-
cd = config_dct
|
5886
|
-
for k in sec.split('.'):
|
5887
|
-
cd = cd.setdefault(k, {})
|
5888
|
-
cd.update(cp.items(sec))
|
5889
|
-
return config_dct
|
6517
|
+
# Strip all ANSI escape sequences from child log files.
|
6518
|
+
strip_ansi: bool = False
|
5890
6519
|
|
5891
|
-
|
5892
|
-
return json.loads(f.read())
|
6520
|
+
#
|
5893
6521
|
|
6522
|
+
# The path to the activity log of the supervisord process. This option can include the value %(here)s, which expands
|
6523
|
+
# to the directory in which the supervisord configuration file was found.
|
6524
|
+
logfile: str = 'supervisord.log'
|
5894
6525
|
|
5895
|
-
|
5896
|
-
|
5897
|
-
|
5898
|
-
*,
|
5899
|
-
prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
|
5900
|
-
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
5901
|
-
) -> T:
|
5902
|
-
with open(path) as cf:
|
5903
|
-
config_dct = parse_config_file(os.path.basename(path), cf)
|
6526
|
+
# The maximum number of bytes that may be consumed by the activity log file before it is rotated (suffix multipliers
|
6527
|
+
# like “KB”, “MB”, and “GB” can be used in the value). Set this value to 0 to indicate an unlimited log size.
|
6528
|
+
logfile_max_bytes: int = 50 * 1024 * 1024
|
5904
6529
|
|
5905
|
-
|
5906
|
-
|
6530
|
+
# The number of backups to keep around resulting from activity log file rotation. If set to 0, no backups will be
|
6531
|
+
# kept.
|
6532
|
+
logfile_backups: int = 10
|
5907
6533
|
|
5908
|
-
|
6534
|
+
# The logging level, dictating what is written to the supervisord activity log. One of critical, error, warn, info,
|
6535
|
+
# debug, trace, or blather. Note that at log level debug, the supervisord log file will record the stderr/stdout
|
6536
|
+
# output of its child processes and extended info about process state changes, which is useful for debugging a
|
6537
|
+
# process which isn't starting properly.
|
6538
|
+
loglevel: int = logging.INFO
|
5909
6539
|
|
6540
|
+
# The directory used for AUTO child log files. This option can include the value %(here)s, which expands to the
|
6541
|
+
# directory in which the supervisord configuration file was found.
|
6542
|
+
child_logdir: str = '/dev/null'
|
5910
6543
|
|
5911
|
-
|
6544
|
+
# If true and not daemonized, logs will not be directed to stdout.
|
6545
|
+
silent: bool = False
|
5912
6546
|
|
6547
|
+
#
|
5913
6548
|
|
5914
|
-
|
5915
|
-
o: ta.Union[
|
5916
|
-
ta.Sequence[ConfigMapping],
|
5917
|
-
ta.Mapping[str, ConfigMapping],
|
5918
|
-
None,
|
5919
|
-
],
|
5920
|
-
*,
|
5921
|
-
name_key: str = 'name',
|
5922
|
-
) -> ta.Optional[ta.Sequence[ConfigMapping]]:
|
5923
|
-
if o is None:
|
5924
|
-
return None
|
6549
|
+
groups: ta.Optional[ta.Sequence[ProcessGroupConfig]] = None
|
5925
6550
|
|
5926
|
-
|
5927
|
-
|
5928
|
-
for k, v in o.items():
|
5929
|
-
check.isinstance(v, ta.Mapping)
|
5930
|
-
if name_key in v:
|
5931
|
-
n = v[name_key]
|
5932
|
-
if k != n:
|
5933
|
-
raise KeyError(f'Given names do not match: {n} != {k}')
|
5934
|
-
lst.append(v)
|
5935
|
-
else:
|
5936
|
-
lst.append({name_key: k, **v})
|
6551
|
+
# TODO: implement - make sure to accept broken symlinks
|
6552
|
+
group_config_dirs: ta.Optional[ta.Sequence[str]] = None
|
5937
6553
|
|
5938
|
-
|
5939
|
-
check.not_isinstance(o, str)
|
5940
|
-
lst.extend(o)
|
6554
|
+
#
|
5941
6555
|
|
5942
|
-
|
5943
|
-
for d in lst:
|
5944
|
-
n = d['name']
|
5945
|
-
if n in d:
|
5946
|
-
raise KeyError(f'Duplicate name: {n}')
|
5947
|
-
seen.add(n)
|
6556
|
+
http_port: ta.Optional[int] = None
|
5948
6557
|
|
5949
|
-
|
6558
|
+
#
|
6559
|
+
|
6560
|
+
@classmethod
|
6561
|
+
def new(
|
6562
|
+
cls,
|
6563
|
+
*,
|
6564
|
+
umask: ta.Union[int, str] = 0o22,
|
6565
|
+
directory: ta.Optional[str] = None,
|
6566
|
+
logfile: str = 'supervisord.log',
|
6567
|
+
logfile_max_bytes: ta.Union[int, str] = 50 * 1024 * 1024,
|
6568
|
+
loglevel: ta.Union[int, str] = logging.INFO,
|
6569
|
+
pidfile: str = 'supervisord.pid',
|
6570
|
+
child_logdir: ta.Optional[str] = None,
|
6571
|
+
**kwargs: ta.Any,
|
6572
|
+
) -> 'ServerConfig':
|
6573
|
+
return cls(
|
6574
|
+
umask=parse_octal(umask),
|
6575
|
+
directory=check_existing_dir(directory) if directory is not None else None,
|
6576
|
+
logfile=check_path_with_existing_dir(logfile),
|
6577
|
+
logfile_max_bytes=parse_bytes_size(logfile_max_bytes),
|
6578
|
+
loglevel=parse_logging_level(loglevel),
|
6579
|
+
pidfile=check_path_with_existing_dir(pidfile),
|
6580
|
+
child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
|
6581
|
+
**kwargs,
|
6582
|
+
)
|
5950
6583
|
|
5951
6584
|
|
5952
6585
|
##
|
5953
6586
|
|
5954
6587
|
|
5955
|
-
def
|
5956
|
-
|
5957
|
-
|
5958
|
-
out
|
6588
|
+
def prepare_process_group_config(dct: ConfigMap) -> ConfigMap:
|
6589
|
+
out = dict(dct)
|
6590
|
+
out['processes'] = build_config_named_children(out.get('processes'))
|
6591
|
+
return out
|
5959
6592
|
|
5960
|
-
for i, (section, settings) in enumerate(settings_by_section.items()):
|
5961
|
-
if i:
|
5962
|
-
out.write('\n')
|
5963
6593
|
|
5964
|
-
|
6594
|
+
def prepare_server_config(dct: ta.Mapping[str, ta.Any]) -> ta.Mapping[str, ta.Any]:
|
6595
|
+
out = dict(dct)
|
6596
|
+
group_dcts = build_config_named_children(out.get('groups'))
|
6597
|
+
out['groups'] = [prepare_process_group_config(group_dct) for group_dct in group_dcts or []]
|
6598
|
+
return out
|
5965
6599
|
|
5966
|
-
for k, v in settings.items():
|
5967
|
-
if isinstance(v, str):
|
5968
|
-
out.write(f'{k}={v}\n')
|
5969
|
-
else:
|
5970
|
-
for vv in v:
|
5971
|
-
out.write(f'{k}={vv}\n')
|
5972
6600
|
|
5973
|
-
|
6601
|
+
##
|
6602
|
+
|
6603
|
+
|
6604
|
+
def parse_logging_level(value: ta.Union[str, int]) -> int:
|
6605
|
+
if isinstance(value, int):
|
6606
|
+
return value
|
6607
|
+
s = str(value).lower()
|
6608
|
+
level = logging.getLevelNamesMapping().get(s.upper())
|
6609
|
+
if level is None:
|
6610
|
+
raise ValueError(f'bad logging level name {value!r}')
|
6611
|
+
return level
|
5974
6612
|
|
5975
6613
|
|
5976
6614
|
########################################
|
@@ -6120,6 +6758,38 @@ class UnsupportedMethodHttpHandlerError(Exception):
|
|
6120
6758
|
pass
|
6121
6759
|
|
6122
6760
|
|
6761
|
+
########################################
|
6762
|
+
# ../../../omlish/lite/configs.py
|
6763
|
+
|
6764
|
+
|
6765
|
+
##
|
6766
|
+
|
6767
|
+
|
6768
|
+
def load_config_file_obj(
|
6769
|
+
f: str,
|
6770
|
+
cls: ta.Type[T],
|
6771
|
+
*,
|
6772
|
+
prepare: ta.Union[
|
6773
|
+
ta.Callable[[ConfigMap], ConfigMap],
|
6774
|
+
ta.Iterable[ta.Callable[[ConfigMap], ConfigMap]],
|
6775
|
+
] = (),
|
6776
|
+
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
6777
|
+
) -> T:
|
6778
|
+
config_data = DEFAULT_CONFIG_FILE_LOADER.load_file(f)
|
6779
|
+
|
6780
|
+
config_dct = config_data.as_map()
|
6781
|
+
|
6782
|
+
if prepare is not None:
|
6783
|
+
if isinstance(prepare, ta.Iterable):
|
6784
|
+
pfs = list(prepare)
|
6785
|
+
else:
|
6786
|
+
pfs = [prepare]
|
6787
|
+
for pf in pfs:
|
6788
|
+
config_dct = pf(config_dct)
|
6789
|
+
|
6790
|
+
return msh.unmarshal_obj(config_dct, cls)
|
6791
|
+
|
6792
|
+
|
6123
6793
|
########################################
|
6124
6794
|
# ../../../omlish/logs/standard.py
|
6125
6795
|
"""
|
@@ -6242,303 +6912,184 @@ def configure_standard_logging(
|
|
6242
6912
|
return StandardConfiguredLogHandler(handler)
|
6243
6913
|
|
6244
6914
|
|
6245
|
-
########################################
|
6246
|
-
# ../
|
6247
|
-
|
6248
|
-
|
6249
|
-
##
|
6250
|
-
|
6251
|
-
|
6252
|
-
class RestartWhenExitUnexpected:
|
6253
|
-
pass
|
6254
|
-
|
6255
|
-
|
6256
|
-
class RestartUnconditionally:
|
6257
|
-
pass
|
6258
|
-
|
6259
|
-
|
6260
|
-
##
|
6261
|
-
|
6262
|
-
|
6263
|
-
@dc.dataclass(frozen=True)
|
6264
|
-
class ProcessConfig:
|
6265
|
-
# A Python string expression that is used to compose the supervisor process name for this process. You usually don't
|
6266
|
-
# need to worry about setting this unless you change numprocs. The string expression is evaluated against a
|
6267
|
-
# dictionary that includes group_name, host_node_name, process_num, program_name, and here (the directory of the
|
6268
|
-
# supervisord config file).
|
6269
|
-
name: str
|
6270
|
-
|
6271
|
-
# The command that will be run when this program is started. The command can be either absolute (e.g.
|
6272
|
-
# /path/to/programname) or relative (e.g. programname). If it is relative, the supervisord's environment $PATH will
|
6273
|
-
# be searched for the executable. Programs can accept arguments, e.g. /path/to/program foo bar. The command line can
|
6274
|
-
# use double quotes to group arguments with spaces in them to pass to the program, e.g. /path/to/program/name -p
|
6275
|
-
# "foo bar". Note that the value of command may include Python string expressions, e.g. /path/to/programname
|
6276
|
-
# --port=80%(process_num)02d might expand to /path/to/programname --port=8000 at runtime. String expressions are
|
6277
|
-
# evaluated against a dictionary containing the keys group_name, host_node_name, program_name, process_num,
|
6278
|
-
# numprocs, here (the directory of the supervisord config file), and all supervisord's environment variables
|
6279
|
-
# prefixed with ENV_. Controlled programs should themselves not be daemons, as supervisord assumes it is responsible
|
6280
|
-
# for daemonizing its subprocesses
|
6281
|
-
command: str
|
6282
|
-
|
6283
|
-
#
|
6284
|
-
|
6285
|
-
# Supervisor will start as many instances of this program as named by numprocs. Note that if numprocs > 1, the
|
6286
|
-
# process_name expression must include %(process_num)s (or any other valid Python string expression that includes
|
6287
|
-
# process_num) within it.
|
6288
|
-
num_procs: int = 1
|
6289
|
-
|
6290
|
-
# An integer offset that is used to compute the number at which process_num starts.
|
6291
|
-
num_procs_start: int = 0
|
6292
|
-
|
6293
|
-
#
|
6294
|
-
|
6295
|
-
# Instruct supervisord to use this UNIX user account as the account which runs the program. The user can only be
|
6296
|
-
# switched if supervisord is run as the root user. If supervisord can't switch to the specified user, the program
|
6297
|
-
# will not be started.
|
6298
|
-
#
|
6299
|
-
# Note: The user will be changed using setuid only. This does not start a login shell and does not change
|
6300
|
-
# environment variables like USER or HOME
|
6301
|
-
user: ta.Optional[str] = None
|
6302
|
-
uid: ta.Optional[int] = None
|
6303
|
-
|
6304
|
-
# An octal number (e.g. 002, 022) representing the umask of the process.
|
6305
|
-
umask: ta.Optional[int] = None
|
6306
|
-
|
6307
|
-
#
|
6308
|
-
|
6309
|
-
# A file path representing a directory to which supervisord should temporarily chdir before exec'ing the child.
|
6310
|
-
directory: ta.Optional[str] = None
|
6311
|
-
|
6312
|
-
# A list of key/value pairs in the form KEY="val",KEY2="val2" that will be placed in the child process' environment.
|
6313
|
-
# The environment string may contain Python string expressions that will be evaluated against a dictionary
|
6314
|
-
# containing group_name, host_node_name, process_num, program_name, and here (the directory of the supervisord
|
6315
|
-
# config file). Values containing non-alphanumeric characters should be quoted (e.g. KEY="val:123",KEY2="val,456").
|
6316
|
-
# Otherwise, quoting the values is optional but recommended. Note that the subprocess will inherit the environment
|
6317
|
-
# variables of the shell used to start “supervisord” except for the ones overridden here.
|
6318
|
-
environment: ta.Optional[ta.Mapping[str, str]] = None
|
6319
|
-
|
6320
|
-
#
|
6321
|
-
|
6322
|
-
# The relative priority of the program in the start and shutdown ordering. Lower priorities indicate programs that
|
6323
|
-
# start first and shut down last at startup and when aggregate commands are used in various clients (e.g. “start
|
6324
|
-
# all”/”stop all”). Higher priorities indicate programs that start last and shut down first.
|
6325
|
-
priority: int = 999
|
6326
|
-
|
6327
|
-
# If true, this program will start automatically when supervisord is started.
|
6328
|
-
auto_start: bool = True
|
6329
|
-
|
6330
|
-
# Specifies if supervisord should automatically restart a process if it exits when it is in the RUNNING state. May
|
6331
|
-
# be one of false, unexpected, or true. If false, the process will not be autorestarted. If unexpected, the process
|
6332
|
-
# will be restarted when the program exits with an exit code that is not one of the exit codes associated with this
|
6333
|
-
# process' configuration (see exitcodes). If true, the process will be unconditionally restarted when it exits,
|
6334
|
-
# without regard to its exit code.
|
6335
|
-
#
|
6336
|
-
# Note: autorestart controls whether supervisord will autorestart a program if it exits after it has successfully
|
6337
|
-
# started up (the process is in the RUNNING state). supervisord has a different restart mechanism for when the
|
6338
|
-
# process is starting up (the process is in the STARTING state). Retries during process startup are controlled by
|
6339
|
-
# startsecs and startretries.
|
6340
|
-
auto_restart: str = 'unexpected'
|
6915
|
+
########################################
|
6916
|
+
# ../types.py
|
6341
6917
|
|
6342
|
-
# The total number of seconds which the program needs to stay running after a startup to consider the start
|
6343
|
-
# successful (moving the process from the STARTING state to the RUNNING state). Set to 0 to indicate that the
|
6344
|
-
# program needn't stay running for any particular amount of time.
|
6345
|
-
#
|
6346
|
-
# Note: Even if a process exits with an “expected” exit code (see exitcodes), the start will still be considered a
|
6347
|
-
# failure if the process exits quicker than startsecs.
|
6348
|
-
start_secs: int = 1
|
6349
6918
|
|
6350
|
-
|
6351
|
-
# giving up and putting the process into an FATAL state.
|
6352
|
-
#
|
6353
|
-
# Note: After each failed restart, process will be put in BACKOFF state and each retry attempt will take
|
6354
|
-
# increasingly more time.
|
6355
|
-
start_retries: int = 3
|
6919
|
+
##
|
6356
6920
|
|
6357
|
-
# The signal used to kill the program when a stop is requested. This can be specified using the signal's name or its
|
6358
|
-
# number. It is normally one of: TERM, HUP, INT, QUIT, KILL, USR1, or USR2.
|
6359
|
-
stop_signal: int = signal.SIGTERM
|
6360
6921
|
|
6361
|
-
|
6362
|
-
|
6363
|
-
# will attempt to kill it with a final SIGKILL.
|
6364
|
-
stop_wait_secs: int = 10
|
6922
|
+
class ExitNow(Exception): # noqa
|
6923
|
+
pass
|
6365
6924
|
|
6366
|
-
# If true, the flag causes supervisor to send the stop signal to the whole process group and implies killasgroup is
|
6367
|
-
# true. This is useful for programs, such as Flask in debug mode, that do not propagate stop signals to their
|
6368
|
-
# children, leaving them orphaned.
|
6369
|
-
stop_as_group: bool = False
|
6370
6925
|
|
6371
|
-
|
6372
|
-
# taking care of its children as well, useful e.g with Python programs using multiprocessing.
|
6373
|
-
kill_as_group: bool = False
|
6926
|
+
ServerEpoch = ta.NewType('ServerEpoch', int)
|
6374
6927
|
|
6375
|
-
# The list of “expected” exit codes for this program used with autorestart. If the autorestart parameter is set to
|
6376
|
-
# unexpected, and the process exits in any other way than as a result of a supervisor stop request, supervisord will
|
6377
|
-
# restart the process if it exits with an exit code that is not defined in this list.
|
6378
|
-
#
|
6379
|
-
# Note: In Supervisor versions prior to 4.0, the default was 0,2. In Supervisor 4.0, the default was changed to 0.
|
6380
|
-
exitcodes: ta.Sequence[int] = (0,)
|
6381
6928
|
|
6382
|
-
|
6929
|
+
##
|
6383
6930
|
|
6384
|
-
@dc.dataclass(frozen=True)
|
6385
|
-
class Log:
|
6386
|
-
file: ta.Optional[str] = None
|
6387
|
-
capture_max_bytes: ta.Optional[int] = None
|
6388
|
-
events_enabled: bool = False
|
6389
|
-
syslog: bool = False
|
6390
|
-
backups: ta.Optional[int] = None
|
6391
|
-
max_bytes: ta.Optional[int] = None
|
6392
6931
|
|
6393
|
-
|
6394
|
-
|
6932
|
+
@functools.total_ordering
|
6933
|
+
class ConfigPriorityOrdered(abc.ABC):
|
6934
|
+
@property
|
6935
|
+
@abc.abstractmethod
|
6936
|
+
def config(self) -> ta.Any:
|
6937
|
+
raise NotImplementedError
|
6395
6938
|
|
6396
|
-
|
6397
|
-
|
6398
|
-
#
|
6399
|
-
# Note: Do not set redirect_stderr=true in an [eventlistener:x] section. Eventlisteners use stdout and stdin to
|
6400
|
-
# communicate with supervisord. If stderr is redirected, output from stderr will interfere with the eventlistener
|
6401
|
-
# protocol.
|
6402
|
-
redirect_stderr: bool = False
|
6939
|
+
def __lt__(self, other):
|
6940
|
+
return self.config.priority < other.config.priority
|
6403
6941
|
|
6942
|
+
def __eq__(self, other):
|
6943
|
+
return self.config.priority == other.config.priority
|
6404
6944
|
|
6405
|
-
@dc.dataclass(frozen=True)
|
6406
|
-
class ProcessGroupConfig:
|
6407
|
-
name: str
|
6408
6945
|
|
6409
|
-
|
6946
|
+
##
|
6410
6947
|
|
6411
|
-
processes: ta.Optional[ta.Sequence[ProcessConfig]] = None
|
6412
6948
|
|
6949
|
+
class SupervisorStateManager(abc.ABC):
|
6950
|
+
@property
|
6951
|
+
@abc.abstractmethod
|
6952
|
+
def state(self) -> SupervisorState:
|
6953
|
+
raise NotImplementedError
|
6413
6954
|
|
6414
|
-
@
|
6415
|
-
|
6416
|
-
|
6417
|
-
# can only be switched if supervisord is started as the root user.
|
6418
|
-
user: ta.Optional[str] = None
|
6955
|
+
@abc.abstractmethod
|
6956
|
+
def set_state(self, state: SupervisorState) -> None:
|
6957
|
+
raise NotImplementedError
|
6419
6958
|
|
6420
|
-
# If true, supervisord will start in the foreground instead of daemonizing.
|
6421
|
-
nodaemon: bool = False
|
6422
6959
|
|
6423
|
-
|
6424
|
-
umask: int = 0o22
|
6960
|
+
##
|
6425
6961
|
|
6426
|
-
#
|
6427
6962
|
|
6428
|
-
|
6429
|
-
|
6430
|
-
|
6963
|
+
class HasDispatchers(abc.ABC):
|
6964
|
+
@abc.abstractmethod
|
6965
|
+
def get_dispatchers(self) -> 'Dispatchers':
|
6966
|
+
raise NotImplementedError
|
6431
6967
|
|
6432
|
-
# The location in which supervisord keeps its pid file. This option can include the value %(here)s, which expands to
|
6433
|
-
# the directory in which the supervisord configuration file was found.
|
6434
|
-
pidfile: str = 'supervisord.pid'
|
6435
6968
|
|
6436
|
-
|
6437
|
-
|
6969
|
+
class ProcessDispatcher(FdioHandler, abc.ABC):
|
6970
|
+
@property
|
6971
|
+
@abc.abstractmethod
|
6972
|
+
def channel(self) -> ProcessOutputChannel:
|
6973
|
+
raise NotImplementedError
|
6438
6974
|
|
6439
|
-
|
6440
|
-
|
6441
|
-
|
6442
|
-
|
6975
|
+
@property
|
6976
|
+
@abc.abstractmethod
|
6977
|
+
def process(self) -> 'Process':
|
6978
|
+
raise NotImplementedError
|
6443
6979
|
|
6444
|
-
# Prevent supervisord from clearing any existing AUTO child log files at startup time. Useful for debugging
|
6445
|
-
nocleanup: bool = False
|
6446
6980
|
|
6447
|
-
|
6448
|
-
|
6981
|
+
class ProcessOutputDispatcher(ProcessDispatcher, abc.ABC):
|
6982
|
+
@abc.abstractmethod
|
6983
|
+
def remove_logs(self) -> None:
|
6984
|
+
raise NotImplementedError
|
6449
6985
|
|
6450
|
-
|
6986
|
+
@abc.abstractmethod
|
6987
|
+
def reopen_logs(self) -> None:
|
6988
|
+
raise NotImplementedError
|
6451
6989
|
|
6452
|
-
# The path to the activity log of the supervisord process. This option can include the value %(here)s, which expands
|
6453
|
-
# to the directory in which the supervisord configuration file was found.
|
6454
|
-
logfile: str = 'supervisord.log'
|
6455
6990
|
|
6456
|
-
|
6457
|
-
|
6458
|
-
|
6991
|
+
class ProcessInputDispatcher(ProcessDispatcher, abc.ABC):
|
6992
|
+
@abc.abstractmethod
|
6993
|
+
def write(self, chars: ta.Union[bytes, str]) -> None:
|
6994
|
+
raise NotImplementedError
|
6459
6995
|
|
6460
|
-
|
6461
|
-
|
6462
|
-
|
6996
|
+
@abc.abstractmethod
|
6997
|
+
def flush(self) -> None:
|
6998
|
+
raise NotImplementedError
|
6463
6999
|
|
6464
|
-
# The logging level, dictating what is written to the supervisord activity log. One of critical, error, warn, info,
|
6465
|
-
# debug, trace, or blather. Note that at log level debug, the supervisord log file will record the stderr/stdout
|
6466
|
-
# output of its child processes and extended info about process state changes, which is useful for debugging a
|
6467
|
-
# process which isn't starting properly.
|
6468
|
-
loglevel: int = logging.INFO
|
6469
7000
|
|
6470
|
-
|
6471
|
-
# directory in which the supervisord configuration file was found.
|
6472
|
-
child_logdir: str = '/dev/null'
|
7001
|
+
##
|
6473
7002
|
|
6474
|
-
# If true and not daemonized, logs will not be directed to stdout.
|
6475
|
-
silent: bool = False
|
6476
7003
|
|
6477
|
-
|
7004
|
+
class Process(
|
7005
|
+
ConfigPriorityOrdered,
|
7006
|
+
HasDispatchers,
|
7007
|
+
abc.ABC,
|
7008
|
+
):
|
7009
|
+
@property
|
7010
|
+
@abc.abstractmethod
|
7011
|
+
def name(self) -> str:
|
7012
|
+
raise NotImplementedError
|
6478
7013
|
|
6479
|
-
|
7014
|
+
@property
|
7015
|
+
@abc.abstractmethod
|
7016
|
+
def config(self) -> ProcessConfig:
|
7017
|
+
raise NotImplementedError
|
6480
7018
|
|
6481
|
-
|
6482
|
-
|
7019
|
+
@property
|
7020
|
+
@abc.abstractmethod
|
7021
|
+
def group(self) -> 'ProcessGroup':
|
7022
|
+
raise NotImplementedError
|
7023
|
+
|
7024
|
+
@property
|
7025
|
+
@abc.abstractmethod
|
7026
|
+
def pid(self) -> Pid:
|
7027
|
+
raise NotImplementedError
|
6483
7028
|
|
6484
7029
|
#
|
6485
7030
|
|
6486
|
-
|
7031
|
+
@abc.abstractmethod
|
7032
|
+
def finish(self, sts: Rc) -> None:
|
7033
|
+
raise NotImplementedError
|
6487
7034
|
|
6488
|
-
|
7035
|
+
@abc.abstractmethod
|
7036
|
+
def stop(self) -> ta.Optional[str]:
|
7037
|
+
raise NotImplementedError
|
6489
7038
|
|
6490
|
-
@
|
6491
|
-
def
|
6492
|
-
|
6493
|
-
|
6494
|
-
|
6495
|
-
|
6496
|
-
|
6497
|
-
|
6498
|
-
|
6499
|
-
|
6500
|
-
|
6501
|
-
|
6502
|
-
|
6503
|
-
|
6504
|
-
|
6505
|
-
|
6506
|
-
logfile=check_path_with_existing_dir(logfile),
|
6507
|
-
logfile_max_bytes=parse_bytes_size(logfile_max_bytes),
|
6508
|
-
loglevel=parse_logging_level(loglevel),
|
6509
|
-
pidfile=check_path_with_existing_dir(pidfile),
|
6510
|
-
child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
|
6511
|
-
**kwargs,
|
6512
|
-
)
|
7039
|
+
@abc.abstractmethod
|
7040
|
+
def give_up(self) -> None:
|
7041
|
+
raise NotImplementedError
|
7042
|
+
|
7043
|
+
@abc.abstractmethod
|
7044
|
+
def transition(self) -> None:
|
7045
|
+
raise NotImplementedError
|
7046
|
+
|
7047
|
+
@property
|
7048
|
+
@abc.abstractmethod
|
7049
|
+
def state(self) -> ProcessState:
|
7050
|
+
raise NotImplementedError
|
7051
|
+
|
7052
|
+
@abc.abstractmethod
|
7053
|
+
def after_setuid(self) -> None:
|
7054
|
+
raise NotImplementedError
|
6513
7055
|
|
6514
7056
|
|
6515
7057
|
##
|
6516
7058
|
|
6517
7059
|
|
6518
|
-
|
6519
|
-
|
6520
|
-
|
6521
|
-
|
7060
|
+
class ProcessGroup(
|
7061
|
+
ConfigPriorityOrdered,
|
7062
|
+
KeyedCollectionAccessors[str, Process],
|
7063
|
+
abc.ABC,
|
7064
|
+
):
|
7065
|
+
@property
|
7066
|
+
@abc.abstractmethod
|
7067
|
+
def name(self) -> str:
|
7068
|
+
raise NotImplementedError
|
6522
7069
|
|
7070
|
+
@property
|
7071
|
+
@abc.abstractmethod
|
7072
|
+
def config(self) -> ProcessGroupConfig:
|
7073
|
+
raise NotImplementedError
|
6523
7074
|
|
6524
|
-
|
6525
|
-
|
6526
|
-
|
6527
|
-
|
6528
|
-
return out
|
7075
|
+
@property
|
7076
|
+
@abc.abstractmethod
|
7077
|
+
def by_name(self) -> ta.Mapping[str, Process]:
|
7078
|
+
raise NotImplementedError
|
6529
7079
|
|
7080
|
+
#
|
6530
7081
|
|
6531
|
-
|
7082
|
+
@abc.abstractmethod
|
7083
|
+
def stop_all(self) -> None:
|
7084
|
+
raise NotImplementedError
|
6532
7085
|
|
7086
|
+
@abc.abstractmethod
|
7087
|
+
def get_unstopped_processes(self) -> ta.List[Process]:
|
7088
|
+
raise NotImplementedError
|
6533
7089
|
|
6534
|
-
|
6535
|
-
|
6536
|
-
|
6537
|
-
s = str(value).lower()
|
6538
|
-
level = logging.getLevelNamesMapping().get(s.upper())
|
6539
|
-
if level is None:
|
6540
|
-
raise ValueError(f'bad logging level name {value!r}')
|
6541
|
-
return level
|
7090
|
+
@abc.abstractmethod
|
7091
|
+
def before_remove(self) -> None:
|
7092
|
+
raise NotImplementedError
|
6542
7093
|
|
6543
7094
|
|
6544
7095
|
########################################
|
@@ -7047,368 +7598,59 @@ class CoroHttpServer:
|
|
7047
7598
|
##
|
7048
7599
|
|
7049
7600
|
|
7050
|
-
class CoroHttpServerSocketHandler(SocketHandler):
|
7051
|
-
def __init__(
|
7052
|
-
self,
|
7053
|
-
client_address: SocketAddress,
|
7054
|
-
rfile: ta.BinaryIO,
|
7055
|
-
wfile: ta.BinaryIO,
|
7056
|
-
*,
|
7057
|
-
server_factory: CoroHttpServerFactory,
|
7058
|
-
log_handler: ta.Optional[ta.Callable[[CoroHttpServer, CoroHttpServer.AnyLogIo], None]] = None,
|
7059
|
-
) -> None:
|
7060
|
-
super().__init__(
|
7061
|
-
client_address,
|
7062
|
-
rfile,
|
7063
|
-
wfile,
|
7064
|
-
)
|
7065
|
-
|
7066
|
-
self._server_factory = server_factory
|
7067
|
-
self._log_handler = log_handler
|
7068
|
-
|
7069
|
-
def handle(self) -> None:
|
7070
|
-
server = self._server_factory(self._client_address)
|
7071
|
-
|
7072
|
-
gen = server.coro_handle()
|
7073
|
-
|
7074
|
-
o = next(gen)
|
7075
|
-
while True:
|
7076
|
-
if isinstance(o, CoroHttpServer.AnyLogIo):
|
7077
|
-
i = None
|
7078
|
-
if self._log_handler is not None:
|
7079
|
-
self._log_handler(server, o)
|
7080
|
-
|
7081
|
-
elif isinstance(o, CoroHttpServer.ReadIo):
|
7082
|
-
i = self._rfile.read(o.sz)
|
7083
|
-
|
7084
|
-
elif isinstance(o, CoroHttpServer.ReadLineIo):
|
7085
|
-
i = self._rfile.readline(o.sz)
|
7086
|
-
|
7087
|
-
elif isinstance(o, CoroHttpServer.WriteIo):
|
7088
|
-
i = None
|
7089
|
-
self._wfile.write(o.data)
|
7090
|
-
self._wfile.flush()
|
7091
|
-
|
7092
|
-
else:
|
7093
|
-
raise TypeError(o)
|
7094
|
-
|
7095
|
-
try:
|
7096
|
-
if i is not None:
|
7097
|
-
o = gen.send(i)
|
7098
|
-
else:
|
7099
|
-
o = next(gen)
|
7100
|
-
except StopIteration:
|
7101
|
-
break
|
7102
|
-
|
7103
|
-
|
7104
|
-
########################################
|
7105
|
-
# ../types.py
|
7106
|
-
|
7107
|
-
|
7108
|
-
##
|
7109
|
-
|
7110
|
-
|
7111
|
-
class ExitNow(Exception): # noqa
|
7112
|
-
pass
|
7113
|
-
|
7114
|
-
|
7115
|
-
ServerEpoch = ta.NewType('ServerEpoch', int)
|
7116
|
-
|
7117
|
-
|
7118
|
-
##
|
7119
|
-
|
7120
|
-
|
7121
|
-
@functools.total_ordering
|
7122
|
-
class ConfigPriorityOrdered(abc.ABC):
|
7123
|
-
@property
|
7124
|
-
@abc.abstractmethod
|
7125
|
-
def config(self) -> ta.Any:
|
7126
|
-
raise NotImplementedError
|
7127
|
-
|
7128
|
-
def __lt__(self, other):
|
7129
|
-
return self.config.priority < other.config.priority
|
7130
|
-
|
7131
|
-
def __eq__(self, other):
|
7132
|
-
return self.config.priority == other.config.priority
|
7133
|
-
|
7134
|
-
|
7135
|
-
##
|
7136
|
-
|
7137
|
-
|
7138
|
-
class SupervisorStateManager(abc.ABC):
|
7139
|
-
@property
|
7140
|
-
@abc.abstractmethod
|
7141
|
-
def state(self) -> SupervisorState:
|
7142
|
-
raise NotImplementedError
|
7143
|
-
|
7144
|
-
@abc.abstractmethod
|
7145
|
-
def set_state(self, state: SupervisorState) -> None:
|
7146
|
-
raise NotImplementedError
|
7147
|
-
|
7148
|
-
|
7149
|
-
##
|
7150
|
-
|
7151
|
-
|
7152
|
-
class HasDispatchers(abc.ABC):
|
7153
|
-
@abc.abstractmethod
|
7154
|
-
def get_dispatchers(self) -> 'Dispatchers':
|
7155
|
-
raise NotImplementedError
|
7156
|
-
|
7157
|
-
|
7158
|
-
class ProcessDispatcher(FdioHandler, abc.ABC):
|
7159
|
-
@property
|
7160
|
-
@abc.abstractmethod
|
7161
|
-
def channel(self) -> ProcessOutputChannel:
|
7162
|
-
raise NotImplementedError
|
7163
|
-
|
7164
|
-
@property
|
7165
|
-
@abc.abstractmethod
|
7166
|
-
def process(self) -> 'Process':
|
7167
|
-
raise NotImplementedError
|
7168
|
-
|
7169
|
-
|
7170
|
-
class ProcessOutputDispatcher(ProcessDispatcher, abc.ABC):
|
7171
|
-
@abc.abstractmethod
|
7172
|
-
def remove_logs(self) -> None:
|
7173
|
-
raise NotImplementedError
|
7174
|
-
|
7175
|
-
@abc.abstractmethod
|
7176
|
-
def reopen_logs(self) -> None:
|
7177
|
-
raise NotImplementedError
|
7178
|
-
|
7179
|
-
|
7180
|
-
class ProcessInputDispatcher(ProcessDispatcher, abc.ABC):
|
7181
|
-
@abc.abstractmethod
|
7182
|
-
def write(self, chars: ta.Union[bytes, str]) -> None:
|
7183
|
-
raise NotImplementedError
|
7184
|
-
|
7185
|
-
@abc.abstractmethod
|
7186
|
-
def flush(self) -> None:
|
7187
|
-
raise NotImplementedError
|
7188
|
-
|
7189
|
-
|
7190
|
-
##
|
7191
|
-
|
7192
|
-
|
7193
|
-
class Process(
|
7194
|
-
ConfigPriorityOrdered,
|
7195
|
-
HasDispatchers,
|
7196
|
-
abc.ABC,
|
7197
|
-
):
|
7198
|
-
@property
|
7199
|
-
@abc.abstractmethod
|
7200
|
-
def name(self) -> str:
|
7201
|
-
raise NotImplementedError
|
7202
|
-
|
7203
|
-
@property
|
7204
|
-
@abc.abstractmethod
|
7205
|
-
def config(self) -> ProcessConfig:
|
7206
|
-
raise NotImplementedError
|
7207
|
-
|
7208
|
-
@property
|
7209
|
-
@abc.abstractmethod
|
7210
|
-
def group(self) -> 'ProcessGroup':
|
7211
|
-
raise NotImplementedError
|
7212
|
-
|
7213
|
-
@property
|
7214
|
-
@abc.abstractmethod
|
7215
|
-
def pid(self) -> Pid:
|
7216
|
-
raise NotImplementedError
|
7217
|
-
|
7218
|
-
#
|
7219
|
-
|
7220
|
-
@abc.abstractmethod
|
7221
|
-
def finish(self, sts: Rc) -> None:
|
7222
|
-
raise NotImplementedError
|
7223
|
-
|
7224
|
-
@abc.abstractmethod
|
7225
|
-
def stop(self) -> ta.Optional[str]:
|
7226
|
-
raise NotImplementedError
|
7227
|
-
|
7228
|
-
@abc.abstractmethod
|
7229
|
-
def give_up(self) -> None:
|
7230
|
-
raise NotImplementedError
|
7231
|
-
|
7232
|
-
@abc.abstractmethod
|
7233
|
-
def transition(self) -> None:
|
7234
|
-
raise NotImplementedError
|
7235
|
-
|
7236
|
-
@property
|
7237
|
-
@abc.abstractmethod
|
7238
|
-
def state(self) -> ProcessState:
|
7239
|
-
raise NotImplementedError
|
7240
|
-
|
7241
|
-
@abc.abstractmethod
|
7242
|
-
def after_setuid(self) -> None:
|
7243
|
-
raise NotImplementedError
|
7244
|
-
|
7245
|
-
|
7246
|
-
##
|
7247
|
-
|
7248
|
-
|
7249
|
-
class ProcessGroup(
|
7250
|
-
ConfigPriorityOrdered,
|
7251
|
-
KeyedCollectionAccessors[str, Process],
|
7252
|
-
abc.ABC,
|
7253
|
-
):
|
7254
|
-
@property
|
7255
|
-
@abc.abstractmethod
|
7256
|
-
def name(self) -> str:
|
7257
|
-
raise NotImplementedError
|
7258
|
-
|
7259
|
-
@property
|
7260
|
-
@abc.abstractmethod
|
7261
|
-
def config(self) -> ProcessGroupConfig:
|
7262
|
-
raise NotImplementedError
|
7263
|
-
|
7264
|
-
@property
|
7265
|
-
@abc.abstractmethod
|
7266
|
-
def by_name(self) -> ta.Mapping[str, Process]:
|
7267
|
-
raise NotImplementedError
|
7268
|
-
|
7269
|
-
#
|
7270
|
-
|
7271
|
-
@abc.abstractmethod
|
7272
|
-
def stop_all(self) -> None:
|
7273
|
-
raise NotImplementedError
|
7274
|
-
|
7275
|
-
@abc.abstractmethod
|
7276
|
-
def get_unstopped_processes(self) -> ta.List[Process]:
|
7277
|
-
raise NotImplementedError
|
7278
|
-
|
7279
|
-
@abc.abstractmethod
|
7280
|
-
def before_remove(self) -> None:
|
7281
|
-
raise NotImplementedError
|
7282
|
-
|
7283
|
-
|
7284
|
-
########################################
|
7285
|
-
# ../../../omlish/http/coro/fdio.py
|
7286
|
-
|
7287
|
-
|
7288
|
-
class CoroHttpServerConnectionFdioHandler(SocketFdioHandler):
|
7601
|
+
class CoroHttpServerSocketHandler(SocketHandler):
|
7289
7602
|
def __init__(
|
7290
7603
|
self,
|
7291
|
-
|
7292
|
-
|
7293
|
-
|
7604
|
+
client_address: SocketAddress,
|
7605
|
+
rfile: ta.BinaryIO,
|
7606
|
+
wfile: ta.BinaryIO,
|
7294
7607
|
*,
|
7295
|
-
|
7296
|
-
write_size: int = 0x10000,
|
7608
|
+
server_factory: CoroHttpServerFactory,
|
7297
7609
|
log_handler: ta.Optional[ta.Callable[[CoroHttpServer, CoroHttpServer.AnyLogIo], None]] = None,
|
7298
7610
|
) -> None:
|
7299
|
-
|
7300
|
-
|
7301
|
-
|
7302
|
-
|
7303
|
-
self._handler = handler
|
7304
|
-
self._read_size = read_size
|
7305
|
-
self._write_size = write_size
|
7306
|
-
self._log_handler = log_handler
|
7307
|
-
|
7308
|
-
self._read_buf = ReadableListBuffer()
|
7309
|
-
self._write_buf: IncrementalWriteBuffer | None = None
|
7310
|
-
|
7311
|
-
self._coro_srv = CoroHttpServer(
|
7312
|
-
addr,
|
7313
|
-
handler=self._handler,
|
7611
|
+
super().__init__(
|
7612
|
+
client_address,
|
7613
|
+
rfile,
|
7614
|
+
wfile,
|
7314
7615
|
)
|
7315
|
-
self._srv_coro: ta.Optional[ta.Generator[CoroHttpServer.Io, ta.Optional[bytes], None]] = self._coro_srv.coro_handle() # noqa
|
7316
7616
|
|
7317
|
-
self.
|
7318
|
-
self.
|
7617
|
+
self._server_factory = server_factory
|
7618
|
+
self._log_handler = log_handler
|
7319
7619
|
|
7320
|
-
|
7620
|
+
def handle(self) -> None:
|
7621
|
+
server = self._server_factory(self._client_address)
|
7321
7622
|
|
7322
|
-
|
7323
|
-
coro = check.not_none(self._srv_coro)
|
7623
|
+
gen = server.coro_handle()
|
7324
7624
|
|
7325
|
-
|
7326
|
-
o = self._cur_io
|
7625
|
+
o = next(gen)
|
7327
7626
|
while True:
|
7328
|
-
if o is None:
|
7329
|
-
try:
|
7330
|
-
if d is not None:
|
7331
|
-
o = coro.send(d)
|
7332
|
-
d = None
|
7333
|
-
else:
|
7334
|
-
o = next(coro)
|
7335
|
-
except StopIteration:
|
7336
|
-
self.close()
|
7337
|
-
o = None
|
7338
|
-
break
|
7339
|
-
|
7340
7627
|
if isinstance(o, CoroHttpServer.AnyLogIo):
|
7628
|
+
i = None
|
7341
7629
|
if self._log_handler is not None:
|
7342
|
-
self._log_handler(
|
7343
|
-
o = None
|
7630
|
+
self._log_handler(server, o)
|
7344
7631
|
|
7345
7632
|
elif isinstance(o, CoroHttpServer.ReadIo):
|
7346
|
-
|
7347
|
-
break
|
7348
|
-
o = None
|
7633
|
+
i = self._rfile.read(o.sz)
|
7349
7634
|
|
7350
7635
|
elif isinstance(o, CoroHttpServer.ReadLineIo):
|
7351
|
-
|
7352
|
-
break
|
7353
|
-
o = None
|
7636
|
+
i = self._rfile.readline(o.sz)
|
7354
7637
|
|
7355
7638
|
elif isinstance(o, CoroHttpServer.WriteIo):
|
7356
|
-
|
7357
|
-
self.
|
7358
|
-
|
7639
|
+
i = None
|
7640
|
+
self._wfile.write(o.data)
|
7641
|
+
self._wfile.flush()
|
7359
7642
|
|
7360
7643
|
else:
|
7361
7644
|
raise TypeError(o)
|
7362
7645
|
|
7363
|
-
|
7364
|
-
|
7365
|
-
|
7366
|
-
|
7367
|
-
|
7368
|
-
|
7369
|
-
|
7370
|
-
def writable(self) -> bool:
|
7371
|
-
return self._write_buf is not None
|
7372
|
-
|
7373
|
-
#
|
7374
|
-
|
7375
|
-
def on_readable(self) -> None:
|
7376
|
-
try:
|
7377
|
-
buf = check.not_none(self._sock).recv(self._read_size)
|
7378
|
-
except BlockingIOError:
|
7379
|
-
return
|
7380
|
-
except ConnectionResetError:
|
7381
|
-
self.close()
|
7382
|
-
return
|
7383
|
-
if not buf:
|
7384
|
-
self.close()
|
7385
|
-
return
|
7386
|
-
|
7387
|
-
self._read_buf.feed(buf)
|
7388
|
-
|
7389
|
-
if isinstance(self._cur_io, CoroHttpServer.AnyReadIo):
|
7390
|
-
self._next_io()
|
7391
|
-
|
7392
|
-
def on_writable(self) -> None:
|
7393
|
-
check.isinstance(self._cur_io, CoroHttpServer.WriteIo)
|
7394
|
-
wb = check.not_none(self._write_buf)
|
7395
|
-
while wb.rem > 0:
|
7396
|
-
def send(d: bytes) -> int:
|
7397
|
-
try:
|
7398
|
-
return check.not_none(self._sock).send(d)
|
7399
|
-
except ConnectionResetError:
|
7400
|
-
self.close()
|
7401
|
-
return 0
|
7402
|
-
except BlockingIOError:
|
7403
|
-
return 0
|
7404
|
-
if not wb.write(send):
|
7646
|
+
try:
|
7647
|
+
if i is not None:
|
7648
|
+
o = gen.send(i)
|
7649
|
+
else:
|
7650
|
+
o = next(gen)
|
7651
|
+
except StopIteration:
|
7405
7652
|
break
|
7406
7653
|
|
7407
|
-
if wb.rem < 1:
|
7408
|
-
self._write_buf = None
|
7409
|
-
self._cur_io = None
|
7410
|
-
self._next_io()
|
7411
|
-
|
7412
7654
|
|
7413
7655
|
########################################
|
7414
7656
|
# ../dispatchers.py
|
@@ -8110,6 +8352,135 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
8110
8352
|
os.umask(self._config.umask)
|
8111
8353
|
|
8112
8354
|
|
8355
|
+
########################################
|
8356
|
+
# ../../../omlish/http/coro/fdio.py
|
8357
|
+
|
8358
|
+
|
8359
|
+
class CoroHttpServerConnectionFdioHandler(SocketFdioHandler):
|
8360
|
+
def __init__(
|
8361
|
+
self,
|
8362
|
+
addr: SocketAddress,
|
8363
|
+
sock: socket.socket,
|
8364
|
+
handler: HttpHandler,
|
8365
|
+
*,
|
8366
|
+
read_size: int = 0x10000,
|
8367
|
+
write_size: int = 0x10000,
|
8368
|
+
log_handler: ta.Optional[ta.Callable[[CoroHttpServer, CoroHttpServer.AnyLogIo], None]] = None,
|
8369
|
+
) -> None:
|
8370
|
+
check.state(not sock.getblocking())
|
8371
|
+
|
8372
|
+
super().__init__(addr, sock)
|
8373
|
+
|
8374
|
+
self._handler = handler
|
8375
|
+
self._read_size = read_size
|
8376
|
+
self._write_size = write_size
|
8377
|
+
self._log_handler = log_handler
|
8378
|
+
|
8379
|
+
self._read_buf = ReadableListBuffer()
|
8380
|
+
self._write_buf: IncrementalWriteBuffer | None = None
|
8381
|
+
|
8382
|
+
self._coro_srv = CoroHttpServer(
|
8383
|
+
addr,
|
8384
|
+
handler=self._handler,
|
8385
|
+
)
|
8386
|
+
self._srv_coro: ta.Optional[ta.Generator[CoroHttpServer.Io, ta.Optional[bytes], None]] = self._coro_srv.coro_handle() # noqa
|
8387
|
+
|
8388
|
+
self._cur_io: CoroHttpServer.Io | None = None
|
8389
|
+
self._next_io()
|
8390
|
+
|
8391
|
+
#
|
8392
|
+
|
8393
|
+
def _next_io(self) -> None: # noqa
|
8394
|
+
coro = check.not_none(self._srv_coro)
|
8395
|
+
|
8396
|
+
d: bytes | None = None
|
8397
|
+
o = self._cur_io
|
8398
|
+
while True:
|
8399
|
+
if o is None:
|
8400
|
+
try:
|
8401
|
+
if d is not None:
|
8402
|
+
o = coro.send(d)
|
8403
|
+
d = None
|
8404
|
+
else:
|
8405
|
+
o = next(coro)
|
8406
|
+
except StopIteration:
|
8407
|
+
self.close()
|
8408
|
+
o = None
|
8409
|
+
break
|
8410
|
+
|
8411
|
+
if isinstance(o, CoroHttpServer.AnyLogIo):
|
8412
|
+
if self._log_handler is not None:
|
8413
|
+
self._log_handler(self._coro_srv, o)
|
8414
|
+
o = None
|
8415
|
+
|
8416
|
+
elif isinstance(o, CoroHttpServer.ReadIo):
|
8417
|
+
if (d := self._read_buf.read(o.sz)) is None:
|
8418
|
+
break
|
8419
|
+
o = None
|
8420
|
+
|
8421
|
+
elif isinstance(o, CoroHttpServer.ReadLineIo):
|
8422
|
+
if (d := self._read_buf.read_until(b'\n')) is None:
|
8423
|
+
break
|
8424
|
+
o = None
|
8425
|
+
|
8426
|
+
elif isinstance(o, CoroHttpServer.WriteIo):
|
8427
|
+
check.none(self._write_buf)
|
8428
|
+
self._write_buf = IncrementalWriteBuffer(o.data, write_size=self._write_size)
|
8429
|
+
break
|
8430
|
+
|
8431
|
+
else:
|
8432
|
+
raise TypeError(o)
|
8433
|
+
|
8434
|
+
self._cur_io = o
|
8435
|
+
|
8436
|
+
#
|
8437
|
+
|
8438
|
+
def readable(self) -> bool:
|
8439
|
+
return True
|
8440
|
+
|
8441
|
+
def writable(self) -> bool:
|
8442
|
+
return self._write_buf is not None
|
8443
|
+
|
8444
|
+
#
|
8445
|
+
|
8446
|
+
def on_readable(self) -> None:
|
8447
|
+
try:
|
8448
|
+
buf = check.not_none(self._sock).recv(self._read_size)
|
8449
|
+
except BlockingIOError:
|
8450
|
+
return
|
8451
|
+
except ConnectionResetError:
|
8452
|
+
self.close()
|
8453
|
+
return
|
8454
|
+
if not buf:
|
8455
|
+
self.close()
|
8456
|
+
return
|
8457
|
+
|
8458
|
+
self._read_buf.feed(buf)
|
8459
|
+
|
8460
|
+
if isinstance(self._cur_io, CoroHttpServer.AnyReadIo):
|
8461
|
+
self._next_io()
|
8462
|
+
|
8463
|
+
def on_writable(self) -> None:
|
8464
|
+
check.isinstance(self._cur_io, CoroHttpServer.WriteIo)
|
8465
|
+
wb = check.not_none(self._write_buf)
|
8466
|
+
while wb.rem > 0:
|
8467
|
+
def send(d: bytes) -> int:
|
8468
|
+
try:
|
8469
|
+
return check.not_none(self._sock).send(d)
|
8470
|
+
except ConnectionResetError:
|
8471
|
+
self.close()
|
8472
|
+
return 0
|
8473
|
+
except BlockingIOError:
|
8474
|
+
return 0
|
8475
|
+
if not wb.write(send):
|
8476
|
+
break
|
8477
|
+
|
8478
|
+
if wb.rem < 1:
|
8479
|
+
self._write_buf = None
|
8480
|
+
self._cur_io = None
|
8481
|
+
self._next_io()
|
8482
|
+
|
8483
|
+
|
8113
8484
|
########################################
|
8114
8485
|
# ../groups.py
|
8115
8486
|
|
@@ -9685,7 +10056,7 @@ def main(
|
|
9685
10056
|
|
9686
10057
|
# if we hup, restart by making a new Supervisor()
|
9687
10058
|
for epoch in itertools.count():
|
9688
|
-
config =
|
10059
|
+
config = load_config_file_obj(
|
9689
10060
|
os.path.expanduser(cf),
|
9690
10061
|
ServerConfig,
|
9691
10062
|
prepare=prepare_server_config,
|