ominfra 0.0.0.dev192__py3-none-any.whl → 0.0.0.dev194__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/clouds/aws/journald2aws/main.py +5 -2
- ominfra/manage/deploy/conf/manager.py +4 -4
- ominfra/manage/deploy/conf/specs.py +2 -2
- ominfra/manage/main.py +3 -3
- ominfra/scripts/journald2aws.py +448 -122
- ominfra/scripts/manage.py +2019 -1700
- ominfra/scripts/supervisor.py +2113 -1742
- ominfra/supervisor/configs.py +4 -3
- ominfra/supervisor/main.py +2 -2
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/METADATA +4 -4
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/RECORD +15 -16
- ominfra/configs.py +0 -129
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev192.dist-info → ominfra-0.0.0.dev194.dist-info}/top_level.txt +0 -0
ominfra/scripts/supervisor.py
CHANGED
@@ -36,6 +36,7 @@ import abc
|
|
36
36
|
import base64
|
37
37
|
import collections
|
38
38
|
import collections.abc
|
39
|
+
import configparser
|
39
40
|
import contextlib
|
40
41
|
import contextvars
|
41
42
|
import ctypes as ct
|
@@ -79,7 +80,7 @@ import types
|
|
79
80
|
import typing as ta
|
80
81
|
import uuid
|
81
82
|
import warnings
|
82
|
-
import weakref
|
83
|
+
import weakref
|
83
84
|
|
84
85
|
|
85
86
|
########################################
|
@@ -92,15 +93,21 @@ if sys.version_info < (3, 8):
|
|
92
93
|
########################################
|
93
94
|
|
94
95
|
|
95
|
-
# ../../omdev/toml/parser.py
|
96
|
-
TomlParseFloat = ta.Callable[[str], ta.Any]
|
97
|
-
TomlKey = ta.Tuple[str, ...]
|
98
|
-
TomlPos = int # ta.TypeAlias
|
99
|
-
|
100
96
|
# utils/collections.py
|
101
97
|
K = ta.TypeVar('K')
|
102
98
|
V = ta.TypeVar('V')
|
103
99
|
|
100
|
+
# ../../omlish/configs/types.py
|
101
|
+
ConfigMap = ta.Mapping[str, ta.Any]
|
102
|
+
|
103
|
+
# ../../omlish/formats/ini/sections.py
|
104
|
+
IniSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
|
105
|
+
|
106
|
+
# ../../omlish/formats/toml/parser.py
|
107
|
+
TomlParseFloat = ta.Callable[[str], ta.Any]
|
108
|
+
TomlKey = ta.Tuple[str, ...]
|
109
|
+
TomlPos = int # ta.TypeAlias
|
110
|
+
|
104
111
|
# ../../omlish/lite/cached.py
|
105
112
|
T = ta.TypeVar('T')
|
106
113
|
CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
@@ -125,6 +132,9 @@ SocketAddress = ta.Any
|
|
125
132
|
EventCallback = ta.Callable[['Event'], None]
|
126
133
|
ProcessOutputChannel = ta.Literal['stdout', 'stderr'] # ta.TypeAlias
|
127
134
|
|
135
|
+
# ../../omlish/configs/formats.py
|
136
|
+
ConfigDataT = ta.TypeVar('ConfigDataT', bound='ConfigData')
|
137
|
+
|
128
138
|
# ../../omlish/http/parsing.py
|
129
139
|
HttpHeaders = http.client.HTTPMessage # ta.TypeAlias
|
130
140
|
|
@@ -141,10 +151,6 @@ InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
|
|
141
151
|
# ../../omlish/sockets/handlers.py
|
142
152
|
SocketHandlerFactory = ta.Callable[[SocketAddress, ta.BinaryIO, ta.BinaryIO], 'SocketHandler']
|
143
153
|
|
144
|
-
# ../configs.py
|
145
|
-
ConfigMapping = ta.Mapping[str, ta.Any]
|
146
|
-
IniConfigSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]] # ta.TypeAlias
|
147
|
-
|
148
154
|
# ../../omlish/http/handlers.py
|
149
155
|
HttpHandler = ta.Callable[['HttpHandlerRequest'], 'HttpHandlerResponse'] # ta.TypeAlias
|
150
156
|
|
@@ -153,1291 +159,1462 @@ CoroHttpServerFactory = ta.Callable[[SocketAddress], 'CoroHttpServer']
|
|
153
159
|
|
154
160
|
|
155
161
|
########################################
|
156
|
-
#
|
157
|
-
# SPDX-License-Identifier: MIT
|
158
|
-
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
159
|
-
# Licensed to PSF under a Contributor Agreement.
|
160
|
-
#
|
161
|
-
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
162
|
-
# --------------------------------------------
|
163
|
-
#
|
164
|
-
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
165
|
-
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
166
|
-
# documentation.
|
167
|
-
#
|
168
|
-
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
169
|
-
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
170
|
-
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
171
|
-
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
172
|
-
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
173
|
-
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
174
|
-
#
|
175
|
-
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
176
|
-
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
177
|
-
# any such work a brief summary of the changes made to Python.
|
178
|
-
#
|
179
|
-
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
180
|
-
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
181
|
-
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
182
|
-
# RIGHTS.
|
183
|
-
#
|
184
|
-
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
185
|
-
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
186
|
-
# ADVISED OF THE POSSIBILITY THEREOF.
|
187
|
-
#
|
188
|
-
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
189
|
-
#
|
190
|
-
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
191
|
-
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
192
|
-
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
193
|
-
#
|
194
|
-
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
195
|
-
# License Agreement.
|
196
|
-
#
|
197
|
-
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
162
|
+
# ../exceptions.py
|
198
163
|
|
199
164
|
|
200
|
-
|
165
|
+
class ProcessError(Exception):
|
166
|
+
"""Specialized exceptions used when attempting to start a process."""
|
201
167
|
|
202
168
|
|
203
|
-
|
169
|
+
class BadCommandError(ProcessError):
|
170
|
+
"""Indicates the command could not be parsed properly."""
|
204
171
|
|
205
|
-
TOML_RE_NUMBER = re.compile(
|
206
|
-
r"""
|
207
|
-
0
|
208
|
-
(?:
|
209
|
-
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
210
|
-
|
|
211
|
-
b[01](?:_?[01])* # bin
|
212
|
-
|
|
213
|
-
o[0-7](?:_?[0-7])* # oct
|
214
|
-
)
|
215
|
-
|
|
216
|
-
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
217
|
-
(?P<floatpart>
|
218
|
-
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
219
|
-
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
220
|
-
)
|
221
|
-
""",
|
222
|
-
flags=re.VERBOSE,
|
223
|
-
)
|
224
|
-
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
225
|
-
TOML_RE_DATETIME = re.compile(
|
226
|
-
rf"""
|
227
|
-
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
228
|
-
(?:
|
229
|
-
[Tt ]
|
230
|
-
{_TOML_TIME_RE_STR}
|
231
|
-
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
232
|
-
)?
|
233
|
-
""",
|
234
|
-
flags=re.VERBOSE,
|
235
|
-
)
|
236
172
|
|
173
|
+
class NotExecutableError(ProcessError):
|
174
|
+
"""
|
175
|
+
Indicates that the filespec cannot be executed because its path resolves to a file which is not executable, or which
|
176
|
+
is a directory.
|
177
|
+
"""
|
237
178
|
|
238
|
-
def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
|
239
|
-
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
240
179
|
|
241
|
-
|
242
|
-
"""
|
243
|
-
(
|
244
|
-
year_str,
|
245
|
-
month_str,
|
246
|
-
day_str,
|
247
|
-
hour_str,
|
248
|
-
minute_str,
|
249
|
-
sec_str,
|
250
|
-
micros_str,
|
251
|
-
zulu_time,
|
252
|
-
offset_sign_str,
|
253
|
-
offset_hour_str,
|
254
|
-
offset_minute_str,
|
255
|
-
) = match.groups()
|
256
|
-
year, month, day = int(year_str), int(month_str), int(day_str)
|
257
|
-
if hour_str is None:
|
258
|
-
return datetime.date(year, month, day)
|
259
|
-
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
260
|
-
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
261
|
-
if offset_sign_str:
|
262
|
-
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
263
|
-
offset_hour_str, offset_minute_str, offset_sign_str,
|
264
|
-
)
|
265
|
-
elif zulu_time:
|
266
|
-
tz = datetime.UTC
|
267
|
-
else: # local date-time
|
268
|
-
tz = None
|
269
|
-
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
180
|
+
class NotFoundError(ProcessError):
|
181
|
+
"""Indicates that the filespec cannot be executed because it could not be found."""
|
270
182
|
|
271
183
|
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
hours=sign * int(hour_str),
|
278
|
-
minutes=sign * int(minute_str),
|
279
|
-
),
|
280
|
-
)
|
184
|
+
class NoPermissionError(ProcessError):
|
185
|
+
"""
|
186
|
+
Indicates that the file cannot be executed because the supervisor process does not possess the appropriate UNIX
|
187
|
+
filesystem permission to execute the file.
|
188
|
+
"""
|
281
189
|
|
282
190
|
|
283
|
-
|
284
|
-
|
285
|
-
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
286
|
-
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
191
|
+
########################################
|
192
|
+
# ../privileges.py
|
287
193
|
|
288
194
|
|
289
|
-
def
|
290
|
-
|
291
|
-
|
292
|
-
|
195
|
+
def drop_privileges(user: ta.Union[int, str, None]) -> ta.Optional[str]:
|
196
|
+
"""
|
197
|
+
Drop privileges to become the specified user, which may be a username or uid. Called for supervisord startup and
|
198
|
+
when spawning subprocesses. Returns None on success or a string error message if privileges could not be dropped.
|
199
|
+
"""
|
293
200
|
|
201
|
+
if user is None:
|
202
|
+
return 'No user specified to setuid to!'
|
294
203
|
|
295
|
-
|
204
|
+
# get uid for user, which can be a number or username
|
205
|
+
try:
|
206
|
+
uid = int(user)
|
207
|
+
except ValueError:
|
208
|
+
try:
|
209
|
+
pwrec = pwd.getpwnam(user) # type: ignore
|
210
|
+
except KeyError:
|
211
|
+
return f"Can't find username {user!r}"
|
212
|
+
uid = pwrec[2]
|
213
|
+
else:
|
214
|
+
try:
|
215
|
+
pwrec = pwd.getpwuid(uid)
|
216
|
+
except KeyError:
|
217
|
+
return f"Can't find uid {uid!r}"
|
296
218
|
|
297
|
-
|
298
|
-
# functions.
|
299
|
-
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
300
|
-
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
219
|
+
current_uid = os.getuid()
|
301
220
|
|
302
|
-
|
303
|
-
|
221
|
+
if current_uid == uid:
|
222
|
+
# do nothing and return successfully if the uid is already the current one. this allows a supervisord running as
|
223
|
+
# an unprivileged user "foo" to start a process where the config has "user=foo" (same user) in it.
|
224
|
+
return None
|
304
225
|
|
305
|
-
|
226
|
+
if current_uid != 0:
|
227
|
+
return "Can't drop privilege as nonroot user"
|
306
228
|
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
229
|
+
gid = pwrec[3]
|
230
|
+
if hasattr(os, 'setgroups'):
|
231
|
+
user = pwrec[0]
|
232
|
+
groups = [grprec[2] for grprec in grp.getgrall() if user in grprec[3]]
|
312
233
|
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
'\\\\': '\u005C', # backslash
|
322
|
-
},
|
323
|
-
)
|
234
|
+
# always put our primary gid first in this list, otherwise we can lose group info since sometimes the first
|
235
|
+
# group in the setgroups list gets overwritten on the subsequent setgid call (at least on freebsd 9 with
|
236
|
+
# python 2.7 - this will be safe though for all unix /python version combos)
|
237
|
+
groups.insert(0, gid)
|
238
|
+
try:
|
239
|
+
os.setgroups(groups)
|
240
|
+
except OSError:
|
241
|
+
return 'Could not set groups of effective user'
|
324
242
|
|
243
|
+
try:
|
244
|
+
os.setgid(gid)
|
245
|
+
except OSError:
|
246
|
+
return 'Could not set group id of effective user'
|
325
247
|
|
326
|
-
|
327
|
-
"""An error raised if a document is not valid TOML."""
|
248
|
+
os.setuid(uid)
|
328
249
|
|
250
|
+
return None
|
329
251
|
|
330
|
-
def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
|
331
|
-
"""Parse TOML from a binary file object."""
|
332
|
-
b = fp.read()
|
333
|
-
try:
|
334
|
-
s = b.decode()
|
335
|
-
except AttributeError:
|
336
|
-
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
337
|
-
return toml_loads(s, parse_float=parse_float)
|
338
252
|
|
253
|
+
########################################
|
254
|
+
# ../states.py
|
339
255
|
|
340
|
-
def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
|
341
|
-
"""Parse TOML from a string."""
|
342
256
|
|
343
|
-
|
344
|
-
try:
|
345
|
-
src = s.replace('\r\n', '\n')
|
346
|
-
except (AttributeError, TypeError):
|
347
|
-
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
348
|
-
pos = 0
|
349
|
-
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
350
|
-
header: TomlKey = ()
|
351
|
-
parse_float = toml_make_safe_parse_float(parse_float)
|
257
|
+
##
|
352
258
|
|
353
|
-
# Parse one statement at a time (typically means one line in TOML source)
|
354
|
-
while True:
|
355
|
-
# 1. Skip line leading whitespace
|
356
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
357
259
|
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
char = src[pos]
|
368
|
-
except IndexError:
|
369
|
-
break
|
370
|
-
if char == '\n':
|
371
|
-
pos += 1
|
372
|
-
continue
|
373
|
-
if char in TOML_KEY_INITIAL_CHARS:
|
374
|
-
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
375
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
376
|
-
elif char == '[':
|
377
|
-
try:
|
378
|
-
second_char: ta.Optional[str] = src[pos + 1]
|
379
|
-
except IndexError:
|
380
|
-
second_char = None
|
381
|
-
out.flags.finalize_pending()
|
382
|
-
if second_char == '[':
|
383
|
-
pos, header = toml_create_list_rule(src, pos, out)
|
384
|
-
else:
|
385
|
-
pos, header = toml_create_dict_rule(src, pos, out)
|
386
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
387
|
-
elif char != '#':
|
388
|
-
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
260
|
+
class ProcessState(enum.IntEnum):
|
261
|
+
STOPPED = 0
|
262
|
+
STARTING = 10
|
263
|
+
RUNNING = 20
|
264
|
+
BACKOFF = 30
|
265
|
+
STOPPING = 40
|
266
|
+
EXITED = 100
|
267
|
+
FATAL = 200
|
268
|
+
UNKNOWN = 1000
|
389
269
|
|
390
|
-
|
391
|
-
|
270
|
+
@property
|
271
|
+
def stopped(self) -> bool:
|
272
|
+
return self in STOPPED_STATES
|
392
273
|
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
except IndexError:
|
397
|
-
break
|
398
|
-
if char != '\n':
|
399
|
-
raise toml_suffixed_err(
|
400
|
-
src, pos, 'Expected newline or end of document after a statement',
|
401
|
-
)
|
402
|
-
pos += 1
|
274
|
+
@property
|
275
|
+
def running(self) -> bool:
|
276
|
+
return self in RUNNING_STATES
|
403
277
|
|
404
|
-
|
278
|
+
@property
|
279
|
+
def signalable(self) -> bool:
|
280
|
+
return self in SIGNALABLE_STATES
|
405
281
|
|
406
282
|
|
407
|
-
|
408
|
-
|
283
|
+
# http://supervisord.org/subprocess.html
|
284
|
+
STATE_TRANSITIONS = {
|
285
|
+
ProcessState.STOPPED: (ProcessState.STARTING,),
|
286
|
+
ProcessState.STARTING: (ProcessState.RUNNING, ProcessState.BACKOFF, ProcessState.STOPPING),
|
287
|
+
ProcessState.RUNNING: (ProcessState.STOPPING, ProcessState.EXITED),
|
288
|
+
ProcessState.BACKOFF: (ProcessState.STARTING, ProcessState.FATAL),
|
289
|
+
ProcessState.STOPPING: (ProcessState.STOPPED,),
|
290
|
+
ProcessState.EXITED: (ProcessState.STARTING,),
|
291
|
+
ProcessState.FATAL: (ProcessState.STARTING,),
|
292
|
+
}
|
409
293
|
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
294
|
+
STOPPED_STATES = (
|
295
|
+
ProcessState.STOPPED,
|
296
|
+
ProcessState.EXITED,
|
297
|
+
ProcessState.FATAL,
|
298
|
+
ProcessState.UNKNOWN,
|
299
|
+
)
|
414
300
|
|
415
|
-
|
416
|
-
|
417
|
-
|
301
|
+
RUNNING_STATES = (
|
302
|
+
ProcessState.RUNNING,
|
303
|
+
ProcessState.BACKOFF,
|
304
|
+
ProcessState.STARTING,
|
305
|
+
)
|
418
306
|
|
419
|
-
|
420
|
-
|
307
|
+
SIGNALABLE_STATES = (
|
308
|
+
ProcessState.RUNNING,
|
309
|
+
ProcessState.STARTING,
|
310
|
+
ProcessState.STOPPING,
|
311
|
+
)
|
421
312
|
|
422
|
-
def finalize_pending(self) -> None:
|
423
|
-
for key, flag in self._pending_flags:
|
424
|
-
self.set(key, flag, recursive=False)
|
425
|
-
self._pending_flags.clear()
|
426
313
|
|
427
|
-
|
428
|
-
cont = self._flags
|
429
|
-
for k in key[:-1]:
|
430
|
-
if k not in cont:
|
431
|
-
return
|
432
|
-
cont = cont[k]['nested']
|
433
|
-
cont.pop(key[-1], None)
|
314
|
+
##
|
434
315
|
|
435
|
-
def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
|
436
|
-
cont = self._flags
|
437
|
-
key_parent, key_stem = key[:-1], key[-1]
|
438
|
-
for k in key_parent:
|
439
|
-
if k not in cont:
|
440
|
-
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
441
|
-
cont = cont[k]['nested']
|
442
|
-
if key_stem not in cont:
|
443
|
-
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
444
|
-
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
445
316
|
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
if k not in cont:
|
452
|
-
return False
|
453
|
-
inner_cont = cont[k]
|
454
|
-
if flag in inner_cont['recursive_flags']:
|
455
|
-
return True
|
456
|
-
cont = inner_cont['nested']
|
457
|
-
key_stem = key[-1]
|
458
|
-
if key_stem in cont:
|
459
|
-
cont = cont[key_stem]
|
460
|
-
return flag in cont['flags'] or flag in cont['recursive_flags']
|
461
|
-
return False
|
317
|
+
class SupervisorState(enum.IntEnum):
|
318
|
+
FATAL = 2
|
319
|
+
RUNNING = 1
|
320
|
+
RESTARTING = 0
|
321
|
+
SHUTDOWN = -1
|
462
322
|
|
463
323
|
|
464
|
-
|
465
|
-
|
466
|
-
# The parsed content of the TOML document
|
467
|
-
self.dict: ta.Dict[str, ta.Any] = {}
|
324
|
+
########################################
|
325
|
+
# ../utils/collections.py
|
468
326
|
|
469
|
-
def get_or_create_nest(
|
470
|
-
self,
|
471
|
-
key: TomlKey,
|
472
|
-
*,
|
473
|
-
access_lists: bool = True,
|
474
|
-
) -> dict:
|
475
|
-
cont: ta.Any = self.dict
|
476
|
-
for k in key:
|
477
|
-
if k not in cont:
|
478
|
-
cont[k] = {}
|
479
|
-
cont = cont[k]
|
480
|
-
if access_lists and isinstance(cont, list):
|
481
|
-
cont = cont[-1]
|
482
|
-
if not isinstance(cont, dict):
|
483
|
-
raise KeyError('There is no nest behind this key')
|
484
|
-
return cont
|
485
327
|
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
if not isinstance(list_, list):
|
492
|
-
raise KeyError('An object other than list found behind this key')
|
493
|
-
list_.append({})
|
494
|
-
else:
|
495
|
-
cont[last_key] = [{}]
|
328
|
+
class KeyedCollectionAccessors(abc.ABC, ta.Generic[K, V]):
|
329
|
+
@property
|
330
|
+
@abc.abstractmethod
|
331
|
+
def _by_key(self) -> ta.Mapping[K, V]:
|
332
|
+
raise NotImplementedError
|
496
333
|
|
334
|
+
def __iter__(self) -> ta.Iterator[V]:
|
335
|
+
return iter(self._by_key.values())
|
497
336
|
|
498
|
-
|
499
|
-
|
500
|
-
flags: TomlFlags
|
337
|
+
def __len__(self) -> int:
|
338
|
+
return len(self._by_key)
|
501
339
|
|
340
|
+
def __contains__(self, key: K) -> bool:
|
341
|
+
return key in self._by_key
|
502
342
|
|
503
|
-
def
|
504
|
-
|
505
|
-
while src[pos] in chars:
|
506
|
-
pos += 1
|
507
|
-
except IndexError:
|
508
|
-
pass
|
509
|
-
return pos
|
343
|
+
def __getitem__(self, key: K) -> V:
|
344
|
+
return self._by_key[key]
|
510
345
|
|
346
|
+
def get(self, key: K, default: ta.Optional[V] = None) -> ta.Optional[V]:
|
347
|
+
return self._by_key.get(key, default)
|
511
348
|
|
512
|
-
def
|
513
|
-
|
514
|
-
pos: TomlPos,
|
515
|
-
expect: str,
|
516
|
-
*,
|
517
|
-
error_on: ta.FrozenSet[str],
|
518
|
-
error_on_eof: bool,
|
519
|
-
) -> TomlPos:
|
520
|
-
try:
|
521
|
-
new_pos = src.index(expect, pos)
|
522
|
-
except ValueError:
|
523
|
-
new_pos = len(src)
|
524
|
-
if error_on_eof:
|
525
|
-
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
349
|
+
def items(self) -> ta.Iterator[ta.Tuple[K, V]]:
|
350
|
+
return iter(self._by_key.items())
|
526
351
|
|
527
|
-
if not error_on.isdisjoint(src[pos:new_pos]):
|
528
|
-
while src[pos] not in error_on:
|
529
|
-
pos += 1
|
530
|
-
raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
|
531
|
-
return new_pos
|
532
352
|
|
353
|
+
class KeyedCollection(KeyedCollectionAccessors[K, V]):
|
354
|
+
def __init__(self, items: ta.Iterable[V]) -> None:
|
355
|
+
super().__init__()
|
533
356
|
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
return toml_skip_until(
|
541
|
-
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
542
|
-
)
|
543
|
-
return pos
|
357
|
+
by_key: ta.Dict[K, V] = {}
|
358
|
+
for v in items:
|
359
|
+
if (k := self._key(v)) in by_key:
|
360
|
+
raise KeyError(f'key {k} of {v} already registered by {by_key[k]}')
|
361
|
+
by_key[k] = v
|
362
|
+
self.__by_key = by_key
|
544
363
|
|
364
|
+
@property
|
365
|
+
def _by_key(self) -> ta.Mapping[K, V]:
|
366
|
+
return self.__by_key
|
545
367
|
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
550
|
-
pos = toml_skip_comment(src, pos)
|
551
|
-
if pos == pos_before_skip:
|
552
|
-
return pos
|
368
|
+
@abc.abstractmethod
|
369
|
+
def _key(self, v: V) -> K:
|
370
|
+
raise NotImplementedError
|
553
371
|
|
554
372
|
|
555
|
-
|
556
|
-
|
557
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
558
|
-
pos, key = toml_parse_key(src, pos)
|
373
|
+
########################################
|
374
|
+
# ../utils/diag.py
|
559
375
|
|
560
|
-
if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
|
561
|
-
raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
|
562
|
-
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
563
|
-
try:
|
564
|
-
out.data.get_or_create_nest(key)
|
565
|
-
except KeyError:
|
566
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
567
376
|
|
568
|
-
|
569
|
-
|
570
|
-
|
377
|
+
def compact_traceback() -> ta.Tuple[
|
378
|
+
ta.Tuple[str, str, int],
|
379
|
+
ta.Type[BaseException],
|
380
|
+
BaseException,
|
381
|
+
types.TracebackType,
|
382
|
+
]:
|
383
|
+
t, v, tb = sys.exc_info()
|
384
|
+
if not tb:
|
385
|
+
raise RuntimeError('No traceback')
|
571
386
|
|
387
|
+
tbinfo = []
|
388
|
+
while tb:
|
389
|
+
tbinfo.append((
|
390
|
+
tb.tb_frame.f_code.co_filename,
|
391
|
+
tb.tb_frame.f_code.co_name,
|
392
|
+
str(tb.tb_lineno),
|
393
|
+
))
|
394
|
+
tb = tb.tb_next
|
572
395
|
|
573
|
-
|
574
|
-
|
575
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
576
|
-
pos, key = toml_parse_key(src, pos)
|
396
|
+
# just to be safe
|
397
|
+
del tb
|
577
398
|
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
out.flags.unset_all(key)
|
582
|
-
# ...but this key precisely is still prohibited from table declaration
|
583
|
-
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
584
|
-
try:
|
585
|
-
out.data.append_nest_to_list(key)
|
586
|
-
except KeyError:
|
587
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
399
|
+
file, function, line = tbinfo[-1]
|
400
|
+
info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo]) # noqa
|
401
|
+
return (file, function, line), t, v, info # type: ignore
|
588
402
|
|
589
|
-
if not src.startswith(']]', pos):
|
590
|
-
raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
|
591
|
-
return pos + 2, key
|
592
403
|
|
404
|
+
########################################
|
405
|
+
# ../utils/fs.py
|
593
406
|
|
594
|
-
def toml_key_value_rule(
|
595
|
-
src: str,
|
596
|
-
pos: TomlPos,
|
597
|
-
out: TomlOutput,
|
598
|
-
header: TomlKey,
|
599
|
-
parse_float: TomlParseFloat,
|
600
|
-
) -> TomlPos:
|
601
|
-
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
602
|
-
key_parent, key_stem = key[:-1], key[-1]
|
603
|
-
abs_key_parent = header + key_parent
|
604
407
|
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
# table sections.
|
612
|
-
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
408
|
+
def try_unlink(path: str) -> bool:
|
409
|
+
try:
|
410
|
+
os.unlink(path)
|
411
|
+
except OSError:
|
412
|
+
return False
|
413
|
+
return True
|
613
414
|
|
614
|
-
if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
|
615
|
-
raise toml_suffixed_err(
|
616
|
-
src,
|
617
|
-
pos,
|
618
|
-
f'Cannot mutate immutable namespace {abs_key_parent}',
|
619
|
-
)
|
620
415
|
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
if key_stem in nest:
|
626
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
627
|
-
# Mark inline table and array namespaces recursively immutable
|
628
|
-
if isinstance(value, (dict, list)):
|
629
|
-
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
630
|
-
nest[key_stem] = value
|
631
|
-
return pos
|
416
|
+
def mktempfile(suffix: str, prefix: str, dir: str) -> str: # noqa
|
417
|
+
fd, filename = tempfile.mkstemp(suffix, prefix, dir)
|
418
|
+
os.close(fd)
|
419
|
+
return filename
|
632
420
|
|
633
421
|
|
634
|
-
def
|
635
|
-
|
636
|
-
pos: TomlPos,
|
637
|
-
parse_float: TomlParseFloat,
|
638
|
-
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
639
|
-
pos, key = toml_parse_key(src, pos)
|
640
|
-
try:
|
641
|
-
char: ta.Optional[str] = src[pos]
|
642
|
-
except IndexError:
|
643
|
-
char = None
|
644
|
-
if char != '=':
|
645
|
-
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
646
|
-
pos += 1
|
647
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
648
|
-
pos, value = toml_parse_value(src, pos, parse_float)
|
649
|
-
return pos, key, value
|
422
|
+
def get_path() -> ta.Sequence[str]:
|
423
|
+
"""Return a list corresponding to $PATH, or a default."""
|
650
424
|
|
425
|
+
path = ['/bin', '/usr/bin', '/usr/local/bin']
|
426
|
+
if 'PATH' in os.environ:
|
427
|
+
p = os.environ['PATH']
|
428
|
+
if p:
|
429
|
+
path = p.split(os.pathsep)
|
430
|
+
return path
|
651
431
|
|
652
|
-
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
653
|
-
pos, key_part = toml_parse_key_part(src, pos)
|
654
|
-
key: TomlKey = (key_part,)
|
655
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
656
|
-
while True:
|
657
|
-
try:
|
658
|
-
char: ta.Optional[str] = src[pos]
|
659
|
-
except IndexError:
|
660
|
-
char = None
|
661
|
-
if char != '.':
|
662
|
-
return pos, key
|
663
|
-
pos += 1
|
664
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
665
|
-
pos, key_part = toml_parse_key_part(src, pos)
|
666
|
-
key += (key_part,)
|
667
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
668
432
|
|
433
|
+
def check_existing_dir(v: str) -> str:
|
434
|
+
nv = os.path.expanduser(v)
|
435
|
+
if os.path.isdir(nv):
|
436
|
+
return nv
|
437
|
+
raise ValueError(f'{v} is not an existing directory')
|
669
438
|
|
670
|
-
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
671
|
-
try:
|
672
|
-
char: ta.Optional[str] = src[pos]
|
673
|
-
except IndexError:
|
674
|
-
char = None
|
675
|
-
if char in TOML_BARE_KEY_CHARS:
|
676
|
-
start_pos = pos
|
677
|
-
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
678
|
-
return pos, src[start_pos:pos]
|
679
|
-
if char == "'":
|
680
|
-
return toml_parse_literal_str(src, pos)
|
681
|
-
if char == '"':
|
682
|
-
return toml_parse_one_line_basic_str(src, pos)
|
683
|
-
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
684
439
|
|
440
|
+
def check_path_with_existing_dir(v: str) -> str:
|
441
|
+
nv = os.path.expanduser(v)
|
442
|
+
dir = os.path.dirname(nv) # noqa
|
443
|
+
if not dir:
|
444
|
+
# relative pathname with no directory component
|
445
|
+
return nv
|
446
|
+
if os.path.isdir(dir):
|
447
|
+
return nv
|
448
|
+
raise ValueError(f'The directory named as part of the path {v} does not exist')
|
685
449
|
|
686
|
-
def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
687
|
-
pos += 1
|
688
|
-
return toml_parse_basic_str(src, pos, multiline=False)
|
689
450
|
|
451
|
+
########################################
|
452
|
+
# ../utils/ostypes.py
|
690
453
|
|
691
|
-
def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
|
692
|
-
pos += 1
|
693
|
-
array: list = []
|
694
454
|
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
while True:
|
699
|
-
pos, val = toml_parse_value(src, pos, parse_float)
|
700
|
-
array.append(val)
|
701
|
-
pos = toml_skip_comments_and_array_ws(src, pos)
|
455
|
+
Fd = ta.NewType('Fd', int)
|
456
|
+
Pid = ta.NewType('Pid', int)
|
457
|
+
Rc = ta.NewType('Rc', int)
|
702
458
|
|
703
|
-
|
704
|
-
|
705
|
-
return pos + 1, array
|
706
|
-
if c != ',':
|
707
|
-
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
708
|
-
pos += 1
|
459
|
+
Uid = ta.NewType('Uid', int)
|
460
|
+
Gid = ta.NewType('Gid', int)
|
709
461
|
|
710
|
-
pos = toml_skip_comments_and_array_ws(src, pos)
|
711
|
-
if src.startswith(']', pos):
|
712
|
-
return pos + 1, array
|
713
462
|
|
463
|
+
########################################
|
464
|
+
# ../utils/signals.py
|
714
465
|
|
715
|
-
def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
|
716
|
-
pos += 1
|
717
|
-
nested_dict = TomlNestedDict()
|
718
|
-
flags = TomlFlags()
|
719
466
|
|
720
|
-
|
721
|
-
if src.startswith('}', pos):
|
722
|
-
return pos + 1, nested_dict.dict
|
723
|
-
while True:
|
724
|
-
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
725
|
-
key_parent, key_stem = key[:-1], key[-1]
|
726
|
-
if flags.is_(key, TomlFlags.FROZEN):
|
727
|
-
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
728
|
-
try:
|
729
|
-
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
730
|
-
except KeyError:
|
731
|
-
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
732
|
-
if key_stem in nest:
|
733
|
-
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
734
|
-
nest[key_stem] = value
|
735
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
736
|
-
c = src[pos:pos + 1]
|
737
|
-
if c == '}':
|
738
|
-
return pos + 1, nested_dict.dict
|
739
|
-
if c != ',':
|
740
|
-
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
741
|
-
if isinstance(value, (dict, list)):
|
742
|
-
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
743
|
-
pos += 1
|
744
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
467
|
+
##
|
745
468
|
|
746
469
|
|
747
|
-
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
752
|
-
) -> ta.Tuple[TomlPos, str]:
|
753
|
-
escape_id = src[pos:pos + 2]
|
754
|
-
pos += 2
|
755
|
-
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
756
|
-
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
757
|
-
# newline.
|
758
|
-
if escape_id != '\\\n':
|
759
|
-
pos = toml_skip_chars(src, pos, TOML_WS)
|
760
|
-
try:
|
761
|
-
char = src[pos]
|
762
|
-
except IndexError:
|
763
|
-
return pos, ''
|
764
|
-
if char != '\n':
|
765
|
-
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
766
|
-
pos += 1
|
767
|
-
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
768
|
-
return pos, ''
|
769
|
-
if escape_id == '\\u':
|
770
|
-
return toml_parse_hex_char(src, pos, 4)
|
771
|
-
if escape_id == '\\U':
|
772
|
-
return toml_parse_hex_char(src, pos, 8)
|
470
|
+
_SIGS_BY_NUM: ta.Mapping[int, signal.Signals] = {s.value: s for s in signal.Signals}
|
471
|
+
_SIGS_BY_NAME: ta.Mapping[str, signal.Signals] = {s.name: s for s in signal.Signals}
|
472
|
+
|
473
|
+
|
474
|
+
def sig_num(value: ta.Union[int, str]) -> int:
|
773
475
|
try:
|
774
|
-
|
775
|
-
except KeyError:
|
776
|
-
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
476
|
+
num = int(value)
|
777
477
|
|
478
|
+
except (ValueError, TypeError):
|
479
|
+
name = value.strip().upper() # type: ignore
|
480
|
+
if not name.startswith('SIG'):
|
481
|
+
name = f'SIG{name}'
|
778
482
|
|
779
|
-
|
780
|
-
|
483
|
+
if (sn := _SIGS_BY_NAME.get(name)) is None:
|
484
|
+
raise ValueError(f'value {value!r} is not a valid signal name') # noqa
|
485
|
+
num = sn
|
781
486
|
|
487
|
+
if num not in _SIGS_BY_NUM:
|
488
|
+
raise ValueError(f'value {value!r} is not a valid signal number')
|
782
489
|
|
783
|
-
|
784
|
-
hex_str = src[pos:pos + hex_len]
|
785
|
-
if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
|
786
|
-
raise toml_suffixed_err(src, pos, 'Invalid hex value')
|
787
|
-
pos += hex_len
|
788
|
-
hex_int = int(hex_str, 16)
|
789
|
-
if not toml_is_unicode_scalar_value(hex_int):
|
790
|
-
raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
|
791
|
-
return pos, chr(hex_int)
|
490
|
+
return num
|
792
491
|
|
793
492
|
|
794
|
-
def
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
|
799
|
-
)
|
800
|
-
return pos + 1, src[start_pos:pos] # Skip ending apostrophe
|
493
|
+
def sig_name(num: int) -> str:
|
494
|
+
if (sig := _SIGS_BY_NUM.get(num)) is not None:
|
495
|
+
return sig.name
|
496
|
+
return f'signal {sig}'
|
801
497
|
|
802
498
|
|
803
|
-
|
804
|
-
pos += 3
|
805
|
-
if src.startswith('\n', pos):
|
806
|
-
pos += 1
|
499
|
+
##
|
807
500
|
|
808
|
-
if literal:
|
809
|
-
delim = "'"
|
810
|
-
end_pos = toml_skip_until(
|
811
|
-
src,
|
812
|
-
pos,
|
813
|
-
"'''",
|
814
|
-
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
815
|
-
error_on_eof=True,
|
816
|
-
)
|
817
|
-
result = src[pos:end_pos]
|
818
|
-
pos = end_pos + 3
|
819
|
-
else:
|
820
|
-
delim = '"'
|
821
|
-
pos, result = toml_parse_basic_str(src, pos, multiline=True)
|
822
501
|
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
pos += 1
|
827
|
-
if not src.startswith(delim, pos):
|
828
|
-
return pos, result + delim
|
829
|
-
pos += 1
|
830
|
-
return pos, result + (delim * 2)
|
502
|
+
class SignalReceiver:
|
503
|
+
def __init__(self) -> None:
|
504
|
+
super().__init__()
|
831
505
|
|
506
|
+
self._signals_recvd: ta.List[int] = []
|
832
507
|
|
833
|
-
def
|
834
|
-
|
835
|
-
|
836
|
-
|
508
|
+
def receive(self, sig: int, frame: ta.Any = None) -> None:
|
509
|
+
if sig not in self._signals_recvd:
|
510
|
+
self._signals_recvd.append(sig)
|
511
|
+
|
512
|
+
def install(self, *sigs: int) -> None:
|
513
|
+
for sig in sigs:
|
514
|
+
signal.signal(sig, self.receive)
|
515
|
+
|
516
|
+
def get_signal(self) -> ta.Optional[int]:
|
517
|
+
if self._signals_recvd:
|
518
|
+
sig = self._signals_recvd.pop(0)
|
519
|
+
else:
|
520
|
+
sig = None
|
521
|
+
return sig
|
522
|
+
|
523
|
+
|
524
|
+
########################################
|
525
|
+
# ../utils/strings.py
|
526
|
+
|
527
|
+
|
528
|
+
##
|
529
|
+
|
530
|
+
|
531
|
+
def as_bytes(s: ta.Union[str, bytes], encoding: str = 'utf8') -> bytes:
|
532
|
+
if isinstance(s, bytes):
|
533
|
+
return s
|
837
534
|
else:
|
838
|
-
|
839
|
-
parse_escapes = toml_parse_basic_str_escape
|
840
|
-
result = ''
|
841
|
-
start_pos = pos
|
842
|
-
while True:
|
843
|
-
try:
|
844
|
-
char = src[pos]
|
845
|
-
except IndexError:
|
846
|
-
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
847
|
-
if char == '"':
|
848
|
-
if not multiline:
|
849
|
-
return pos + 1, result + src[start_pos:pos]
|
850
|
-
if src.startswith('"""', pos):
|
851
|
-
return pos + 3, result + src[start_pos:pos]
|
852
|
-
pos += 1
|
853
|
-
continue
|
854
|
-
if char == '\\':
|
855
|
-
result += src[start_pos:pos]
|
856
|
-
pos, parsed_escape = parse_escapes(src, pos)
|
857
|
-
result += parsed_escape
|
858
|
-
start_pos = pos
|
859
|
-
continue
|
860
|
-
if char in error_on:
|
861
|
-
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
862
|
-
pos += 1
|
535
|
+
return s.encode(encoding)
|
863
536
|
|
864
537
|
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
parse_float: TomlParseFloat,
|
869
|
-
) -> ta.Tuple[TomlPos, ta.Any]:
|
870
|
-
try:
|
871
|
-
char: ta.Optional[str] = src[pos]
|
872
|
-
except IndexError:
|
873
|
-
char = None
|
538
|
+
@ta.overload
|
539
|
+
def find_prefix_at_end(haystack: str, needle: str) -> int:
|
540
|
+
...
|
874
541
|
|
875
|
-
# IMPORTANT: order conditions based on speed of checking and likelihood
|
876
542
|
|
877
|
-
|
878
|
-
|
879
|
-
|
880
|
-
return toml_parse_multiline_str(src, pos, literal=False)
|
881
|
-
return toml_parse_one_line_basic_str(src, pos)
|
543
|
+
@ta.overload
|
544
|
+
def find_prefix_at_end(haystack: bytes, needle: bytes) -> int:
|
545
|
+
...
|
882
546
|
|
883
|
-
# Literal strings
|
884
|
-
if char == "'":
|
885
|
-
if src.startswith("'''", pos):
|
886
|
-
return toml_parse_multiline_str(src, pos, literal=True)
|
887
|
-
return toml_parse_literal_str(src, pos)
|
888
547
|
|
889
|
-
|
890
|
-
|
891
|
-
|
892
|
-
|
893
|
-
|
894
|
-
if src.startswith('false', pos):
|
895
|
-
return pos + 5, False
|
548
|
+
def find_prefix_at_end(haystack, needle):
|
549
|
+
l = len(needle) - 1
|
550
|
+
while l and not haystack.endswith(needle[:l]):
|
551
|
+
l -= 1
|
552
|
+
return l
|
896
553
|
|
897
|
-
# Arrays
|
898
|
-
if char == '[':
|
899
|
-
return toml_parse_array(src, pos, parse_float)
|
900
554
|
|
901
|
-
|
902
|
-
|
903
|
-
|
555
|
+
##
|
556
|
+
|
557
|
+
|
558
|
+
ANSI_ESCAPE_BEGIN = b'\x1b['
|
559
|
+
ANSI_TERMINATORS = (b'H', b'f', b'A', b'B', b'C', b'D', b'R', b's', b'u', b'J', b'K', b'h', b'l', b'p', b'm')
|
560
|
+
|
561
|
+
|
562
|
+
def strip_escapes(s: bytes) -> bytes:
|
563
|
+
"""Remove all ANSI color escapes from the given string."""
|
564
|
+
|
565
|
+
result = b''
|
566
|
+
show = 1
|
567
|
+
i = 0
|
568
|
+
l = len(s)
|
569
|
+
while i < l:
|
570
|
+
if show == 0 and s[i:i + 1] in ANSI_TERMINATORS:
|
571
|
+
show = 1
|
572
|
+
elif show:
|
573
|
+
n = s.find(ANSI_ESCAPE_BEGIN, i)
|
574
|
+
if n == -1:
|
575
|
+
return result + s[i:]
|
576
|
+
else:
|
577
|
+
result = result + s[i:n]
|
578
|
+
i = n
|
579
|
+
show = 0
|
580
|
+
i += 1
|
581
|
+
return result
|
582
|
+
|
583
|
+
|
584
|
+
##
|
585
|
+
|
586
|
+
|
587
|
+
class SuffixMultiplier:
|
588
|
+
# d is a dictionary of suffixes to integer multipliers. If no suffixes match, default is the multiplier. Matches are
|
589
|
+
# case insensitive. Return values are in the fundamental unit.
|
590
|
+
def __init__(self, d, default=1):
|
591
|
+
super().__init__()
|
592
|
+
self._d = d
|
593
|
+
self._default = default
|
594
|
+
# all keys must be the same size
|
595
|
+
self._keysz = None
|
596
|
+
for k in d:
|
597
|
+
if self._keysz is None:
|
598
|
+
self._keysz = len(k)
|
599
|
+
elif self._keysz != len(k): # type: ignore
|
600
|
+
raise ValueError(k)
|
601
|
+
|
602
|
+
def __call__(self, v: ta.Union[str, int]) -> int:
|
603
|
+
if isinstance(v, int):
|
604
|
+
return v
|
605
|
+
v = v.lower()
|
606
|
+
for s, m in self._d.items():
|
607
|
+
if v[-self._keysz:] == s: # type: ignore
|
608
|
+
return int(v[:-self._keysz]) * m # type: ignore
|
609
|
+
return int(v) * self._default
|
610
|
+
|
611
|
+
|
612
|
+
parse_bytes_size = SuffixMultiplier({
|
613
|
+
'kb': 1024,
|
614
|
+
'mb': 1024 * 1024,
|
615
|
+
'gb': 1024 * 1024 * 1024,
|
616
|
+
})
|
617
|
+
|
618
|
+
|
619
|
+
#
|
620
|
+
|
621
|
+
|
622
|
+
def parse_octal(arg: ta.Union[str, int]) -> int:
|
623
|
+
if isinstance(arg, int):
|
624
|
+
return arg
|
625
|
+
try:
|
626
|
+
return int(arg, 8)
|
627
|
+
except (TypeError, ValueError):
|
628
|
+
raise ValueError(f'{arg} can not be converted to an octal type') # noqa
|
629
|
+
|
630
|
+
|
631
|
+
########################################
|
632
|
+
# ../../../omlish/configs/types.py
|
904
633
|
|
905
|
-
# Dates and times
|
906
|
-
datetime_match = TOML_RE_DATETIME.match(src, pos)
|
907
|
-
if datetime_match:
|
908
|
-
try:
|
909
|
-
datetime_obj = toml_match_to_datetime(datetime_match)
|
910
|
-
except ValueError as e:
|
911
|
-
raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
|
912
|
-
return datetime_match.end(), datetime_obj
|
913
|
-
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
914
|
-
if localtime_match:
|
915
|
-
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
916
634
|
|
917
|
-
|
918
|
-
# located after handling of dates and times.
|
919
|
-
number_match = TOML_RE_NUMBER.match(src, pos)
|
920
|
-
if number_match:
|
921
|
-
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
635
|
+
#
|
922
636
|
|
923
|
-
# Special floats
|
924
|
-
first_three = src[pos:pos + 3]
|
925
|
-
if first_three in {'inf', 'nan'}:
|
926
|
-
return pos + 3, parse_float(first_three)
|
927
|
-
first_four = src[pos:pos + 4]
|
928
|
-
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
929
|
-
return pos + 4, parse_float(first_four)
|
930
637
|
|
931
|
-
|
638
|
+
########################################
|
639
|
+
# ../../../omlish/formats/ini/sections.py
|
932
640
|
|
933
641
|
|
934
|
-
|
935
|
-
"""Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
|
642
|
+
##
|
936
643
|
|
937
|
-
def coord_repr(src: str, pos: TomlPos) -> str:
|
938
|
-
if pos >= len(src):
|
939
|
-
return 'end of document'
|
940
|
-
line = src.count('\n', 0, pos) + 1
|
941
|
-
if line == 1:
|
942
|
-
column = pos + 1
|
943
|
-
else:
|
944
|
-
column = pos - src.rindex('\n', 0, pos)
|
945
|
-
return f'line {line}, column {column}'
|
946
644
|
|
947
|
-
|
645
|
+
def extract_ini_sections(cp: configparser.ConfigParser) -> IniSectionSettingsMap:
|
646
|
+
config_dct: ta.Dict[str, ta.Any] = {}
|
647
|
+
for sec in cp.sections():
|
648
|
+
cd = config_dct
|
649
|
+
for k in sec.split('.'):
|
650
|
+
cd = cd.setdefault(k, {})
|
651
|
+
cd.update(cp.items(sec))
|
652
|
+
return config_dct
|
948
653
|
|
949
654
|
|
950
|
-
|
951
|
-
return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
|
655
|
+
##
|
952
656
|
|
953
657
|
|
954
|
-
def
|
955
|
-
|
658
|
+
def render_ini_sections(
|
659
|
+
settings_by_section: IniSectionSettingsMap,
|
660
|
+
) -> str:
|
661
|
+
out = io.StringIO()
|
956
662
|
|
957
|
-
|
958
|
-
|
959
|
-
|
960
|
-
# The default `float` callable never returns illegal types. Optimize it.
|
961
|
-
if parse_float is float:
|
962
|
-
return float
|
663
|
+
for i, (section, settings) in enumerate(settings_by_section.items()):
|
664
|
+
if i:
|
665
|
+
out.write('\n')
|
963
666
|
|
964
|
-
|
965
|
-
float_value = parse_float(float_str)
|
966
|
-
if isinstance(float_value, (dict, list)):
|
967
|
-
raise ValueError('parse_float must not return dicts or lists') # noqa
|
968
|
-
return float_value
|
667
|
+
out.write(f'[{section}]\n')
|
969
668
|
|
970
|
-
|
669
|
+
for k, v in settings.items():
|
670
|
+
if isinstance(v, str):
|
671
|
+
out.write(f'{k}={v}\n')
|
672
|
+
else:
|
673
|
+
for vv in v:
|
674
|
+
out.write(f'{k}={vv}\n')
|
675
|
+
|
676
|
+
return out.getvalue()
|
971
677
|
|
972
678
|
|
973
679
|
########################################
|
974
|
-
#
|
680
|
+
# ../../../omlish/formats/toml/parser.py
|
681
|
+
# SPDX-License-Identifier: MIT
|
682
|
+
# SPDX-FileCopyrightText: 2021 Taneli Hukkinen
|
683
|
+
# Licensed to PSF under a Contributor Agreement.
|
684
|
+
#
|
685
|
+
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
686
|
+
# --------------------------------------------
|
687
|
+
#
|
688
|
+
# 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization
|
689
|
+
# ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated
|
690
|
+
# documentation.
|
691
|
+
#
|
692
|
+
# 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive,
|
693
|
+
# royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative
|
694
|
+
# works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License
|
695
|
+
# Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009,
|
696
|
+
# 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; All
|
697
|
+
# Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee.
|
698
|
+
#
|
699
|
+
# 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and
|
700
|
+
# wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in
|
701
|
+
# any such work a brief summary of the changes made to Python.
|
702
|
+
#
|
703
|
+
# 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES,
|
704
|
+
# EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY
|
705
|
+
# OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY
|
706
|
+
# RIGHTS.
|
707
|
+
#
|
708
|
+
# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL
|
709
|
+
# DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF
|
710
|
+
# ADVISED OF THE POSSIBILITY THEREOF.
|
711
|
+
#
|
712
|
+
# 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions.
|
713
|
+
#
|
714
|
+
# 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint
|
715
|
+
# venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade
|
716
|
+
# name in a trademark sense to endorse or promote products or services of Licensee, or any third party.
|
717
|
+
#
|
718
|
+
# 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this
|
719
|
+
# License Agreement.
|
720
|
+
#
|
721
|
+
# https://github.com/python/cpython/blob/9ce90206b7a4649600218cf0bd4826db79c9a312/Lib/tomllib/_parser.py
|
975
722
|
|
976
723
|
|
977
|
-
|
978
|
-
"""Specialized exceptions used when attempting to start a process."""
|
724
|
+
##
|
979
725
|
|
980
726
|
|
981
|
-
|
982
|
-
|
727
|
+
_TOML_TIME_RE_STR = r'([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?'
|
728
|
+
|
729
|
+
TOML_RE_NUMBER = re.compile(
|
730
|
+
r"""
|
731
|
+
0
|
732
|
+
(?:
|
733
|
+
x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex
|
734
|
+
|
|
735
|
+
b[01](?:_?[01])* # bin
|
736
|
+
|
|
737
|
+
o[0-7](?:_?[0-7])* # oct
|
738
|
+
)
|
739
|
+
|
|
740
|
+
[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part
|
741
|
+
(?P<floatpart>
|
742
|
+
(?:\.[0-9](?:_?[0-9])*)? # optional fractional part
|
743
|
+
(?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part
|
744
|
+
)
|
745
|
+
""",
|
746
|
+
flags=re.VERBOSE,
|
747
|
+
)
|
748
|
+
TOML_RE_LOCALTIME = re.compile(_TOML_TIME_RE_STR)
|
749
|
+
TOML_RE_DATETIME = re.compile(
|
750
|
+
rf"""
|
751
|
+
([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27
|
752
|
+
(?:
|
753
|
+
[Tt ]
|
754
|
+
{_TOML_TIME_RE_STR}
|
755
|
+
(?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset
|
756
|
+
)?
|
757
|
+
""",
|
758
|
+
flags=re.VERBOSE,
|
759
|
+
)
|
983
760
|
|
984
761
|
|
985
|
-
|
986
|
-
"""
|
987
|
-
|
988
|
-
|
762
|
+
def toml_match_to_datetime(match: re.Match) -> ta.Union[datetime.datetime, datetime.date]:
|
763
|
+
"""Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`.
|
764
|
+
|
765
|
+
Raises ValueError if the match does not correspond to a valid date or datetime.
|
989
766
|
"""
|
767
|
+
(
|
768
|
+
year_str,
|
769
|
+
month_str,
|
770
|
+
day_str,
|
771
|
+
hour_str,
|
772
|
+
minute_str,
|
773
|
+
sec_str,
|
774
|
+
micros_str,
|
775
|
+
zulu_time,
|
776
|
+
offset_sign_str,
|
777
|
+
offset_hour_str,
|
778
|
+
offset_minute_str,
|
779
|
+
) = match.groups()
|
780
|
+
year, month, day = int(year_str), int(month_str), int(day_str)
|
781
|
+
if hour_str is None:
|
782
|
+
return datetime.date(year, month, day)
|
783
|
+
hour, minute, sec = int(hour_str), int(minute_str), int(sec_str)
|
784
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
785
|
+
if offset_sign_str:
|
786
|
+
tz: ta.Optional[datetime.tzinfo] = toml_cached_tz(
|
787
|
+
offset_hour_str, offset_minute_str, offset_sign_str,
|
788
|
+
)
|
789
|
+
elif zulu_time:
|
790
|
+
tz = datetime.UTC
|
791
|
+
else: # local date-time
|
792
|
+
tz = None
|
793
|
+
return datetime.datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz)
|
990
794
|
|
991
795
|
|
992
|
-
|
993
|
-
|
796
|
+
@functools.lru_cache() # noqa
|
797
|
+
def toml_cached_tz(hour_str: str, minute_str: str, sign_str: str) -> datetime.timezone:
|
798
|
+
sign = 1 if sign_str == '+' else -1
|
799
|
+
return datetime.timezone(
|
800
|
+
datetime.timedelta(
|
801
|
+
hours=sign * int(hour_str),
|
802
|
+
minutes=sign * int(minute_str),
|
803
|
+
),
|
804
|
+
)
|
994
805
|
|
995
806
|
|
996
|
-
|
997
|
-
|
998
|
-
|
999
|
-
|
1000
|
-
"""
|
807
|
+
def toml_match_to_localtime(match: re.Match) -> datetime.time:
|
808
|
+
hour_str, minute_str, sec_str, micros_str = match.groups()
|
809
|
+
micros = int(micros_str.ljust(6, '0')) if micros_str else 0
|
810
|
+
return datetime.time(int(hour_str), int(minute_str), int(sec_str), micros)
|
1001
811
|
|
1002
812
|
|
1003
|
-
|
1004
|
-
|
813
|
+
def toml_match_to_number(match: re.Match, parse_float: TomlParseFloat) -> ta.Any:
|
814
|
+
if match.group('floatpart'):
|
815
|
+
return parse_float(match.group())
|
816
|
+
return int(match.group(), 0)
|
1005
817
|
|
1006
818
|
|
1007
|
-
|
1008
|
-
"""
|
1009
|
-
Drop privileges to become the specified user, which may be a username or uid. Called for supervisord startup and
|
1010
|
-
when spawning subprocesses. Returns None on success or a string error message if privileges could not be dropped.
|
1011
|
-
"""
|
819
|
+
TOML_ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
|
1012
820
|
|
1013
|
-
|
1014
|
-
|
821
|
+
# Neither of these sets include quotation mark or backslash. They are currently handled as separate cases in the parser
|
822
|
+
# functions.
|
823
|
+
TOML_ILLEGAL_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t')
|
824
|
+
TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS = TOML_ASCII_CTRL - frozenset('\t\n')
|
1015
825
|
|
1016
|
-
|
1017
|
-
|
1018
|
-
uid = int(user)
|
1019
|
-
except ValueError:
|
1020
|
-
try:
|
1021
|
-
pwrec = pwd.getpwnam(user) # type: ignore
|
1022
|
-
except KeyError:
|
1023
|
-
return f"Can't find username {user!r}"
|
1024
|
-
uid = pwrec[2]
|
1025
|
-
else:
|
1026
|
-
try:
|
1027
|
-
pwrec = pwd.getpwuid(uid)
|
1028
|
-
except KeyError:
|
1029
|
-
return f"Can't find uid {uid!r}"
|
826
|
+
TOML_ILLEGAL_LITERAL_STR_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
827
|
+
TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1030
828
|
|
1031
|
-
|
829
|
+
TOML_ILLEGAL_COMMENT_CHARS = TOML_ILLEGAL_BASIC_STR_CHARS
|
1032
830
|
|
1033
|
-
|
1034
|
-
|
1035
|
-
|
1036
|
-
|
831
|
+
TOML_WS = frozenset(' \t')
|
832
|
+
TOML_WS_AND_NEWLINE = TOML_WS | frozenset('\n')
|
833
|
+
TOML_BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + '-_')
|
834
|
+
TOML_KEY_INITIAL_CHARS = TOML_BARE_KEY_CHARS | frozenset("\"'")
|
835
|
+
TOML_HEXDIGIT_CHARS = frozenset(string.hexdigits)
|
1037
836
|
|
1038
|
-
|
1039
|
-
|
837
|
+
TOML_BASIC_STR_ESCAPE_REPLACEMENTS = types.MappingProxyType(
|
838
|
+
{
|
839
|
+
'\\b': '\u0008', # backspace
|
840
|
+
'\\t': '\u0009', # tab
|
841
|
+
'\\n': '\u000A', # linefeed
|
842
|
+
'\\f': '\u000C', # form feed
|
843
|
+
'\\r': '\u000D', # carriage return
|
844
|
+
'\\"': '\u0022', # quote
|
845
|
+
'\\\\': '\u005C', # backslash
|
846
|
+
},
|
847
|
+
)
|
1040
848
|
|
1041
|
-
gid = pwrec[3]
|
1042
|
-
if hasattr(os, 'setgroups'):
|
1043
|
-
user = pwrec[0]
|
1044
|
-
groups = [grprec[2] for grprec in grp.getgrall() if user in grprec[3]]
|
1045
849
|
|
1046
|
-
|
1047
|
-
|
1048
|
-
# python 2.7 - this will be safe though for all unix /python version combos)
|
1049
|
-
groups.insert(0, gid)
|
1050
|
-
try:
|
1051
|
-
os.setgroups(groups)
|
1052
|
-
except OSError:
|
1053
|
-
return 'Could not set groups of effective user'
|
850
|
+
class TomlDecodeError(ValueError):
|
851
|
+
"""An error raised if a document is not valid TOML."""
|
1054
852
|
|
853
|
+
|
854
|
+
def toml_load(fp: ta.BinaryIO, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]:
|
855
|
+
"""Parse TOML from a binary file object."""
|
856
|
+
b = fp.read()
|
1055
857
|
try:
|
1056
|
-
|
1057
|
-
except
|
1058
|
-
|
858
|
+
s = b.decode()
|
859
|
+
except AttributeError:
|
860
|
+
raise TypeError("File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`") from None
|
861
|
+
return toml_loads(s, parse_float=parse_float)
|
1059
862
|
|
1060
|
-
os.setuid(uid)
|
1061
863
|
|
1062
|
-
|
864
|
+
def toml_loads(s: str, /, *, parse_float: TomlParseFloat = float) -> ta.Dict[str, ta.Any]: # noqa: C901
|
865
|
+
"""Parse TOML from a string."""
|
1063
866
|
|
867
|
+
# The spec allows converting "\r\n" to "\n", even in string literals. Let's do so to simplify parsing.
|
868
|
+
try:
|
869
|
+
src = s.replace('\r\n', '\n')
|
870
|
+
except (AttributeError, TypeError):
|
871
|
+
raise TypeError(f"Expected str object, not '{type(s).__qualname__}'") from None
|
872
|
+
pos = 0
|
873
|
+
out = TomlOutput(TomlNestedDict(), TomlFlags())
|
874
|
+
header: TomlKey = ()
|
875
|
+
parse_float = toml_make_safe_parse_float(parse_float)
|
1064
876
|
|
1065
|
-
|
1066
|
-
|
877
|
+
# Parse one statement at a time (typically means one line in TOML source)
|
878
|
+
while True:
|
879
|
+
# 1. Skip line leading whitespace
|
880
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1067
881
|
|
882
|
+
# 2. Parse rules. Expect one of the following:
|
883
|
+
# - end of file
|
884
|
+
# - end of line
|
885
|
+
# - comment
|
886
|
+
# - key/value pair
|
887
|
+
# - append dict to list (and move to its namespace)
|
888
|
+
# - create dict (and move to its namespace)
|
889
|
+
# Skip trailing whitespace when applicable.
|
890
|
+
try:
|
891
|
+
char = src[pos]
|
892
|
+
except IndexError:
|
893
|
+
break
|
894
|
+
if char == '\n':
|
895
|
+
pos += 1
|
896
|
+
continue
|
897
|
+
if char in TOML_KEY_INITIAL_CHARS:
|
898
|
+
pos = toml_key_value_rule(src, pos, out, header, parse_float)
|
899
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
900
|
+
elif char == '[':
|
901
|
+
try:
|
902
|
+
second_char: ta.Optional[str] = src[pos + 1]
|
903
|
+
except IndexError:
|
904
|
+
second_char = None
|
905
|
+
out.flags.finalize_pending()
|
906
|
+
if second_char == '[':
|
907
|
+
pos, header = toml_create_list_rule(src, pos, out)
|
908
|
+
else:
|
909
|
+
pos, header = toml_create_dict_rule(src, pos, out)
|
910
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
911
|
+
elif char != '#':
|
912
|
+
raise toml_suffixed_err(src, pos, 'Invalid statement')
|
1068
913
|
|
1069
|
-
|
914
|
+
# 3. Skip comment
|
915
|
+
pos = toml_skip_comment(src, pos)
|
1070
916
|
|
917
|
+
# 4. Expect end of line or end of file
|
918
|
+
try:
|
919
|
+
char = src[pos]
|
920
|
+
except IndexError:
|
921
|
+
break
|
922
|
+
if char != '\n':
|
923
|
+
raise toml_suffixed_err(
|
924
|
+
src, pos, 'Expected newline or end of document after a statement',
|
925
|
+
)
|
926
|
+
pos += 1
|
1071
927
|
|
1072
|
-
|
1073
|
-
STOPPED = 0
|
1074
|
-
STARTING = 10
|
1075
|
-
RUNNING = 20
|
1076
|
-
BACKOFF = 30
|
1077
|
-
STOPPING = 40
|
1078
|
-
EXITED = 100
|
1079
|
-
FATAL = 200
|
1080
|
-
UNKNOWN = 1000
|
928
|
+
return out.data.dict
|
1081
929
|
|
1082
|
-
@property
|
1083
|
-
def stopped(self) -> bool:
|
1084
|
-
return self in STOPPED_STATES
|
1085
930
|
|
1086
|
-
|
1087
|
-
|
1088
|
-
return self in RUNNING_STATES
|
931
|
+
class TomlFlags:
|
932
|
+
"""Flags that map to parsed keys/namespaces."""
|
1089
933
|
|
1090
|
-
|
1091
|
-
|
1092
|
-
|
934
|
+
# Marks an immutable namespace (inline array or inline table).
|
935
|
+
FROZEN = 0
|
936
|
+
# Marks a nest that has been explicitly created and can no longer be opened using the "[table]" syntax.
|
937
|
+
EXPLICIT_NEST = 1
|
1093
938
|
|
939
|
+
def __init__(self) -> None:
|
940
|
+
self._flags: ta.Dict[str, dict] = {}
|
941
|
+
self._pending_flags: ta.Set[ta.Tuple[TomlKey, int]] = set()
|
1094
942
|
|
1095
|
-
|
1096
|
-
|
1097
|
-
ProcessState.STOPPED: (ProcessState.STARTING,),
|
1098
|
-
ProcessState.STARTING: (ProcessState.RUNNING, ProcessState.BACKOFF, ProcessState.STOPPING),
|
1099
|
-
ProcessState.RUNNING: (ProcessState.STOPPING, ProcessState.EXITED),
|
1100
|
-
ProcessState.BACKOFF: (ProcessState.STARTING, ProcessState.FATAL),
|
1101
|
-
ProcessState.STOPPING: (ProcessState.STOPPED,),
|
1102
|
-
ProcessState.EXITED: (ProcessState.STARTING,),
|
1103
|
-
ProcessState.FATAL: (ProcessState.STARTING,),
|
1104
|
-
}
|
943
|
+
def add_pending(self, key: TomlKey, flag: int) -> None:
|
944
|
+
self._pending_flags.add((key, flag))
|
1105
945
|
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
1109
|
-
|
1110
|
-
ProcessState.UNKNOWN,
|
1111
|
-
)
|
946
|
+
def finalize_pending(self) -> None:
|
947
|
+
for key, flag in self._pending_flags:
|
948
|
+
self.set(key, flag, recursive=False)
|
949
|
+
self._pending_flags.clear()
|
1112
950
|
|
1113
|
-
|
1114
|
-
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
951
|
+
def unset_all(self, key: TomlKey) -> None:
|
952
|
+
cont = self._flags
|
953
|
+
for k in key[:-1]:
|
954
|
+
if k not in cont:
|
955
|
+
return
|
956
|
+
cont = cont[k]['nested']
|
957
|
+
cont.pop(key[-1], None)
|
1118
958
|
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
959
|
+
def set(self, key: TomlKey, flag: int, *, recursive: bool) -> None: # noqa: A003
|
960
|
+
cont = self._flags
|
961
|
+
key_parent, key_stem = key[:-1], key[-1]
|
962
|
+
for k in key_parent:
|
963
|
+
if k not in cont:
|
964
|
+
cont[k] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
965
|
+
cont = cont[k]['nested']
|
966
|
+
if key_stem not in cont:
|
967
|
+
cont[key_stem] = {'flags': set(), 'recursive_flags': set(), 'nested': {}}
|
968
|
+
cont[key_stem]['recursive_flags' if recursive else 'flags'].add(flag)
|
1124
969
|
|
970
|
+
def is_(self, key: TomlKey, flag: int) -> bool:
|
971
|
+
if not key:
|
972
|
+
return False # document root has no flags
|
973
|
+
cont = self._flags
|
974
|
+
for k in key[:-1]:
|
975
|
+
if k not in cont:
|
976
|
+
return False
|
977
|
+
inner_cont = cont[k]
|
978
|
+
if flag in inner_cont['recursive_flags']:
|
979
|
+
return True
|
980
|
+
cont = inner_cont['nested']
|
981
|
+
key_stem = key[-1]
|
982
|
+
if key_stem in cont:
|
983
|
+
cont = cont[key_stem]
|
984
|
+
return flag in cont['flags'] or flag in cont['recursive_flags']
|
985
|
+
return False
|
1125
986
|
|
1126
|
-
##
|
1127
987
|
|
988
|
+
class TomlNestedDict:
|
989
|
+
def __init__(self) -> None:
|
990
|
+
# The parsed content of the TOML document
|
991
|
+
self.dict: ta.Dict[str, ta.Any] = {}
|
1128
992
|
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
993
|
+
def get_or_create_nest(
|
994
|
+
self,
|
995
|
+
key: TomlKey,
|
996
|
+
*,
|
997
|
+
access_lists: bool = True,
|
998
|
+
) -> dict:
|
999
|
+
cont: ta.Any = self.dict
|
1000
|
+
for k in key:
|
1001
|
+
if k not in cont:
|
1002
|
+
cont[k] = {}
|
1003
|
+
cont = cont[k]
|
1004
|
+
if access_lists and isinstance(cont, list):
|
1005
|
+
cont = cont[-1]
|
1006
|
+
if not isinstance(cont, dict):
|
1007
|
+
raise KeyError('There is no nest behind this key')
|
1008
|
+
return cont
|
1134
1009
|
|
1010
|
+
def append_nest_to_list(self, key: TomlKey) -> None:
|
1011
|
+
cont = self.get_or_create_nest(key[:-1])
|
1012
|
+
last_key = key[-1]
|
1013
|
+
if last_key in cont:
|
1014
|
+
list_ = cont[last_key]
|
1015
|
+
if not isinstance(list_, list):
|
1016
|
+
raise KeyError('An object other than list found behind this key')
|
1017
|
+
list_.append({})
|
1018
|
+
else:
|
1019
|
+
cont[last_key] = [{}]
|
1135
1020
|
|
1136
|
-
########################################
|
1137
|
-
# ../utils/collections.py
|
1138
1021
|
|
1022
|
+
class TomlOutput(ta.NamedTuple):
|
1023
|
+
data: TomlNestedDict
|
1024
|
+
flags: TomlFlags
|
1139
1025
|
|
1140
|
-
class KeyedCollectionAccessors(abc.ABC, ta.Generic[K, V]):
|
1141
|
-
@property
|
1142
|
-
@abc.abstractmethod
|
1143
|
-
def _by_key(self) -> ta.Mapping[K, V]:
|
1144
|
-
raise NotImplementedError
|
1145
1026
|
|
1146
|
-
|
1147
|
-
|
1027
|
+
def toml_skip_chars(src: str, pos: TomlPos, chars: ta.Iterable[str]) -> TomlPos:
|
1028
|
+
try:
|
1029
|
+
while src[pos] in chars:
|
1030
|
+
pos += 1
|
1031
|
+
except IndexError:
|
1032
|
+
pass
|
1033
|
+
return pos
|
1148
1034
|
|
1149
|
-
def __len__(self) -> int:
|
1150
|
-
return len(self._by_key)
|
1151
1035
|
|
1152
|
-
|
1153
|
-
|
1036
|
+
def toml_skip_until(
|
1037
|
+
src: str,
|
1038
|
+
pos: TomlPos,
|
1039
|
+
expect: str,
|
1040
|
+
*,
|
1041
|
+
error_on: ta.FrozenSet[str],
|
1042
|
+
error_on_eof: bool,
|
1043
|
+
) -> TomlPos:
|
1044
|
+
try:
|
1045
|
+
new_pos = src.index(expect, pos)
|
1046
|
+
except ValueError:
|
1047
|
+
new_pos = len(src)
|
1048
|
+
if error_on_eof:
|
1049
|
+
raise toml_suffixed_err(src, new_pos, f'Expected {expect!r}') from None
|
1050
|
+
|
1051
|
+
if not error_on.isdisjoint(src[pos:new_pos]):
|
1052
|
+
while src[pos] not in error_on:
|
1053
|
+
pos += 1
|
1054
|
+
raise toml_suffixed_err(src, pos, f'Found invalid character {src[pos]!r}')
|
1055
|
+
return new_pos
|
1154
1056
|
|
1155
|
-
def __getitem__(self, key: K) -> V:
|
1156
|
-
return self._by_key[key]
|
1157
1057
|
|
1158
|
-
|
1159
|
-
|
1058
|
+
def toml_skip_comment(src: str, pos: TomlPos) -> TomlPos:
|
1059
|
+
try:
|
1060
|
+
char: ta.Optional[str] = src[pos]
|
1061
|
+
except IndexError:
|
1062
|
+
char = None
|
1063
|
+
if char == '#':
|
1064
|
+
return toml_skip_until(
|
1065
|
+
src, pos + 1, '\n', error_on=TOML_ILLEGAL_COMMENT_CHARS, error_on_eof=False,
|
1066
|
+
)
|
1067
|
+
return pos
|
1160
1068
|
|
1161
|
-
def items(self) -> ta.Iterator[ta.Tuple[K, V]]:
|
1162
|
-
return iter(self._by_key.items())
|
1163
1069
|
|
1070
|
+
def toml_skip_comments_and_array_ws(src: str, pos: TomlPos) -> TomlPos:
|
1071
|
+
while True:
|
1072
|
+
pos_before_skip = pos
|
1073
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1074
|
+
pos = toml_skip_comment(src, pos)
|
1075
|
+
if pos == pos_before_skip:
|
1076
|
+
return pos
|
1164
1077
|
|
1165
|
-
class KeyedCollection(KeyedCollectionAccessors[K, V]):
|
1166
|
-
def __init__(self, items: ta.Iterable[V]) -> None:
|
1167
|
-
super().__init__()
|
1168
1078
|
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1173
|
-
by_key[k] = v
|
1174
|
-
self.__by_key = by_key
|
1079
|
+
def toml_create_dict_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
1080
|
+
pos += 1 # Skip "["
|
1081
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1082
|
+
pos, key = toml_parse_key(src, pos)
|
1175
1083
|
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1084
|
+
if out.flags.is_(key, TomlFlags.EXPLICIT_NEST) or out.flags.is_(key, TomlFlags.FROZEN):
|
1085
|
+
raise toml_suffixed_err(src, pos, f'Cannot declare {key} twice')
|
1086
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
1087
|
+
try:
|
1088
|
+
out.data.get_or_create_nest(key)
|
1089
|
+
except KeyError:
|
1090
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1179
1091
|
|
1180
|
-
|
1181
|
-
|
1182
|
-
|
1092
|
+
if not src.startswith(']', pos):
|
1093
|
+
raise toml_suffixed_err(src, pos, "Expected ']' at the end of a table declaration")
|
1094
|
+
return pos + 1, key
|
1183
1095
|
|
1184
1096
|
|
1185
|
-
|
1186
|
-
#
|
1097
|
+
def toml_create_list_rule(src: str, pos: TomlPos, out: TomlOutput) -> ta.Tuple[TomlPos, TomlKey]:
|
1098
|
+
pos += 2 # Skip "[["
|
1099
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1100
|
+
pos, key = toml_parse_key(src, pos)
|
1187
1101
|
|
1102
|
+
if out.flags.is_(key, TomlFlags.FROZEN):
|
1103
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1104
|
+
# Free the namespace now that it points to another empty list item...
|
1105
|
+
out.flags.unset_all(key)
|
1106
|
+
# ...but this key precisely is still prohibited from table declaration
|
1107
|
+
out.flags.set(key, TomlFlags.EXPLICIT_NEST, recursive=False)
|
1108
|
+
try:
|
1109
|
+
out.data.append_nest_to_list(key)
|
1110
|
+
except KeyError:
|
1111
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1188
1112
|
|
1189
|
-
|
1190
|
-
|
1191
|
-
|
1192
|
-
BaseException,
|
1193
|
-
types.TracebackType,
|
1194
|
-
]:
|
1195
|
-
t, v, tb = sys.exc_info()
|
1196
|
-
if not tb:
|
1197
|
-
raise RuntimeError('No traceback')
|
1113
|
+
if not src.startswith(']]', pos):
|
1114
|
+
raise toml_suffixed_err(src, pos, "Expected ']]' at the end of an array declaration")
|
1115
|
+
return pos + 2, key
|
1198
1116
|
|
1199
|
-
tbinfo = []
|
1200
|
-
while tb:
|
1201
|
-
tbinfo.append((
|
1202
|
-
tb.tb_frame.f_code.co_filename,
|
1203
|
-
tb.tb_frame.f_code.co_name,
|
1204
|
-
str(tb.tb_lineno),
|
1205
|
-
))
|
1206
|
-
tb = tb.tb_next
|
1207
1117
|
|
1208
|
-
|
1209
|
-
|
1118
|
+
def toml_key_value_rule(
|
1119
|
+
src: str,
|
1120
|
+
pos: TomlPos,
|
1121
|
+
out: TomlOutput,
|
1122
|
+
header: TomlKey,
|
1123
|
+
parse_float: TomlParseFloat,
|
1124
|
+
) -> TomlPos:
|
1125
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1126
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1127
|
+
abs_key_parent = header + key_parent
|
1210
1128
|
|
1211
|
-
|
1212
|
-
|
1213
|
-
|
1129
|
+
relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
|
1130
|
+
for cont_key in relative_path_cont_keys:
|
1131
|
+
# Check that dotted key syntax does not redefine an existing table
|
1132
|
+
if out.flags.is_(cont_key, TomlFlags.EXPLICIT_NEST):
|
1133
|
+
raise toml_suffixed_err(src, pos, f'Cannot redefine namespace {cont_key}')
|
1134
|
+
# Containers in the relative path can't be opened with the table syntax or dotted key/value syntax in following
|
1135
|
+
# table sections.
|
1136
|
+
out.flags.add_pending(cont_key, TomlFlags.EXPLICIT_NEST)
|
1214
1137
|
|
1138
|
+
if out.flags.is_(abs_key_parent, TomlFlags.FROZEN):
|
1139
|
+
raise toml_suffixed_err(
|
1140
|
+
src,
|
1141
|
+
pos,
|
1142
|
+
f'Cannot mutate immutable namespace {abs_key_parent}',
|
1143
|
+
)
|
1215
1144
|
|
1216
|
-
|
1217
|
-
|
1145
|
+
try:
|
1146
|
+
nest = out.data.get_or_create_nest(abs_key_parent)
|
1147
|
+
except KeyError:
|
1148
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1149
|
+
if key_stem in nest:
|
1150
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value')
|
1151
|
+
# Mark inline table and array namespaces recursively immutable
|
1152
|
+
if isinstance(value, (dict, list)):
|
1153
|
+
out.flags.set(header + key, TomlFlags.FROZEN, recursive=True)
|
1154
|
+
nest[key_stem] = value
|
1155
|
+
return pos
|
1218
1156
|
|
1219
1157
|
|
1220
|
-
def
|
1158
|
+
def toml_parse_key_value_pair(
|
1159
|
+
src: str,
|
1160
|
+
pos: TomlPos,
|
1161
|
+
parse_float: TomlParseFloat,
|
1162
|
+
) -> ta.Tuple[TomlPos, TomlKey, ta.Any]:
|
1163
|
+
pos, key = toml_parse_key(src, pos)
|
1221
1164
|
try:
|
1222
|
-
|
1223
|
-
except
|
1224
|
-
|
1225
|
-
|
1165
|
+
char: ta.Optional[str] = src[pos]
|
1166
|
+
except IndexError:
|
1167
|
+
char = None
|
1168
|
+
if char != '=':
|
1169
|
+
raise toml_suffixed_err(src, pos, "Expected '=' after a key in a key/value pair")
|
1170
|
+
pos += 1
|
1171
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1172
|
+
pos, value = toml_parse_value(src, pos, parse_float)
|
1173
|
+
return pos, key, value
|
1226
1174
|
|
1227
1175
|
|
1228
|
-
def
|
1229
|
-
|
1230
|
-
|
1231
|
-
|
1176
|
+
def toml_parse_key(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, TomlKey]:
|
1177
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
1178
|
+
key: TomlKey = (key_part,)
|
1179
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1180
|
+
while True:
|
1181
|
+
try:
|
1182
|
+
char: ta.Optional[str] = src[pos]
|
1183
|
+
except IndexError:
|
1184
|
+
char = None
|
1185
|
+
if char != '.':
|
1186
|
+
return pos, key
|
1187
|
+
pos += 1
|
1188
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1189
|
+
pos, key_part = toml_parse_key_part(src, pos)
|
1190
|
+
key += (key_part,)
|
1191
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1232
1192
|
|
1233
1193
|
|
1234
|
-
def
|
1235
|
-
|
1194
|
+
def toml_parse_key_part(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1195
|
+
try:
|
1196
|
+
char: ta.Optional[str] = src[pos]
|
1197
|
+
except IndexError:
|
1198
|
+
char = None
|
1199
|
+
if char in TOML_BARE_KEY_CHARS:
|
1200
|
+
start_pos = pos
|
1201
|
+
pos = toml_skip_chars(src, pos, TOML_BARE_KEY_CHARS)
|
1202
|
+
return pos, src[start_pos:pos]
|
1203
|
+
if char == "'":
|
1204
|
+
return toml_parse_literal_str(src, pos)
|
1205
|
+
if char == '"':
|
1206
|
+
return toml_parse_one_line_basic_str(src, pos)
|
1207
|
+
raise toml_suffixed_err(src, pos, 'Invalid initial character for a key part')
|
1236
1208
|
|
1237
|
-
path = ['/bin', '/usr/bin', '/usr/local/bin']
|
1238
|
-
if 'PATH' in os.environ:
|
1239
|
-
p = os.environ['PATH']
|
1240
|
-
if p:
|
1241
|
-
path = p.split(os.pathsep)
|
1242
|
-
return path
|
1243
1209
|
|
1210
|
+
def toml_parse_one_line_basic_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1211
|
+
pos += 1
|
1212
|
+
return toml_parse_basic_str(src, pos, multiline=False)
|
1244
1213
|
|
1245
|
-
def check_existing_dir(v: str) -> str:
|
1246
|
-
nv = os.path.expanduser(v)
|
1247
|
-
if os.path.isdir(nv):
|
1248
|
-
return nv
|
1249
|
-
raise ValueError(f'{v} is not an existing directory')
|
1250
1214
|
|
1215
|
+
def toml_parse_array(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, list]:
|
1216
|
+
pos += 1
|
1217
|
+
array: list = []
|
1251
1218
|
|
1252
|
-
|
1253
|
-
|
1254
|
-
|
1255
|
-
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1219
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1220
|
+
if src.startswith(']', pos):
|
1221
|
+
return pos + 1, array
|
1222
|
+
while True:
|
1223
|
+
pos, val = toml_parse_value(src, pos, parse_float)
|
1224
|
+
array.append(val)
|
1225
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1226
|
+
|
1227
|
+
c = src[pos:pos + 1]
|
1228
|
+
if c == ']':
|
1229
|
+
return pos + 1, array
|
1230
|
+
if c != ',':
|
1231
|
+
raise toml_suffixed_err(src, pos, 'Unclosed array')
|
1232
|
+
pos += 1
|
1261
1233
|
|
1234
|
+
pos = toml_skip_comments_and_array_ws(src, pos)
|
1235
|
+
if src.startswith(']', pos):
|
1236
|
+
return pos + 1, array
|
1262
1237
|
|
1263
|
-
########################################
|
1264
|
-
# ../utils/ostypes.py
|
1265
1238
|
|
1239
|
+
def toml_parse_inline_table(src: str, pos: TomlPos, parse_float: TomlParseFloat) -> ta.Tuple[TomlPos, dict]:
|
1240
|
+
pos += 1
|
1241
|
+
nested_dict = TomlNestedDict()
|
1242
|
+
flags = TomlFlags()
|
1266
1243
|
|
1267
|
-
|
1268
|
-
|
1269
|
-
|
1244
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1245
|
+
if src.startswith('}', pos):
|
1246
|
+
return pos + 1, nested_dict.dict
|
1247
|
+
while True:
|
1248
|
+
pos, key, value = toml_parse_key_value_pair(src, pos, parse_float)
|
1249
|
+
key_parent, key_stem = key[:-1], key[-1]
|
1250
|
+
if flags.is_(key, TomlFlags.FROZEN):
|
1251
|
+
raise toml_suffixed_err(src, pos, f'Cannot mutate immutable namespace {key}')
|
1252
|
+
try:
|
1253
|
+
nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
|
1254
|
+
except KeyError:
|
1255
|
+
raise toml_suffixed_err(src, pos, 'Cannot overwrite a value') from None
|
1256
|
+
if key_stem in nest:
|
1257
|
+
raise toml_suffixed_err(src, pos, f'Duplicate inline table key {key_stem!r}')
|
1258
|
+
nest[key_stem] = value
|
1259
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1260
|
+
c = src[pos:pos + 1]
|
1261
|
+
if c == '}':
|
1262
|
+
return pos + 1, nested_dict.dict
|
1263
|
+
if c != ',':
|
1264
|
+
raise toml_suffixed_err(src, pos, 'Unclosed inline table')
|
1265
|
+
if isinstance(value, (dict, list)):
|
1266
|
+
flags.set(key, TomlFlags.FROZEN, recursive=True)
|
1267
|
+
pos += 1
|
1268
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1270
1269
|
|
1271
|
-
Uid = ta.NewType('Uid', int)
|
1272
|
-
Gid = ta.NewType('Gid', int)
|
1273
1270
|
|
1271
|
+
def toml_parse_basic_str_escape(
|
1272
|
+
src: str,
|
1273
|
+
pos: TomlPos,
|
1274
|
+
*,
|
1275
|
+
multiline: bool = False,
|
1276
|
+
) -> ta.Tuple[TomlPos, str]:
|
1277
|
+
escape_id = src[pos:pos + 2]
|
1278
|
+
pos += 2
|
1279
|
+
if multiline and escape_id in {'\\ ', '\\\t', '\\\n'}:
|
1280
|
+
# Skip whitespace until next non-whitespace character or end of the doc. Error if non-whitespace is found before
|
1281
|
+
# newline.
|
1282
|
+
if escape_id != '\\\n':
|
1283
|
+
pos = toml_skip_chars(src, pos, TOML_WS)
|
1284
|
+
try:
|
1285
|
+
char = src[pos]
|
1286
|
+
except IndexError:
|
1287
|
+
return pos, ''
|
1288
|
+
if char != '\n':
|
1289
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string")
|
1290
|
+
pos += 1
|
1291
|
+
pos = toml_skip_chars(src, pos, TOML_WS_AND_NEWLINE)
|
1292
|
+
return pos, ''
|
1293
|
+
if escape_id == '\\u':
|
1294
|
+
return toml_parse_hex_char(src, pos, 4)
|
1295
|
+
if escape_id == '\\U':
|
1296
|
+
return toml_parse_hex_char(src, pos, 8)
|
1297
|
+
try:
|
1298
|
+
return pos, TOML_BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
|
1299
|
+
except KeyError:
|
1300
|
+
raise toml_suffixed_err(src, pos, "Unescaped '\\' in a string") from None
|
1274
1301
|
|
1275
|
-
########################################
|
1276
|
-
# ../utils/signals.py
|
1277
1302
|
|
1303
|
+
def toml_parse_basic_str_escape_multiline(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1304
|
+
return toml_parse_basic_str_escape(src, pos, multiline=True)
|
1278
1305
|
|
1279
|
-
##
|
1280
1306
|
|
1307
|
+
def toml_parse_hex_char(src: str, pos: TomlPos, hex_len: int) -> ta.Tuple[TomlPos, str]:
|
1308
|
+
hex_str = src[pos:pos + hex_len]
|
1309
|
+
if len(hex_str) != hex_len or not TOML_HEXDIGIT_CHARS.issuperset(hex_str):
|
1310
|
+
raise toml_suffixed_err(src, pos, 'Invalid hex value')
|
1311
|
+
pos += hex_len
|
1312
|
+
hex_int = int(hex_str, 16)
|
1313
|
+
if not toml_is_unicode_scalar_value(hex_int):
|
1314
|
+
raise toml_suffixed_err(src, pos, 'Escaped character is not a Unicode scalar value')
|
1315
|
+
return pos, chr(hex_int)
|
1281
1316
|
|
1282
|
-
_SIGS_BY_NUM: ta.Mapping[int, signal.Signals] = {s.value: s for s in signal.Signals}
|
1283
|
-
_SIGS_BY_NAME: ta.Mapping[str, signal.Signals] = {s.name: s for s in signal.Signals}
|
1284
1317
|
|
1318
|
+
def toml_parse_literal_str(src: str, pos: TomlPos) -> ta.Tuple[TomlPos, str]:
|
1319
|
+
pos += 1 # Skip starting apostrophe
|
1320
|
+
start_pos = pos
|
1321
|
+
pos = toml_skip_until(
|
1322
|
+
src, pos, "'", error_on=TOML_ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True,
|
1323
|
+
)
|
1324
|
+
return pos + 1, src[start_pos:pos] # Skip ending apostrophe
|
1285
1325
|
|
1286
|
-
def sig_num(value: ta.Union[int, str]) -> int:
|
1287
|
-
try:
|
1288
|
-
num = int(value)
|
1289
1326
|
|
1290
|
-
|
1291
|
-
|
1292
|
-
|
1293
|
-
|
1327
|
+
def toml_parse_multiline_str(src: str, pos: TomlPos, *, literal: bool) -> ta.Tuple[TomlPos, str]:
|
1328
|
+
pos += 3
|
1329
|
+
if src.startswith('\n', pos):
|
1330
|
+
pos += 1
|
1294
1331
|
|
1295
|
-
|
1296
|
-
|
1297
|
-
|
1332
|
+
if literal:
|
1333
|
+
delim = "'"
|
1334
|
+
end_pos = toml_skip_until(
|
1335
|
+
src,
|
1336
|
+
pos,
|
1337
|
+
"'''",
|
1338
|
+
error_on=TOML_ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
|
1339
|
+
error_on_eof=True,
|
1340
|
+
)
|
1341
|
+
result = src[pos:end_pos]
|
1342
|
+
pos = end_pos + 3
|
1343
|
+
else:
|
1344
|
+
delim = '"'
|
1345
|
+
pos, result = toml_parse_basic_str(src, pos, multiline=True)
|
1298
1346
|
|
1299
|
-
if
|
1300
|
-
|
1347
|
+
# Add at maximum two extra apostrophes/quotes if the end sequence is 4 or 5 chars long instead of just 3.
|
1348
|
+
if not src.startswith(delim, pos):
|
1349
|
+
return pos, result
|
1350
|
+
pos += 1
|
1351
|
+
if not src.startswith(delim, pos):
|
1352
|
+
return pos, result + delim
|
1353
|
+
pos += 1
|
1354
|
+
return pos, result + (delim * 2)
|
1301
1355
|
|
1302
|
-
return num
|
1303
1356
|
|
1357
|
+
def toml_parse_basic_str(src: str, pos: TomlPos, *, multiline: bool) -> ta.Tuple[TomlPos, str]:
|
1358
|
+
if multiline:
|
1359
|
+
error_on = TOML_ILLEGAL_MULTILINE_BASIC_STR_CHARS
|
1360
|
+
parse_escapes = toml_parse_basic_str_escape_multiline
|
1361
|
+
else:
|
1362
|
+
error_on = TOML_ILLEGAL_BASIC_STR_CHARS
|
1363
|
+
parse_escapes = toml_parse_basic_str_escape
|
1364
|
+
result = ''
|
1365
|
+
start_pos = pos
|
1366
|
+
while True:
|
1367
|
+
try:
|
1368
|
+
char = src[pos]
|
1369
|
+
except IndexError:
|
1370
|
+
raise toml_suffixed_err(src, pos, 'Unterminated string') from None
|
1371
|
+
if char == '"':
|
1372
|
+
if not multiline:
|
1373
|
+
return pos + 1, result + src[start_pos:pos]
|
1374
|
+
if src.startswith('"""', pos):
|
1375
|
+
return pos + 3, result + src[start_pos:pos]
|
1376
|
+
pos += 1
|
1377
|
+
continue
|
1378
|
+
if char == '\\':
|
1379
|
+
result += src[start_pos:pos]
|
1380
|
+
pos, parsed_escape = parse_escapes(src, pos)
|
1381
|
+
result += parsed_escape
|
1382
|
+
start_pos = pos
|
1383
|
+
continue
|
1384
|
+
if char in error_on:
|
1385
|
+
raise toml_suffixed_err(src, pos, f'Illegal character {char!r}')
|
1386
|
+
pos += 1
|
1304
1387
|
|
1305
|
-
def sig_name(num: int) -> str:
|
1306
|
-
if (sig := _SIGS_BY_NUM.get(num)) is not None:
|
1307
|
-
return sig.name
|
1308
|
-
return f'signal {sig}'
|
1309
1388
|
|
1389
|
+
def toml_parse_value( # noqa: C901
|
1390
|
+
src: str,
|
1391
|
+
pos: TomlPos,
|
1392
|
+
parse_float: TomlParseFloat,
|
1393
|
+
) -> ta.Tuple[TomlPos, ta.Any]:
|
1394
|
+
try:
|
1395
|
+
char: ta.Optional[str] = src[pos]
|
1396
|
+
except IndexError:
|
1397
|
+
char = None
|
1310
1398
|
|
1311
|
-
|
1399
|
+
# IMPORTANT: order conditions based on speed of checking and likelihood
|
1312
1400
|
|
1401
|
+
# Basic strings
|
1402
|
+
if char == '"':
|
1403
|
+
if src.startswith('"""', pos):
|
1404
|
+
return toml_parse_multiline_str(src, pos, literal=False)
|
1405
|
+
return toml_parse_one_line_basic_str(src, pos)
|
1313
1406
|
|
1314
|
-
|
1315
|
-
|
1316
|
-
|
1407
|
+
# Literal strings
|
1408
|
+
if char == "'":
|
1409
|
+
if src.startswith("'''", pos):
|
1410
|
+
return toml_parse_multiline_str(src, pos, literal=True)
|
1411
|
+
return toml_parse_literal_str(src, pos)
|
1317
1412
|
|
1318
|
-
|
1413
|
+
# Booleans
|
1414
|
+
if char == 't':
|
1415
|
+
if src.startswith('true', pos):
|
1416
|
+
return pos + 4, True
|
1417
|
+
if char == 'f':
|
1418
|
+
if src.startswith('false', pos):
|
1419
|
+
return pos + 5, False
|
1319
1420
|
|
1320
|
-
|
1321
|
-
|
1322
|
-
|
1421
|
+
# Arrays
|
1422
|
+
if char == '[':
|
1423
|
+
return toml_parse_array(src, pos, parse_float)
|
1323
1424
|
|
1324
|
-
|
1325
|
-
|
1326
|
-
|
1425
|
+
# Inline tables
|
1426
|
+
if char == '{':
|
1427
|
+
return toml_parse_inline_table(src, pos, parse_float)
|
1327
1428
|
|
1328
|
-
|
1329
|
-
|
1330
|
-
|
1331
|
-
|
1332
|
-
|
1333
|
-
|
1429
|
+
# Dates and times
|
1430
|
+
datetime_match = TOML_RE_DATETIME.match(src, pos)
|
1431
|
+
if datetime_match:
|
1432
|
+
try:
|
1433
|
+
datetime_obj = toml_match_to_datetime(datetime_match)
|
1434
|
+
except ValueError as e:
|
1435
|
+
raise toml_suffixed_err(src, pos, 'Invalid date or datetime') from e
|
1436
|
+
return datetime_match.end(), datetime_obj
|
1437
|
+
localtime_match = TOML_RE_LOCALTIME.match(src, pos)
|
1438
|
+
if localtime_match:
|
1439
|
+
return localtime_match.end(), toml_match_to_localtime(localtime_match)
|
1334
1440
|
|
1441
|
+
# Integers and "normal" floats. The regex will greedily match any type starting with a decimal char, so needs to be
|
1442
|
+
# located after handling of dates and times.
|
1443
|
+
number_match = TOML_RE_NUMBER.match(src, pos)
|
1444
|
+
if number_match:
|
1445
|
+
return number_match.end(), toml_match_to_number(number_match, parse_float)
|
1335
1446
|
|
1336
|
-
|
1337
|
-
|
1447
|
+
# Special floats
|
1448
|
+
first_three = src[pos:pos + 3]
|
1449
|
+
if first_three in {'inf', 'nan'}:
|
1450
|
+
return pos + 3, parse_float(first_three)
|
1451
|
+
first_four = src[pos:pos + 4]
|
1452
|
+
if first_four in {'-inf', '+inf', '-nan', '+nan'}:
|
1453
|
+
return pos + 4, parse_float(first_four)
|
1338
1454
|
|
1455
|
+
raise toml_suffixed_err(src, pos, 'Invalid value')
|
1339
1456
|
|
1340
|
-
##
|
1341
1457
|
|
1458
|
+
def toml_suffixed_err(src: str, pos: TomlPos, msg: str) -> TomlDecodeError:
|
1459
|
+
"""Return a `TomlDecodeError` where error message is suffixed with coordinates in source."""
|
1342
1460
|
|
1343
|
-
def
|
1344
|
-
|
1345
|
-
|
1346
|
-
|
1347
|
-
|
1461
|
+
def coord_repr(src: str, pos: TomlPos) -> str:
|
1462
|
+
if pos >= len(src):
|
1463
|
+
return 'end of document'
|
1464
|
+
line = src.count('\n', 0, pos) + 1
|
1465
|
+
if line == 1:
|
1466
|
+
column = pos + 1
|
1467
|
+
else:
|
1468
|
+
column = pos - src.rindex('\n', 0, pos)
|
1469
|
+
return f'line {line}, column {column}'
|
1348
1470
|
|
1471
|
+
return TomlDecodeError(f'{msg} (at {coord_repr(src, pos)})')
|
1349
1472
|
|
1350
|
-
@ta.overload
|
1351
|
-
def find_prefix_at_end(haystack: str, needle: str) -> int:
|
1352
|
-
...
|
1353
1473
|
|
1474
|
+
def toml_is_unicode_scalar_value(codepoint: int) -> bool:
|
1475
|
+
return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
|
1354
1476
|
|
1355
|
-
@ta.overload
|
1356
|
-
def find_prefix_at_end(haystack: bytes, needle: bytes) -> int:
|
1357
|
-
...
|
1358
1477
|
|
1478
|
+
def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
|
1479
|
+
"""A decorator to make `parse_float` safe.
|
1359
1480
|
|
1360
|
-
|
1361
|
-
|
1362
|
-
|
1363
|
-
|
1364
|
-
|
1481
|
+
`parse_float` must not return dicts or lists, because these types would be mixed with parsed TOML tables and arrays,
|
1482
|
+
thus confusing the parser. The returned decorated callable raises `ValueError` instead of returning illegal types.
|
1483
|
+
"""
|
1484
|
+
# The default `float` callable never returns illegal types. Optimize it.
|
1485
|
+
if parse_float is float:
|
1486
|
+
return float
|
1365
1487
|
|
1488
|
+
def safe_parse_float(float_str: str) -> ta.Any:
|
1489
|
+
float_value = parse_float(float_str)
|
1490
|
+
if isinstance(float_value, (dict, list)):
|
1491
|
+
raise ValueError('parse_float must not return dicts or lists') # noqa
|
1492
|
+
return float_value
|
1366
1493
|
|
1367
|
-
|
1494
|
+
return safe_parse_float
|
1368
1495
|
|
1369
1496
|
|
1370
|
-
|
1371
|
-
|
1497
|
+
########################################
|
1498
|
+
# ../../../omlish/formats/toml/writer.py
|
1372
1499
|
|
1373
1500
|
|
1374
|
-
|
1375
|
-
|
1501
|
+
class TomlWriter:
|
1502
|
+
@dc.dataclass(frozen=True)
|
1503
|
+
class Literal:
|
1504
|
+
s: str
|
1376
1505
|
|
1377
|
-
|
1378
|
-
|
1379
|
-
|
1380
|
-
l = len(s)
|
1381
|
-
while i < l:
|
1382
|
-
if show == 0 and s[i:i + 1] in ANSI_TERMINATORS:
|
1383
|
-
show = 1
|
1384
|
-
elif show:
|
1385
|
-
n = s.find(ANSI_ESCAPE_BEGIN, i)
|
1386
|
-
if n == -1:
|
1387
|
-
return result + s[i:]
|
1388
|
-
else:
|
1389
|
-
result = result + s[i:n]
|
1390
|
-
i = n
|
1391
|
-
show = 0
|
1392
|
-
i += 1
|
1393
|
-
return result
|
1506
|
+
def __init__(self, out: ta.TextIO) -> None:
|
1507
|
+
super().__init__()
|
1508
|
+
self._out = out
|
1394
1509
|
|
1510
|
+
self._indent = 0
|
1511
|
+
self._wrote_indent = False
|
1395
1512
|
|
1396
|
-
|
1513
|
+
#
|
1397
1514
|
|
1515
|
+
def _w(self, s: str) -> None:
|
1516
|
+
if not self._wrote_indent:
|
1517
|
+
self._out.write(' ' * self._indent)
|
1518
|
+
self._wrote_indent = True
|
1519
|
+
self._out.write(s)
|
1398
1520
|
|
1399
|
-
|
1400
|
-
|
1401
|
-
|
1402
|
-
def __init__(self, d, default=1):
|
1403
|
-
super().__init__()
|
1404
|
-
self._d = d
|
1405
|
-
self._default = default
|
1406
|
-
# all keys must be the same size
|
1407
|
-
self._keysz = None
|
1408
|
-
for k in d:
|
1409
|
-
if self._keysz is None:
|
1410
|
-
self._keysz = len(k)
|
1411
|
-
elif self._keysz != len(k): # type: ignore
|
1412
|
-
raise ValueError(k)
|
1521
|
+
def _nl(self) -> None:
|
1522
|
+
self._out.write('\n')
|
1523
|
+
self._wrote_indent = False
|
1413
1524
|
|
1414
|
-
def
|
1415
|
-
|
1416
|
-
|
1417
|
-
|
1418
|
-
|
1419
|
-
|
1420
|
-
return int(v[:-self._keysz]) * m # type: ignore
|
1421
|
-
return int(v) * self._default
|
1525
|
+
def _needs_quote(self, s: str) -> bool:
|
1526
|
+
return (
|
1527
|
+
not s or
|
1528
|
+
any(c in s for c in '\'"\n') or
|
1529
|
+
s[0] not in string.ascii_letters
|
1530
|
+
)
|
1422
1531
|
|
1532
|
+
def _maybe_quote(self, s: str) -> str:
|
1533
|
+
if self._needs_quote(s):
|
1534
|
+
return repr(s)
|
1535
|
+
else:
|
1536
|
+
return s
|
1423
1537
|
|
1424
|
-
|
1425
|
-
'kb': 1024,
|
1426
|
-
'mb': 1024 * 1024,
|
1427
|
-
'gb': 1024 * 1024 * 1024,
|
1428
|
-
})
|
1538
|
+
#
|
1429
1539
|
|
1540
|
+
def write_root(self, obj: ta.Mapping) -> None:
|
1541
|
+
for i, (k, v) in enumerate(obj.items()):
|
1542
|
+
if i:
|
1543
|
+
self._nl()
|
1544
|
+
self._w('[')
|
1545
|
+
self._w(self._maybe_quote(k))
|
1546
|
+
self._w(']')
|
1547
|
+
self._nl()
|
1548
|
+
self.write_table_contents(v)
|
1549
|
+
|
1550
|
+
def write_table_contents(self, obj: ta.Mapping) -> None:
|
1551
|
+
for k, v in obj.items():
|
1552
|
+
self.write_key(k)
|
1553
|
+
self._w(' = ')
|
1554
|
+
self.write_value(v)
|
1555
|
+
self._nl()
|
1556
|
+
|
1557
|
+
def write_array(self, obj: ta.Sequence) -> None:
|
1558
|
+
self._w('[')
|
1559
|
+
self._nl()
|
1560
|
+
self._indent += 1
|
1561
|
+
for e in obj:
|
1562
|
+
self.write_value(e)
|
1563
|
+
self._w(',')
|
1564
|
+
self._nl()
|
1565
|
+
self._indent -= 1
|
1566
|
+
self._w(']')
|
1567
|
+
|
1568
|
+
def write_inline_table(self, obj: ta.Mapping) -> None:
|
1569
|
+
self._w('{')
|
1570
|
+
for i, (k, v) in enumerate(obj.items()):
|
1571
|
+
if i:
|
1572
|
+
self._w(', ')
|
1573
|
+
self.write_key(k)
|
1574
|
+
self._w(' = ')
|
1575
|
+
self.write_value(v)
|
1576
|
+
self._w('}')
|
1577
|
+
|
1578
|
+
def write_inline_array(self, obj: ta.Sequence) -> None:
|
1579
|
+
self._w('[')
|
1580
|
+
for i, e in enumerate(obj):
|
1581
|
+
if i:
|
1582
|
+
self._w(', ')
|
1583
|
+
self.write_value(e)
|
1584
|
+
self._w(']')
|
1585
|
+
|
1586
|
+
def write_key(self, obj: ta.Any) -> None:
|
1587
|
+
if isinstance(obj, TomlWriter.Literal):
|
1588
|
+
self._w(obj.s)
|
1589
|
+
elif isinstance(obj, str):
|
1590
|
+
self._w(self._maybe_quote(obj.replace('_', '-')))
|
1591
|
+
elif isinstance(obj, int):
|
1592
|
+
self._w(repr(str(obj)))
|
1593
|
+
else:
|
1594
|
+
raise TypeError(obj)
|
1430
1595
|
|
1431
|
-
|
1596
|
+
def write_value(self, obj: ta.Any) -> None:
|
1597
|
+
if isinstance(obj, bool):
|
1598
|
+
self._w(str(obj).lower())
|
1599
|
+
elif isinstance(obj, (str, int, float)):
|
1600
|
+
self._w(repr(obj))
|
1601
|
+
elif isinstance(obj, ta.Mapping):
|
1602
|
+
self.write_inline_table(obj)
|
1603
|
+
elif isinstance(obj, ta.Sequence):
|
1604
|
+
if not obj:
|
1605
|
+
self.write_inline_array(obj)
|
1606
|
+
else:
|
1607
|
+
self.write_array(obj)
|
1608
|
+
else:
|
1609
|
+
raise TypeError(obj)
|
1432
1610
|
|
1611
|
+
#
|
1433
1612
|
|
1434
|
-
|
1435
|
-
|
1436
|
-
|
1437
|
-
|
1438
|
-
return
|
1439
|
-
except (TypeError, ValueError):
|
1440
|
-
raise ValueError(f'{arg} can not be converted to an octal type') # noqa
|
1613
|
+
@classmethod
|
1614
|
+
def write_str(cls, obj: ta.Any) -> str:
|
1615
|
+
out = io.StringIO()
|
1616
|
+
cls(out).write_value(obj)
|
1617
|
+
return out.getvalue()
|
1441
1618
|
|
1442
1619
|
|
1443
1620
|
########################################
|
@@ -3069,61 +3246,338 @@ def waitpid(
|
|
3069
3246
|
##
|
3070
3247
|
|
3071
3248
|
|
3072
|
-
def name_to_uid(name: str) -> Uid:
|
3073
|
-
try:
|
3074
|
-
uid = int(name)
|
3075
|
-
except ValueError:
|
3076
|
-
try:
|
3077
|
-
pwdrec = pwd.getpwnam(name)
|
3078
|
-
except KeyError:
|
3079
|
-
raise ValueError(f'Invalid user name {name}') # noqa
|
3080
|
-
uid = pwdrec[2]
|
3081
|
-
else:
|
3082
|
-
try:
|
3083
|
-
pwd.getpwuid(uid) # check if uid is valid
|
3084
|
-
except KeyError:
|
3085
|
-
raise ValueError(f'Invalid user id {name}') # noqa
|
3086
|
-
return Uid(uid)
|
3249
|
+
def name_to_uid(name: str) -> Uid:
|
3250
|
+
try:
|
3251
|
+
uid = int(name)
|
3252
|
+
except ValueError:
|
3253
|
+
try:
|
3254
|
+
pwdrec = pwd.getpwnam(name)
|
3255
|
+
except KeyError:
|
3256
|
+
raise ValueError(f'Invalid user name {name}') # noqa
|
3257
|
+
uid = pwdrec[2]
|
3258
|
+
else:
|
3259
|
+
try:
|
3260
|
+
pwd.getpwuid(uid) # check if uid is valid
|
3261
|
+
except KeyError:
|
3262
|
+
raise ValueError(f'Invalid user id {name}') # noqa
|
3263
|
+
return Uid(uid)
|
3264
|
+
|
3265
|
+
|
3266
|
+
def name_to_gid(name: str) -> Gid:
|
3267
|
+
try:
|
3268
|
+
gid = int(name)
|
3269
|
+
except ValueError:
|
3270
|
+
try:
|
3271
|
+
grprec = grp.getgrnam(name)
|
3272
|
+
except KeyError:
|
3273
|
+
raise ValueError(f'Invalid group name {name}') # noqa
|
3274
|
+
gid = grprec[2]
|
3275
|
+
else:
|
3276
|
+
try:
|
3277
|
+
grp.getgrgid(gid) # check if gid is valid
|
3278
|
+
except KeyError:
|
3279
|
+
raise ValueError(f'Invalid group id {name}') # noqa
|
3280
|
+
return Gid(gid)
|
3281
|
+
|
3282
|
+
|
3283
|
+
def gid_for_uid(uid: Uid) -> Gid:
|
3284
|
+
pwrec = pwd.getpwuid(uid)
|
3285
|
+
return Gid(pwrec[3])
|
3286
|
+
|
3287
|
+
|
3288
|
+
##
|
3289
|
+
|
3290
|
+
|
3291
|
+
@dc.dataclass(frozen=True)
|
3292
|
+
class User:
|
3293
|
+
name: str
|
3294
|
+
uid: Uid
|
3295
|
+
gid: Gid
|
3296
|
+
|
3297
|
+
|
3298
|
+
def get_user(name: str) -> User:
|
3299
|
+
return User(
|
3300
|
+
name=name,
|
3301
|
+
uid=(uid := name_to_uid(name)),
|
3302
|
+
gid=gid_for_uid(uid),
|
3303
|
+
)
|
3304
|
+
|
3305
|
+
|
3306
|
+
########################################
|
3307
|
+
# ../../../omlish/configs/formats.py
|
3308
|
+
"""
|
3309
|
+
Notes:
|
3310
|
+
- necessarily string-oriented
|
3311
|
+
- single file, as this is intended to be amalg'd and thus all included anyway
|
3312
|
+
|
3313
|
+
TODO:
|
3314
|
+
- ConfigDataMapper? to_map -> ConfigMap?
|
3315
|
+
- nginx ?
|
3316
|
+
- raw ?
|
3317
|
+
"""
|
3318
|
+
|
3319
|
+
|
3320
|
+
##
|
3321
|
+
|
3322
|
+
|
3323
|
+
@dc.dataclass(frozen=True)
|
3324
|
+
class ConfigData(abc.ABC): # noqa
|
3325
|
+
@abc.abstractmethod
|
3326
|
+
def as_map(self) -> ConfigMap:
|
3327
|
+
raise NotImplementedError
|
3328
|
+
|
3329
|
+
|
3330
|
+
#
|
3331
|
+
|
3332
|
+
|
3333
|
+
class ConfigLoader(abc.ABC, ta.Generic[ConfigDataT]):
|
3334
|
+
@property
|
3335
|
+
def file_exts(self) -> ta.Sequence[str]:
|
3336
|
+
return ()
|
3337
|
+
|
3338
|
+
def match_file(self, n: str) -> bool:
|
3339
|
+
return '.' in n and n.split('.')[-1] in check.not_isinstance(self.file_exts, str)
|
3340
|
+
|
3341
|
+
#
|
3342
|
+
|
3343
|
+
def load_file(self, p: str) -> ConfigDataT:
|
3344
|
+
with open(p) as f:
|
3345
|
+
return self.load_str(f.read())
|
3346
|
+
|
3347
|
+
@abc.abstractmethod
|
3348
|
+
def load_str(self, s: str) -> ConfigDataT:
|
3349
|
+
raise NotImplementedError
|
3350
|
+
|
3351
|
+
|
3352
|
+
#
|
3353
|
+
|
3354
|
+
|
3355
|
+
class ConfigRenderer(abc.ABC, ta.Generic[ConfigDataT]):
|
3356
|
+
@property
|
3357
|
+
@abc.abstractmethod
|
3358
|
+
def data_cls(self) -> ta.Type[ConfigDataT]:
|
3359
|
+
raise NotImplementedError
|
3360
|
+
|
3361
|
+
def match_data(self, d: ConfigDataT) -> bool:
|
3362
|
+
return isinstance(d, self.data_cls)
|
3363
|
+
|
3364
|
+
#
|
3365
|
+
|
3366
|
+
@abc.abstractmethod
|
3367
|
+
def render(self, d: ConfigDataT) -> str:
|
3368
|
+
raise NotImplementedError
|
3369
|
+
|
3370
|
+
|
3371
|
+
##
|
3372
|
+
|
3373
|
+
|
3374
|
+
@dc.dataclass(frozen=True)
|
3375
|
+
class ObjConfigData(ConfigData, abc.ABC):
|
3376
|
+
obj: ta.Any
|
3377
|
+
|
3378
|
+
def as_map(self) -> ConfigMap:
|
3379
|
+
return check.isinstance(self.obj, collections.abc.Mapping)
|
3380
|
+
|
3381
|
+
|
3382
|
+
##
|
3383
|
+
|
3384
|
+
|
3385
|
+
@dc.dataclass(frozen=True)
|
3386
|
+
class JsonConfigData(ObjConfigData):
|
3387
|
+
pass
|
3388
|
+
|
3389
|
+
|
3390
|
+
class JsonConfigLoader(ConfigLoader[JsonConfigData]):
|
3391
|
+
file_exts = ('json',)
|
3392
|
+
|
3393
|
+
def load_str(self, s: str) -> JsonConfigData:
|
3394
|
+
return JsonConfigData(json.loads(s))
|
3395
|
+
|
3396
|
+
|
3397
|
+
class JsonConfigRenderer(ConfigRenderer[JsonConfigData]):
|
3398
|
+
data_cls = JsonConfigData
|
3399
|
+
|
3400
|
+
def render(self, d: JsonConfigData) -> str:
|
3401
|
+
return json_dumps_pretty(d.obj)
|
3402
|
+
|
3403
|
+
|
3404
|
+
##
|
3405
|
+
|
3406
|
+
|
3407
|
+
@dc.dataclass(frozen=True)
|
3408
|
+
class TomlConfigData(ObjConfigData):
|
3409
|
+
pass
|
3410
|
+
|
3411
|
+
|
3412
|
+
class TomlConfigLoader(ConfigLoader[TomlConfigData]):
|
3413
|
+
file_exts = ('toml',)
|
3414
|
+
|
3415
|
+
def load_str(self, s: str) -> TomlConfigData:
|
3416
|
+
return TomlConfigData(toml_loads(s))
|
3417
|
+
|
3418
|
+
|
3419
|
+
class TomlConfigRenderer(ConfigRenderer[TomlConfigData]):
|
3420
|
+
data_cls = TomlConfigData
|
3421
|
+
|
3422
|
+
def render(self, d: TomlConfigData) -> str:
|
3423
|
+
return TomlWriter.write_str(d.obj)
|
3424
|
+
|
3425
|
+
|
3426
|
+
##
|
3427
|
+
|
3428
|
+
|
3429
|
+
@dc.dataclass(frozen=True)
|
3430
|
+
class YamlConfigData(ObjConfigData):
|
3431
|
+
pass
|
3432
|
+
|
3433
|
+
|
3434
|
+
class YamlConfigLoader(ConfigLoader[YamlConfigData]):
|
3435
|
+
file_exts = ('yaml', 'yml')
|
3436
|
+
|
3437
|
+
def load_str(self, s: str) -> YamlConfigData:
|
3438
|
+
return YamlConfigData(__import__('yaml').safe_load(s))
|
3439
|
+
|
3440
|
+
|
3441
|
+
class YamlConfigRenderer(ConfigRenderer[YamlConfigData]):
|
3442
|
+
data_cls = YamlConfigData
|
3443
|
+
|
3444
|
+
def render(self, d: YamlConfigData) -> str:
|
3445
|
+
return __import__('yaml').safe_dump(d.obj)
|
3446
|
+
|
3447
|
+
|
3448
|
+
##
|
3449
|
+
|
3450
|
+
|
3451
|
+
@dc.dataclass(frozen=True)
|
3452
|
+
class IniConfigData(ConfigData):
|
3453
|
+
sections: IniSectionSettingsMap
|
3454
|
+
|
3455
|
+
def as_map(self) -> ConfigMap:
|
3456
|
+
return self.sections
|
3457
|
+
|
3458
|
+
|
3459
|
+
class IniConfigLoader(ConfigLoader[IniConfigData]):
|
3460
|
+
file_exts = ('ini',)
|
3461
|
+
|
3462
|
+
def load_str(self, s: str) -> IniConfigData:
|
3463
|
+
cp = configparser.ConfigParser()
|
3464
|
+
cp.read_string(s)
|
3465
|
+
return IniConfigData(extract_ini_sections(cp))
|
3466
|
+
|
3467
|
+
|
3468
|
+
class IniConfigRenderer(ConfigRenderer[IniConfigData]):
|
3469
|
+
data_cls = IniConfigData
|
3470
|
+
|
3471
|
+
def render(self, d: IniConfigData) -> str:
|
3472
|
+
return render_ini_sections(d.sections)
|
3473
|
+
|
3474
|
+
|
3475
|
+
##
|
3476
|
+
|
3477
|
+
|
3478
|
+
@dc.dataclass(frozen=True)
|
3479
|
+
class SwitchedConfigFileLoader:
|
3480
|
+
loaders: ta.Sequence[ConfigLoader]
|
3481
|
+
default: ta.Optional[ConfigLoader] = None
|
3482
|
+
|
3483
|
+
def load_file(self, p: str) -> ConfigData:
|
3484
|
+
n = os.path.basename(p)
|
3485
|
+
|
3486
|
+
for l in self.loaders:
|
3487
|
+
if l.match_file(n):
|
3488
|
+
return l.load_file(p)
|
3489
|
+
|
3490
|
+
if (d := self.default) is not None:
|
3491
|
+
return d.load_file(p)
|
3492
|
+
|
3493
|
+
raise NameError(n)
|
3494
|
+
|
3495
|
+
|
3496
|
+
DEFAULT_CONFIG_LOADERS: ta.Sequence[ConfigLoader] = [
|
3497
|
+
JsonConfigLoader(),
|
3498
|
+
TomlConfigLoader(),
|
3499
|
+
YamlConfigLoader(),
|
3500
|
+
IniConfigLoader(),
|
3501
|
+
]
|
3502
|
+
|
3503
|
+
DEFAULT_CONFIG_LOADER: ConfigLoader = JsonConfigLoader()
|
3504
|
+
|
3505
|
+
DEFAULT_CONFIG_FILE_LOADER = SwitchedConfigFileLoader(
|
3506
|
+
loaders=DEFAULT_CONFIG_LOADERS,
|
3507
|
+
default=DEFAULT_CONFIG_LOADER,
|
3508
|
+
)
|
3509
|
+
|
3510
|
+
|
3511
|
+
##
|
3512
|
+
|
3513
|
+
|
3514
|
+
@dc.dataclass(frozen=True)
|
3515
|
+
class SwitchedConfigRenderer:
|
3516
|
+
renderers: ta.Sequence[ConfigRenderer]
|
3517
|
+
|
3518
|
+
def render(self, d: ConfigData) -> str:
|
3519
|
+
for r in self.renderers:
|
3520
|
+
if r.match_data(d):
|
3521
|
+
return r.render(d)
|
3522
|
+
raise TypeError(d)
|
3523
|
+
|
3087
3524
|
|
3525
|
+
DEFAULT_CONFIG_RENDERERS: ta.Sequence[ConfigRenderer] = [
|
3526
|
+
JsonConfigRenderer(),
|
3527
|
+
TomlConfigRenderer(),
|
3528
|
+
YamlConfigRenderer(),
|
3529
|
+
IniConfigRenderer(),
|
3530
|
+
]
|
3088
3531
|
|
3089
|
-
|
3090
|
-
try:
|
3091
|
-
gid = int(name)
|
3092
|
-
except ValueError:
|
3093
|
-
try:
|
3094
|
-
grprec = grp.getgrnam(name)
|
3095
|
-
except KeyError:
|
3096
|
-
raise ValueError(f'Invalid group name {name}') # noqa
|
3097
|
-
gid = grprec[2]
|
3098
|
-
else:
|
3099
|
-
try:
|
3100
|
-
grp.getgrgid(gid) # check if gid is valid
|
3101
|
-
except KeyError:
|
3102
|
-
raise ValueError(f'Invalid group id {name}') # noqa
|
3103
|
-
return Gid(gid)
|
3532
|
+
DEFAULT_CONFIG_RENDERER = SwitchedConfigRenderer(DEFAULT_CONFIG_RENDERERS)
|
3104
3533
|
|
3105
3534
|
|
3106
|
-
|
3107
|
-
|
3108
|
-
|
3535
|
+
########################################
|
3536
|
+
# ../../../omlish/configs/processing/names.py
|
3537
|
+
"""
|
3538
|
+
usecase: supervisor process groups
|
3539
|
+
"""
|
3109
3540
|
|
3110
3541
|
|
3111
3542
|
##
|
3112
3543
|
|
3113
3544
|
|
3114
|
-
|
3115
|
-
|
3116
|
-
|
3117
|
-
|
3118
|
-
|
3545
|
+
def build_config_named_children(
|
3546
|
+
o: ta.Union[
|
3547
|
+
ta.Sequence[ConfigMap],
|
3548
|
+
ta.Mapping[str, ConfigMap],
|
3549
|
+
None,
|
3550
|
+
],
|
3551
|
+
*,
|
3552
|
+
name_key: str = 'name',
|
3553
|
+
) -> ta.Optional[ta.Sequence[ConfigMap]]:
|
3554
|
+
if o is None:
|
3555
|
+
return None
|
3119
3556
|
|
3557
|
+
lst: ta.List[ConfigMap] = []
|
3558
|
+
if isinstance(o, ta.Mapping):
|
3559
|
+
for k, v in o.items():
|
3560
|
+
check.isinstance(v, ta.Mapping)
|
3561
|
+
if name_key in v:
|
3562
|
+
n = v[name_key]
|
3563
|
+
if k != n:
|
3564
|
+
raise KeyError(f'Given names do not match: {n} != {k}')
|
3565
|
+
lst.append(v)
|
3566
|
+
else:
|
3567
|
+
lst.append({name_key: k, **v})
|
3120
3568
|
|
3121
|
-
|
3122
|
-
|
3123
|
-
|
3124
|
-
|
3125
|
-
|
3126
|
-
|
3569
|
+
else:
|
3570
|
+
check.not_isinstance(o, str)
|
3571
|
+
lst.extend(o)
|
3572
|
+
|
3573
|
+
seen = set()
|
3574
|
+
for d in lst:
|
3575
|
+
n = d['name']
|
3576
|
+
if n in d:
|
3577
|
+
raise KeyError(f'Duplicate name: {n}')
|
3578
|
+
seen.add(n)
|
3579
|
+
|
3580
|
+
return lst
|
3127
3581
|
|
3128
3582
|
|
3129
3583
|
########################################
|
@@ -5858,119 +6312,303 @@ class SocketHandler(abc.ABC):
|
|
5858
6312
|
raise NotImplementedError
|
5859
6313
|
|
5860
6314
|
|
5861
|
-
########################################
|
5862
|
-
#
|
6315
|
+
########################################
|
6316
|
+
# ../configs.py
|
6317
|
+
|
6318
|
+
|
6319
|
+
##
|
6320
|
+
|
6321
|
+
|
6322
|
+
class RestartWhenExitUnexpected:
|
6323
|
+
pass
|
6324
|
+
|
6325
|
+
|
6326
|
+
class RestartUnconditionally:
|
6327
|
+
pass
|
6328
|
+
|
6329
|
+
|
6330
|
+
##
|
6331
|
+
|
6332
|
+
|
6333
|
+
@dc.dataclass(frozen=True)
|
6334
|
+
class ProcessConfig:
|
6335
|
+
# A Python string expression that is used to compose the supervisor process name for this process. You usually don't
|
6336
|
+
# need to worry about setting this unless you change numprocs. The string expression is evaluated against a
|
6337
|
+
# dictionary that includes group_name, host_node_name, process_num, program_name, and here (the directory of the
|
6338
|
+
# supervisord config file).
|
6339
|
+
name: str
|
6340
|
+
|
6341
|
+
# The command that will be run when this program is started. The command can be either absolute (e.g.
|
6342
|
+
# /path/to/programname) or relative (e.g. programname). If it is relative, the supervisord's environment $PATH will
|
6343
|
+
# be searched for the executable. Programs can accept arguments, e.g. /path/to/program foo bar. The command line can
|
6344
|
+
# use double quotes to group arguments with spaces in them to pass to the program, e.g. /path/to/program/name -p
|
6345
|
+
# "foo bar". Note that the value of command may include Python string expressions, e.g. /path/to/programname
|
6346
|
+
# --port=80%(process_num)02d might expand to /path/to/programname --port=8000 at runtime. String expressions are
|
6347
|
+
# evaluated against a dictionary containing the keys group_name, host_node_name, program_name, process_num,
|
6348
|
+
# numprocs, here (the directory of the supervisord config file), and all supervisord's environment variables
|
6349
|
+
# prefixed with ENV_. Controlled programs should themselves not be daemons, as supervisord assumes it is responsible
|
6350
|
+
# for daemonizing its subprocesses
|
6351
|
+
command: str
|
6352
|
+
|
6353
|
+
#
|
6354
|
+
|
6355
|
+
# Supervisor will start as many instances of this program as named by numprocs. Note that if numprocs > 1, the
|
6356
|
+
# process_name expression must include %(process_num)s (or any other valid Python string expression that includes
|
6357
|
+
# process_num) within it.
|
6358
|
+
num_procs: int = 1
|
6359
|
+
|
6360
|
+
# An integer offset that is used to compute the number at which process_num starts.
|
6361
|
+
num_procs_start: int = 0
|
6362
|
+
|
6363
|
+
#
|
6364
|
+
|
6365
|
+
# Instruct supervisord to use this UNIX user account as the account which runs the program. The user can only be
|
6366
|
+
# switched if supervisord is run as the root user. If supervisord can't switch to the specified user, the program
|
6367
|
+
# will not be started.
|
6368
|
+
#
|
6369
|
+
# Note: The user will be changed using setuid only. This does not start a login shell and does not change
|
6370
|
+
# environment variables like USER or HOME
|
6371
|
+
user: ta.Optional[str] = None
|
6372
|
+
uid: ta.Optional[int] = None
|
6373
|
+
|
6374
|
+
# An octal number (e.g. 002, 022) representing the umask of the process.
|
6375
|
+
umask: ta.Optional[int] = None
|
6376
|
+
|
6377
|
+
#
|
6378
|
+
|
6379
|
+
# A file path representing a directory to which supervisord should temporarily chdir before exec'ing the child.
|
6380
|
+
directory: ta.Optional[str] = None
|
6381
|
+
|
6382
|
+
# A list of key/value pairs in the form KEY="val",KEY2="val2" that will be placed in the child process' environment.
|
6383
|
+
# The environment string may contain Python string expressions that will be evaluated against a dictionary
|
6384
|
+
# containing group_name, host_node_name, process_num, program_name, and here (the directory of the supervisord
|
6385
|
+
# config file). Values containing non-alphanumeric characters should be quoted (e.g. KEY="val:123",KEY2="val,456").
|
6386
|
+
# Otherwise, quoting the values is optional but recommended. Note that the subprocess will inherit the environment
|
6387
|
+
# variables of the shell used to start “supervisord” except for the ones overridden here.
|
6388
|
+
environment: ta.Optional[ta.Mapping[str, str]] = None
|
6389
|
+
|
6390
|
+
#
|
6391
|
+
|
6392
|
+
# The relative priority of the program in the start and shutdown ordering. Lower priorities indicate programs that
|
6393
|
+
# start first and shut down last at startup and when aggregate commands are used in various clients (e.g. “start
|
6394
|
+
# all”/”stop all”). Higher priorities indicate programs that start last and shut down first.
|
6395
|
+
priority: int = 999
|
6396
|
+
|
6397
|
+
# If true, this program will start automatically when supervisord is started.
|
6398
|
+
auto_start: bool = True
|
6399
|
+
|
6400
|
+
# Specifies if supervisord should automatically restart a process if it exits when it is in the RUNNING state. May
|
6401
|
+
# be one of false, unexpected, or true. If false, the process will not be autorestarted. If unexpected, the process
|
6402
|
+
# will be restarted when the program exits with an exit code that is not one of the exit codes associated with this
|
6403
|
+
# process' configuration (see exitcodes). If true, the process will be unconditionally restarted when it exits,
|
6404
|
+
# without regard to its exit code.
|
6405
|
+
#
|
6406
|
+
# Note: autorestart controls whether supervisord will autorestart a program if it exits after it has successfully
|
6407
|
+
# started up (the process is in the RUNNING state). supervisord has a different restart mechanism for when the
|
6408
|
+
# process is starting up (the process is in the STARTING state). Retries during process startup are controlled by
|
6409
|
+
# startsecs and startretries.
|
6410
|
+
auto_restart: str = 'unexpected'
|
6411
|
+
|
6412
|
+
# The total number of seconds which the program needs to stay running after a startup to consider the start
|
6413
|
+
# successful (moving the process from the STARTING state to the RUNNING state). Set to 0 to indicate that the
|
6414
|
+
# program needn't stay running for any particular amount of time.
|
6415
|
+
#
|
6416
|
+
# Note: Even if a process exits with an “expected” exit code (see exitcodes), the start will still be considered a
|
6417
|
+
# failure if the process exits quicker than startsecs.
|
6418
|
+
start_secs: int = 1
|
6419
|
+
|
6420
|
+
# The number of serial failure attempts that supervisord will allow when attempting to start the program before
|
6421
|
+
# giving up and putting the process into an FATAL state.
|
6422
|
+
#
|
6423
|
+
# Note: After each failed restart, process will be put in BACKOFF state and each retry attempt will take
|
6424
|
+
# increasingly more time.
|
6425
|
+
start_retries: int = 3
|
6426
|
+
|
6427
|
+
# The signal used to kill the program when a stop is requested. This can be specified using the signal's name or its
|
6428
|
+
# number. It is normally one of: TERM, HUP, INT, QUIT, KILL, USR1, or USR2.
|
6429
|
+
stop_signal: int = signal.SIGTERM
|
6430
|
+
|
6431
|
+
# The number of seconds to wait for the OS to return a SIGCHLD to supervisord after the program has been sent a
|
6432
|
+
# stopsignal. If this number of seconds elapses before supervisord receives a SIGCHLD from the process, supervisord
|
6433
|
+
# will attempt to kill it with a final SIGKILL.
|
6434
|
+
stop_wait_secs: int = 10
|
6435
|
+
|
6436
|
+
# If true, the flag causes supervisor to send the stop signal to the whole process group and implies killasgroup is
|
6437
|
+
# true. This is useful for programs, such as Flask in debug mode, that do not propagate stop signals to their
|
6438
|
+
# children, leaving them orphaned.
|
6439
|
+
stop_as_group: bool = False
|
6440
|
+
|
6441
|
+
# If true, when resorting to send SIGKILL to the program to terminate it send it to its whole process group instead,
|
6442
|
+
# taking care of its children as well, useful e.g with Python programs using multiprocessing.
|
6443
|
+
kill_as_group: bool = False
|
6444
|
+
|
6445
|
+
# The list of “expected” exit codes for this program used with autorestart. If the autorestart parameter is set to
|
6446
|
+
# unexpected, and the process exits in any other way than as a result of a supervisor stop request, supervisord will
|
6447
|
+
# restart the process if it exits with an exit code that is not defined in this list.
|
6448
|
+
#
|
6449
|
+
# Note: In Supervisor versions prior to 4.0, the default was 0,2. In Supervisor 4.0, the default was changed to 0.
|
6450
|
+
exitcodes: ta.Sequence[int] = (0,)
|
6451
|
+
|
6452
|
+
#
|
6453
|
+
|
6454
|
+
@dc.dataclass(frozen=True)
|
6455
|
+
class Log:
|
6456
|
+
file: ta.Optional[str] = None
|
6457
|
+
capture_max_bytes: ta.Optional[int] = None
|
6458
|
+
events_enabled: bool = False
|
6459
|
+
syslog: bool = False
|
6460
|
+
backups: ta.Optional[int] = None
|
6461
|
+
max_bytes: ta.Optional[int] = None
|
6462
|
+
|
6463
|
+
stdout: Log = Log()
|
6464
|
+
stderr: Log = Log()
|
6465
|
+
|
6466
|
+
# If true, cause the process' stderr output to be sent back to supervisord on its stdout file descriptor (in UNIX
|
6467
|
+
# shell terms, this is the equivalent of executing /the/program 2>&1).
|
6468
|
+
#
|
6469
|
+
# Note: Do not set redirect_stderr=true in an [eventlistener:x] section. Eventlisteners use stdout and stdin to
|
6470
|
+
# communicate with supervisord. If stderr is redirected, output from stderr will interfere with the eventlistener
|
6471
|
+
# protocol.
|
6472
|
+
redirect_stderr: bool = False
|
6473
|
+
|
6474
|
+
|
6475
|
+
@dc.dataclass(frozen=True)
|
6476
|
+
class ProcessGroupConfig:
|
6477
|
+
name: str
|
6478
|
+
|
6479
|
+
priority: int = 999
|
6480
|
+
|
6481
|
+
processes: ta.Optional[ta.Sequence[ProcessConfig]] = None
|
6482
|
+
|
6483
|
+
|
6484
|
+
@dc.dataclass(frozen=True)
|
6485
|
+
class ServerConfig:
|
6486
|
+
# Instruct supervisord to switch users to this UNIX user account before doing any meaningful processing. The user
|
6487
|
+
# can only be switched if supervisord is started as the root user.
|
6488
|
+
user: ta.Optional[str] = None
|
6489
|
+
|
6490
|
+
# If true, supervisord will start in the foreground instead of daemonizing.
|
6491
|
+
nodaemon: bool = False
|
6492
|
+
|
6493
|
+
# The umask of the supervisord process.
|
6494
|
+
umask: int = 0o22
|
6495
|
+
|
6496
|
+
#
|
5863
6497
|
|
6498
|
+
# When supervisord daemonizes, switch to this directory. This option can include the value %(here)s, which expands
|
6499
|
+
# to the directory in which the supervisord configuration file was found.
|
6500
|
+
directory: ta.Optional[str] = None
|
5864
6501
|
|
5865
|
-
|
6502
|
+
# The location in which supervisord keeps its pid file. This option can include the value %(here)s, which expands to
|
6503
|
+
# the directory in which the supervisord configuration file was found.
|
6504
|
+
pidfile: str = 'supervisord.pid'
|
5866
6505
|
|
6506
|
+
# The identifier string for this supervisor process, used by the RPC interface.
|
6507
|
+
identifier: str = 'supervisor'
|
5867
6508
|
|
5868
|
-
|
5869
|
-
|
5870
|
-
|
5871
|
-
|
5872
|
-
if name.endswith('.toml'):
|
5873
|
-
return toml_loads(f.read())
|
6509
|
+
# The minimum number of file descriptors that must be available before supervisord will start successfully.
|
6510
|
+
min_fds: int = 1024
|
6511
|
+
# The minimum number of process descriptors that must be available before supervisord will start successfully.
|
6512
|
+
min_procs: int = 200
|
5874
6513
|
|
5875
|
-
|
5876
|
-
|
5877
|
-
return yaml.safe_load(f)
|
6514
|
+
# Prevent supervisord from clearing any existing AUTO child log files at startup time. Useful for debugging
|
6515
|
+
nocleanup: bool = False
|
5878
6516
|
|
5879
|
-
|
5880
|
-
|
5881
|
-
cp = configparser.ConfigParser()
|
5882
|
-
cp.read_file(f)
|
5883
|
-
config_dct: ta.Dict[str, ta.Any] = {}
|
5884
|
-
for sec in cp.sections():
|
5885
|
-
cd = config_dct
|
5886
|
-
for k in sec.split('.'):
|
5887
|
-
cd = cd.setdefault(k, {})
|
5888
|
-
cd.update(cp.items(sec))
|
5889
|
-
return config_dct
|
6517
|
+
# Strip all ANSI escape sequences from child log files.
|
6518
|
+
strip_ansi: bool = False
|
5890
6519
|
|
5891
|
-
|
5892
|
-
return json.loads(f.read())
|
6520
|
+
#
|
5893
6521
|
|
6522
|
+
# The path to the activity log of the supervisord process. This option can include the value %(here)s, which expands
|
6523
|
+
# to the directory in which the supervisord configuration file was found.
|
6524
|
+
logfile: str = 'supervisord.log'
|
5894
6525
|
|
5895
|
-
|
5896
|
-
|
5897
|
-
|
5898
|
-
*,
|
5899
|
-
prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
|
5900
|
-
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
5901
|
-
) -> T:
|
5902
|
-
with open(path) as cf:
|
5903
|
-
config_dct = parse_config_file(os.path.basename(path), cf)
|
6526
|
+
# The maximum number of bytes that may be consumed by the activity log file before it is rotated (suffix multipliers
|
6527
|
+
# like “KB”, “MB”, and “GB” can be used in the value). Set this value to 0 to indicate an unlimited log size.
|
6528
|
+
logfile_max_bytes: int = 50 * 1024 * 1024
|
5904
6529
|
|
5905
|
-
|
5906
|
-
|
6530
|
+
# The number of backups to keep around resulting from activity log file rotation. If set to 0, no backups will be
|
6531
|
+
# kept.
|
6532
|
+
logfile_backups: int = 10
|
5907
6533
|
|
5908
|
-
|
6534
|
+
# The logging level, dictating what is written to the supervisord activity log. One of critical, error, warn, info,
|
6535
|
+
# debug, trace, or blather. Note that at log level debug, the supervisord log file will record the stderr/stdout
|
6536
|
+
# output of its child processes and extended info about process state changes, which is useful for debugging a
|
6537
|
+
# process which isn't starting properly.
|
6538
|
+
loglevel: int = logging.INFO
|
5909
6539
|
|
6540
|
+
# The directory used for AUTO child log files. This option can include the value %(here)s, which expands to the
|
6541
|
+
# directory in which the supervisord configuration file was found.
|
6542
|
+
child_logdir: str = '/dev/null'
|
5910
6543
|
|
5911
|
-
|
6544
|
+
# If true and not daemonized, logs will not be directed to stdout.
|
6545
|
+
silent: bool = False
|
5912
6546
|
|
6547
|
+
#
|
5913
6548
|
|
5914
|
-
|
5915
|
-
o: ta.Union[
|
5916
|
-
ta.Sequence[ConfigMapping],
|
5917
|
-
ta.Mapping[str, ConfigMapping],
|
5918
|
-
None,
|
5919
|
-
],
|
5920
|
-
*,
|
5921
|
-
name_key: str = 'name',
|
5922
|
-
) -> ta.Optional[ta.Sequence[ConfigMapping]]:
|
5923
|
-
if o is None:
|
5924
|
-
return None
|
6549
|
+
groups: ta.Optional[ta.Sequence[ProcessGroupConfig]] = None
|
5925
6550
|
|
5926
|
-
|
5927
|
-
|
5928
|
-
for k, v in o.items():
|
5929
|
-
check.isinstance(v, ta.Mapping)
|
5930
|
-
if name_key in v:
|
5931
|
-
n = v[name_key]
|
5932
|
-
if k != n:
|
5933
|
-
raise KeyError(f'Given names do not match: {n} != {k}')
|
5934
|
-
lst.append(v)
|
5935
|
-
else:
|
5936
|
-
lst.append({name_key: k, **v})
|
6551
|
+
# TODO: implement - make sure to accept broken symlinks
|
6552
|
+
group_config_dirs: ta.Optional[ta.Sequence[str]] = None
|
5937
6553
|
|
5938
|
-
|
5939
|
-
check.not_isinstance(o, str)
|
5940
|
-
lst.extend(o)
|
6554
|
+
#
|
5941
6555
|
|
5942
|
-
|
5943
|
-
for d in lst:
|
5944
|
-
n = d['name']
|
5945
|
-
if n in d:
|
5946
|
-
raise KeyError(f'Duplicate name: {n}')
|
5947
|
-
seen.add(n)
|
6556
|
+
http_port: ta.Optional[int] = None
|
5948
6557
|
|
5949
|
-
|
6558
|
+
#
|
6559
|
+
|
6560
|
+
@classmethod
|
6561
|
+
def new(
|
6562
|
+
cls,
|
6563
|
+
*,
|
6564
|
+
umask: ta.Union[int, str] = 0o22,
|
6565
|
+
directory: ta.Optional[str] = None,
|
6566
|
+
logfile: str = 'supervisord.log',
|
6567
|
+
logfile_max_bytes: ta.Union[int, str] = 50 * 1024 * 1024,
|
6568
|
+
loglevel: ta.Union[int, str] = logging.INFO,
|
6569
|
+
pidfile: str = 'supervisord.pid',
|
6570
|
+
child_logdir: ta.Optional[str] = None,
|
6571
|
+
**kwargs: ta.Any,
|
6572
|
+
) -> 'ServerConfig':
|
6573
|
+
return cls(
|
6574
|
+
umask=parse_octal(umask),
|
6575
|
+
directory=check_existing_dir(directory) if directory is not None else None,
|
6576
|
+
logfile=check_path_with_existing_dir(logfile),
|
6577
|
+
logfile_max_bytes=parse_bytes_size(logfile_max_bytes),
|
6578
|
+
loglevel=parse_logging_level(loglevel),
|
6579
|
+
pidfile=check_path_with_existing_dir(pidfile),
|
6580
|
+
child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
|
6581
|
+
**kwargs,
|
6582
|
+
)
|
5950
6583
|
|
5951
6584
|
|
5952
6585
|
##
|
5953
6586
|
|
5954
6587
|
|
5955
|
-
def
|
5956
|
-
|
5957
|
-
|
5958
|
-
out
|
6588
|
+
def prepare_process_group_config(dct: ConfigMap) -> ConfigMap:
|
6589
|
+
out = dict(dct)
|
6590
|
+
out['processes'] = build_config_named_children(out.get('processes'))
|
6591
|
+
return out
|
5959
6592
|
|
5960
|
-
for i, (section, settings) in enumerate(settings_by_section.items()):
|
5961
|
-
if i:
|
5962
|
-
out.write('\n')
|
5963
6593
|
|
5964
|
-
|
6594
|
+
def prepare_server_config(dct: ta.Mapping[str, ta.Any]) -> ta.Mapping[str, ta.Any]:
|
6595
|
+
out = dict(dct)
|
6596
|
+
group_dcts = build_config_named_children(out.get('groups'))
|
6597
|
+
out['groups'] = [prepare_process_group_config(group_dct) for group_dct in group_dcts or []]
|
6598
|
+
return out
|
5965
6599
|
|
5966
|
-
for k, v in settings.items():
|
5967
|
-
if isinstance(v, str):
|
5968
|
-
out.write(f'{k}={v}\n')
|
5969
|
-
else:
|
5970
|
-
for vv in v:
|
5971
|
-
out.write(f'{k}={vv}\n')
|
5972
6600
|
|
5973
|
-
|
6601
|
+
##
|
6602
|
+
|
6603
|
+
|
6604
|
+
def parse_logging_level(value: ta.Union[str, int]) -> int:
|
6605
|
+
if isinstance(value, int):
|
6606
|
+
return value
|
6607
|
+
s = str(value).lower()
|
6608
|
+
level = logging.getLevelNamesMapping().get(s.upper())
|
6609
|
+
if level is None:
|
6610
|
+
raise ValueError(f'bad logging level name {value!r}')
|
6611
|
+
return level
|
5974
6612
|
|
5975
6613
|
|
5976
6614
|
########################################
|
@@ -6120,6 +6758,38 @@ class UnsupportedMethodHttpHandlerError(Exception):
|
|
6120
6758
|
pass
|
6121
6759
|
|
6122
6760
|
|
6761
|
+
########################################
|
6762
|
+
# ../../../omlish/lite/configs.py
|
6763
|
+
|
6764
|
+
|
6765
|
+
##
|
6766
|
+
|
6767
|
+
|
6768
|
+
def load_config_file_obj(
|
6769
|
+
f: str,
|
6770
|
+
cls: ta.Type[T],
|
6771
|
+
*,
|
6772
|
+
prepare: ta.Union[
|
6773
|
+
ta.Callable[[ConfigMap], ConfigMap],
|
6774
|
+
ta.Iterable[ta.Callable[[ConfigMap], ConfigMap]],
|
6775
|
+
] = (),
|
6776
|
+
msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
|
6777
|
+
) -> T:
|
6778
|
+
config_data = DEFAULT_CONFIG_FILE_LOADER.load_file(f)
|
6779
|
+
|
6780
|
+
config_dct = config_data.as_map()
|
6781
|
+
|
6782
|
+
if prepare is not None:
|
6783
|
+
if isinstance(prepare, ta.Iterable):
|
6784
|
+
pfs = list(prepare)
|
6785
|
+
else:
|
6786
|
+
pfs = [prepare]
|
6787
|
+
for pf in pfs:
|
6788
|
+
config_dct = pf(config_dct)
|
6789
|
+
|
6790
|
+
return msh.unmarshal_obj(config_dct, cls)
|
6791
|
+
|
6792
|
+
|
6123
6793
|
########################################
|
6124
6794
|
# ../../../omlish/logs/standard.py
|
6125
6795
|
"""
|
@@ -6242,303 +6912,184 @@ def configure_standard_logging(
|
|
6242
6912
|
return StandardConfiguredLogHandler(handler)
|
6243
6913
|
|
6244
6914
|
|
6245
|
-
########################################
|
6246
|
-
# ../
|
6247
|
-
|
6248
|
-
|
6249
|
-
##
|
6250
|
-
|
6251
|
-
|
6252
|
-
class RestartWhenExitUnexpected:
|
6253
|
-
pass
|
6254
|
-
|
6255
|
-
|
6256
|
-
class RestartUnconditionally:
|
6257
|
-
pass
|
6258
|
-
|
6259
|
-
|
6260
|
-
##
|
6261
|
-
|
6262
|
-
|
6263
|
-
@dc.dataclass(frozen=True)
|
6264
|
-
class ProcessConfig:
|
6265
|
-
# A Python string expression that is used to compose the supervisor process name for this process. You usually don't
|
6266
|
-
# need to worry about setting this unless you change numprocs. The string expression is evaluated against a
|
6267
|
-
# dictionary that includes group_name, host_node_name, process_num, program_name, and here (the directory of the
|
6268
|
-
# supervisord config file).
|
6269
|
-
name: str
|
6270
|
-
|
6271
|
-
# The command that will be run when this program is started. The command can be either absolute (e.g.
|
6272
|
-
# /path/to/programname) or relative (e.g. programname). If it is relative, the supervisord's environment $PATH will
|
6273
|
-
# be searched for the executable. Programs can accept arguments, e.g. /path/to/program foo bar. The command line can
|
6274
|
-
# use double quotes to group arguments with spaces in them to pass to the program, e.g. /path/to/program/name -p
|
6275
|
-
# "foo bar". Note that the value of command may include Python string expressions, e.g. /path/to/programname
|
6276
|
-
# --port=80%(process_num)02d might expand to /path/to/programname --port=8000 at runtime. String expressions are
|
6277
|
-
# evaluated against a dictionary containing the keys group_name, host_node_name, program_name, process_num,
|
6278
|
-
# numprocs, here (the directory of the supervisord config file), and all supervisord's environment variables
|
6279
|
-
# prefixed with ENV_. Controlled programs should themselves not be daemons, as supervisord assumes it is responsible
|
6280
|
-
# for daemonizing its subprocesses
|
6281
|
-
command: str
|
6282
|
-
|
6283
|
-
#
|
6284
|
-
|
6285
|
-
# Supervisor will start as many instances of this program as named by numprocs. Note that if numprocs > 1, the
|
6286
|
-
# process_name expression must include %(process_num)s (or any other valid Python string expression that includes
|
6287
|
-
# process_num) within it.
|
6288
|
-
num_procs: int = 1
|
6289
|
-
|
6290
|
-
# An integer offset that is used to compute the number at which process_num starts.
|
6291
|
-
num_procs_start: int = 0
|
6292
|
-
|
6293
|
-
#
|
6294
|
-
|
6295
|
-
# Instruct supervisord to use this UNIX user account as the account which runs the program. The user can only be
|
6296
|
-
# switched if supervisord is run as the root user. If supervisord can't switch to the specified user, the program
|
6297
|
-
# will not be started.
|
6298
|
-
#
|
6299
|
-
# Note: The user will be changed using setuid only. This does not start a login shell and does not change
|
6300
|
-
# environment variables like USER or HOME
|
6301
|
-
user: ta.Optional[str] = None
|
6302
|
-
uid: ta.Optional[int] = None
|
6303
|
-
|
6304
|
-
# An octal number (e.g. 002, 022) representing the umask of the process.
|
6305
|
-
umask: ta.Optional[int] = None
|
6306
|
-
|
6307
|
-
#
|
6308
|
-
|
6309
|
-
# A file path representing a directory to which supervisord should temporarily chdir before exec'ing the child.
|
6310
|
-
directory: ta.Optional[str] = None
|
6311
|
-
|
6312
|
-
# A list of key/value pairs in the form KEY="val",KEY2="val2" that will be placed in the child process' environment.
|
6313
|
-
# The environment string may contain Python string expressions that will be evaluated against a dictionary
|
6314
|
-
# containing group_name, host_node_name, process_num, program_name, and here (the directory of the supervisord
|
6315
|
-
# config file). Values containing non-alphanumeric characters should be quoted (e.g. KEY="val:123",KEY2="val,456").
|
6316
|
-
# Otherwise, quoting the values is optional but recommended. Note that the subprocess will inherit the environment
|
6317
|
-
# variables of the shell used to start “supervisord” except for the ones overridden here.
|
6318
|
-
environment: ta.Optional[ta.Mapping[str, str]] = None
|
6319
|
-
|
6320
|
-
#
|
6321
|
-
|
6322
|
-
# The relative priority of the program in the start and shutdown ordering. Lower priorities indicate programs that
|
6323
|
-
# start first and shut down last at startup and when aggregate commands are used in various clients (e.g. “start
|
6324
|
-
# all”/”stop all”). Higher priorities indicate programs that start last and shut down first.
|
6325
|
-
priority: int = 999
|
6326
|
-
|
6327
|
-
# If true, this program will start automatically when supervisord is started.
|
6328
|
-
auto_start: bool = True
|
6329
|
-
|
6330
|
-
# Specifies if supervisord should automatically restart a process if it exits when it is in the RUNNING state. May
|
6331
|
-
# be one of false, unexpected, or true. If false, the process will not be autorestarted. If unexpected, the process
|
6332
|
-
# will be restarted when the program exits with an exit code that is not one of the exit codes associated with this
|
6333
|
-
# process' configuration (see exitcodes). If true, the process will be unconditionally restarted when it exits,
|
6334
|
-
# without regard to its exit code.
|
6335
|
-
#
|
6336
|
-
# Note: autorestart controls whether supervisord will autorestart a program if it exits after it has successfully
|
6337
|
-
# started up (the process is in the RUNNING state). supervisord has a different restart mechanism for when the
|
6338
|
-
# process is starting up (the process is in the STARTING state). Retries during process startup are controlled by
|
6339
|
-
# startsecs and startretries.
|
6340
|
-
auto_restart: str = 'unexpected'
|
6915
|
+
########################################
|
6916
|
+
# ../types.py
|
6341
6917
|
|
6342
|
-
# The total number of seconds which the program needs to stay running after a startup to consider the start
|
6343
|
-
# successful (moving the process from the STARTING state to the RUNNING state). Set to 0 to indicate that the
|
6344
|
-
# program needn't stay running for any particular amount of time.
|
6345
|
-
#
|
6346
|
-
# Note: Even if a process exits with an “expected” exit code (see exitcodes), the start will still be considered a
|
6347
|
-
# failure if the process exits quicker than startsecs.
|
6348
|
-
start_secs: int = 1
|
6349
6918
|
|
6350
|
-
|
6351
|
-
# giving up and putting the process into an FATAL state.
|
6352
|
-
#
|
6353
|
-
# Note: After each failed restart, process will be put in BACKOFF state and each retry attempt will take
|
6354
|
-
# increasingly more time.
|
6355
|
-
start_retries: int = 3
|
6919
|
+
##
|
6356
6920
|
|
6357
|
-
# The signal used to kill the program when a stop is requested. This can be specified using the signal's name or its
|
6358
|
-
# number. It is normally one of: TERM, HUP, INT, QUIT, KILL, USR1, or USR2.
|
6359
|
-
stop_signal: int = signal.SIGTERM
|
6360
6921
|
|
6361
|
-
|
6362
|
-
|
6363
|
-
# will attempt to kill it with a final SIGKILL.
|
6364
|
-
stop_wait_secs: int = 10
|
6922
|
+
class ExitNow(Exception): # noqa
|
6923
|
+
pass
|
6365
6924
|
|
6366
|
-
# If true, the flag causes supervisor to send the stop signal to the whole process group and implies killasgroup is
|
6367
|
-
# true. This is useful for programs, such as Flask in debug mode, that do not propagate stop signals to their
|
6368
|
-
# children, leaving them orphaned.
|
6369
|
-
stop_as_group: bool = False
|
6370
6925
|
|
6371
|
-
|
6372
|
-
# taking care of its children as well, useful e.g with Python programs using multiprocessing.
|
6373
|
-
kill_as_group: bool = False
|
6926
|
+
ServerEpoch = ta.NewType('ServerEpoch', int)
|
6374
6927
|
|
6375
|
-
# The list of “expected” exit codes for this program used with autorestart. If the autorestart parameter is set to
|
6376
|
-
# unexpected, and the process exits in any other way than as a result of a supervisor stop request, supervisord will
|
6377
|
-
# restart the process if it exits with an exit code that is not defined in this list.
|
6378
|
-
#
|
6379
|
-
# Note: In Supervisor versions prior to 4.0, the default was 0,2. In Supervisor 4.0, the default was changed to 0.
|
6380
|
-
exitcodes: ta.Sequence[int] = (0,)
|
6381
6928
|
|
6382
|
-
|
6929
|
+
##
|
6383
6930
|
|
6384
|
-
@dc.dataclass(frozen=True)
|
6385
|
-
class Log:
|
6386
|
-
file: ta.Optional[str] = None
|
6387
|
-
capture_max_bytes: ta.Optional[int] = None
|
6388
|
-
events_enabled: bool = False
|
6389
|
-
syslog: bool = False
|
6390
|
-
backups: ta.Optional[int] = None
|
6391
|
-
max_bytes: ta.Optional[int] = None
|
6392
6931
|
|
6393
|
-
|
6394
|
-
|
6932
|
+
@functools.total_ordering
|
6933
|
+
class ConfigPriorityOrdered(abc.ABC):
|
6934
|
+
@property
|
6935
|
+
@abc.abstractmethod
|
6936
|
+
def config(self) -> ta.Any:
|
6937
|
+
raise NotImplementedError
|
6395
6938
|
|
6396
|
-
|
6397
|
-
|
6398
|
-
#
|
6399
|
-
# Note: Do not set redirect_stderr=true in an [eventlistener:x] section. Eventlisteners use stdout and stdin to
|
6400
|
-
# communicate with supervisord. If stderr is redirected, output from stderr will interfere with the eventlistener
|
6401
|
-
# protocol.
|
6402
|
-
redirect_stderr: bool = False
|
6939
|
+
def __lt__(self, other):
|
6940
|
+
return self.config.priority < other.config.priority
|
6403
6941
|
|
6942
|
+
def __eq__(self, other):
|
6943
|
+
return self.config.priority == other.config.priority
|
6404
6944
|
|
6405
|
-
@dc.dataclass(frozen=True)
|
6406
|
-
class ProcessGroupConfig:
|
6407
|
-
name: str
|
6408
6945
|
|
6409
|
-
|
6946
|
+
##
|
6410
6947
|
|
6411
|
-
processes: ta.Optional[ta.Sequence[ProcessConfig]] = None
|
6412
6948
|
|
6949
|
+
class SupervisorStateManager(abc.ABC):
|
6950
|
+
@property
|
6951
|
+
@abc.abstractmethod
|
6952
|
+
def state(self) -> SupervisorState:
|
6953
|
+
raise NotImplementedError
|
6413
6954
|
|
6414
|
-
@
|
6415
|
-
|
6416
|
-
|
6417
|
-
# can only be switched if supervisord is started as the root user.
|
6418
|
-
user: ta.Optional[str] = None
|
6955
|
+
@abc.abstractmethod
|
6956
|
+
def set_state(self, state: SupervisorState) -> None:
|
6957
|
+
raise NotImplementedError
|
6419
6958
|
|
6420
|
-
# If true, supervisord will start in the foreground instead of daemonizing.
|
6421
|
-
nodaemon: bool = False
|
6422
6959
|
|
6423
|
-
|
6424
|
-
umask: int = 0o22
|
6960
|
+
##
|
6425
6961
|
|
6426
|
-
#
|
6427
6962
|
|
6428
|
-
|
6429
|
-
|
6430
|
-
|
6963
|
+
class HasDispatchers(abc.ABC):
|
6964
|
+
@abc.abstractmethod
|
6965
|
+
def get_dispatchers(self) -> 'Dispatchers':
|
6966
|
+
raise NotImplementedError
|
6431
6967
|
|
6432
|
-
# The location in which supervisord keeps its pid file. This option can include the value %(here)s, which expands to
|
6433
|
-
# the directory in which the supervisord configuration file was found.
|
6434
|
-
pidfile: str = 'supervisord.pid'
|
6435
6968
|
|
6436
|
-
|
6437
|
-
|
6969
|
+
class ProcessDispatcher(FdioHandler, abc.ABC):
|
6970
|
+
@property
|
6971
|
+
@abc.abstractmethod
|
6972
|
+
def channel(self) -> ProcessOutputChannel:
|
6973
|
+
raise NotImplementedError
|
6438
6974
|
|
6439
|
-
|
6440
|
-
|
6441
|
-
|
6442
|
-
|
6975
|
+
@property
|
6976
|
+
@abc.abstractmethod
|
6977
|
+
def process(self) -> 'Process':
|
6978
|
+
raise NotImplementedError
|
6443
6979
|
|
6444
|
-
# Prevent supervisord from clearing any existing AUTO child log files at startup time. Useful for debugging
|
6445
|
-
nocleanup: bool = False
|
6446
6980
|
|
6447
|
-
|
6448
|
-
|
6981
|
+
class ProcessOutputDispatcher(ProcessDispatcher, abc.ABC):
|
6982
|
+
@abc.abstractmethod
|
6983
|
+
def remove_logs(self) -> None:
|
6984
|
+
raise NotImplementedError
|
6449
6985
|
|
6450
|
-
|
6986
|
+
@abc.abstractmethod
|
6987
|
+
def reopen_logs(self) -> None:
|
6988
|
+
raise NotImplementedError
|
6451
6989
|
|
6452
|
-
# The path to the activity log of the supervisord process. This option can include the value %(here)s, which expands
|
6453
|
-
# to the directory in which the supervisord configuration file was found.
|
6454
|
-
logfile: str = 'supervisord.log'
|
6455
6990
|
|
6456
|
-
|
6457
|
-
|
6458
|
-
|
6991
|
+
class ProcessInputDispatcher(ProcessDispatcher, abc.ABC):
|
6992
|
+
@abc.abstractmethod
|
6993
|
+
def write(self, chars: ta.Union[bytes, str]) -> None:
|
6994
|
+
raise NotImplementedError
|
6459
6995
|
|
6460
|
-
|
6461
|
-
|
6462
|
-
|
6996
|
+
@abc.abstractmethod
|
6997
|
+
def flush(self) -> None:
|
6998
|
+
raise NotImplementedError
|
6463
6999
|
|
6464
|
-
# The logging level, dictating what is written to the supervisord activity log. One of critical, error, warn, info,
|
6465
|
-
# debug, trace, or blather. Note that at log level debug, the supervisord log file will record the stderr/stdout
|
6466
|
-
# output of its child processes and extended info about process state changes, which is useful for debugging a
|
6467
|
-
# process which isn't starting properly.
|
6468
|
-
loglevel: int = logging.INFO
|
6469
7000
|
|
6470
|
-
|
6471
|
-
# directory in which the supervisord configuration file was found.
|
6472
|
-
child_logdir: str = '/dev/null'
|
7001
|
+
##
|
6473
7002
|
|
6474
|
-
# If true and not daemonized, logs will not be directed to stdout.
|
6475
|
-
silent: bool = False
|
6476
7003
|
|
6477
|
-
|
7004
|
+
class Process(
|
7005
|
+
ConfigPriorityOrdered,
|
7006
|
+
HasDispatchers,
|
7007
|
+
abc.ABC,
|
7008
|
+
):
|
7009
|
+
@property
|
7010
|
+
@abc.abstractmethod
|
7011
|
+
def name(self) -> str:
|
7012
|
+
raise NotImplementedError
|
6478
7013
|
|
6479
|
-
|
7014
|
+
@property
|
7015
|
+
@abc.abstractmethod
|
7016
|
+
def config(self) -> ProcessConfig:
|
7017
|
+
raise NotImplementedError
|
6480
7018
|
|
6481
|
-
|
6482
|
-
|
7019
|
+
@property
|
7020
|
+
@abc.abstractmethod
|
7021
|
+
def group(self) -> 'ProcessGroup':
|
7022
|
+
raise NotImplementedError
|
7023
|
+
|
7024
|
+
@property
|
7025
|
+
@abc.abstractmethod
|
7026
|
+
def pid(self) -> Pid:
|
7027
|
+
raise NotImplementedError
|
6483
7028
|
|
6484
7029
|
#
|
6485
7030
|
|
6486
|
-
|
7031
|
+
@abc.abstractmethod
|
7032
|
+
def finish(self, sts: Rc) -> None:
|
7033
|
+
raise NotImplementedError
|
6487
7034
|
|
6488
|
-
|
7035
|
+
@abc.abstractmethod
|
7036
|
+
def stop(self) -> ta.Optional[str]:
|
7037
|
+
raise NotImplementedError
|
6489
7038
|
|
6490
|
-
@
|
6491
|
-
def
|
6492
|
-
|
6493
|
-
|
6494
|
-
|
6495
|
-
|
6496
|
-
|
6497
|
-
|
6498
|
-
|
6499
|
-
|
6500
|
-
|
6501
|
-
|
6502
|
-
|
6503
|
-
|
6504
|
-
|
6505
|
-
|
6506
|
-
logfile=check_path_with_existing_dir(logfile),
|
6507
|
-
logfile_max_bytes=parse_bytes_size(logfile_max_bytes),
|
6508
|
-
loglevel=parse_logging_level(loglevel),
|
6509
|
-
pidfile=check_path_with_existing_dir(pidfile),
|
6510
|
-
child_logdir=child_logdir if child_logdir else tempfile.gettempdir(),
|
6511
|
-
**kwargs,
|
6512
|
-
)
|
7039
|
+
@abc.abstractmethod
|
7040
|
+
def give_up(self) -> None:
|
7041
|
+
raise NotImplementedError
|
7042
|
+
|
7043
|
+
@abc.abstractmethod
|
7044
|
+
def transition(self) -> None:
|
7045
|
+
raise NotImplementedError
|
7046
|
+
|
7047
|
+
@property
|
7048
|
+
@abc.abstractmethod
|
7049
|
+
def state(self) -> ProcessState:
|
7050
|
+
raise NotImplementedError
|
7051
|
+
|
7052
|
+
@abc.abstractmethod
|
7053
|
+
def after_setuid(self) -> None:
|
7054
|
+
raise NotImplementedError
|
6513
7055
|
|
6514
7056
|
|
6515
7057
|
##
|
6516
7058
|
|
6517
7059
|
|
6518
|
-
|
6519
|
-
|
6520
|
-
|
6521
|
-
|
7060
|
+
class ProcessGroup(
|
7061
|
+
ConfigPriorityOrdered,
|
7062
|
+
KeyedCollectionAccessors[str, Process],
|
7063
|
+
abc.ABC,
|
7064
|
+
):
|
7065
|
+
@property
|
7066
|
+
@abc.abstractmethod
|
7067
|
+
def name(self) -> str:
|
7068
|
+
raise NotImplementedError
|
6522
7069
|
|
7070
|
+
@property
|
7071
|
+
@abc.abstractmethod
|
7072
|
+
def config(self) -> ProcessGroupConfig:
|
7073
|
+
raise NotImplementedError
|
6523
7074
|
|
6524
|
-
|
6525
|
-
|
6526
|
-
|
6527
|
-
|
6528
|
-
return out
|
7075
|
+
@property
|
7076
|
+
@abc.abstractmethod
|
7077
|
+
def by_name(self) -> ta.Mapping[str, Process]:
|
7078
|
+
raise NotImplementedError
|
6529
7079
|
|
7080
|
+
#
|
6530
7081
|
|
6531
|
-
|
7082
|
+
@abc.abstractmethod
|
7083
|
+
def stop_all(self) -> None:
|
7084
|
+
raise NotImplementedError
|
6532
7085
|
|
7086
|
+
@abc.abstractmethod
|
7087
|
+
def get_unstopped_processes(self) -> ta.List[Process]:
|
7088
|
+
raise NotImplementedError
|
6533
7089
|
|
6534
|
-
|
6535
|
-
|
6536
|
-
|
6537
|
-
s = str(value).lower()
|
6538
|
-
level = logging.getLevelNamesMapping().get(s.upper())
|
6539
|
-
if level is None:
|
6540
|
-
raise ValueError(f'bad logging level name {value!r}')
|
6541
|
-
return level
|
7090
|
+
@abc.abstractmethod
|
7091
|
+
def before_remove(self) -> None:
|
7092
|
+
raise NotImplementedError
|
6542
7093
|
|
6543
7094
|
|
6544
7095
|
########################################
|
@@ -7047,368 +7598,59 @@ class CoroHttpServer:
|
|
7047
7598
|
##
|
7048
7599
|
|
7049
7600
|
|
7050
|
-
class CoroHttpServerSocketHandler(SocketHandler):
|
7051
|
-
def __init__(
|
7052
|
-
self,
|
7053
|
-
client_address: SocketAddress,
|
7054
|
-
rfile: ta.BinaryIO,
|
7055
|
-
wfile: ta.BinaryIO,
|
7056
|
-
*,
|
7057
|
-
server_factory: CoroHttpServerFactory,
|
7058
|
-
log_handler: ta.Optional[ta.Callable[[CoroHttpServer, CoroHttpServer.AnyLogIo], None]] = None,
|
7059
|
-
) -> None:
|
7060
|
-
super().__init__(
|
7061
|
-
client_address,
|
7062
|
-
rfile,
|
7063
|
-
wfile,
|
7064
|
-
)
|
7065
|
-
|
7066
|
-
self._server_factory = server_factory
|
7067
|
-
self._log_handler = log_handler
|
7068
|
-
|
7069
|
-
def handle(self) -> None:
|
7070
|
-
server = self._server_factory(self._client_address)
|
7071
|
-
|
7072
|
-
gen = server.coro_handle()
|
7073
|
-
|
7074
|
-
o = next(gen)
|
7075
|
-
while True:
|
7076
|
-
if isinstance(o, CoroHttpServer.AnyLogIo):
|
7077
|
-
i = None
|
7078
|
-
if self._log_handler is not None:
|
7079
|
-
self._log_handler(server, o)
|
7080
|
-
|
7081
|
-
elif isinstance(o, CoroHttpServer.ReadIo):
|
7082
|
-
i = self._rfile.read(o.sz)
|
7083
|
-
|
7084
|
-
elif isinstance(o, CoroHttpServer.ReadLineIo):
|
7085
|
-
i = self._rfile.readline(o.sz)
|
7086
|
-
|
7087
|
-
elif isinstance(o, CoroHttpServer.WriteIo):
|
7088
|
-
i = None
|
7089
|
-
self._wfile.write(o.data)
|
7090
|
-
self._wfile.flush()
|
7091
|
-
|
7092
|
-
else:
|
7093
|
-
raise TypeError(o)
|
7094
|
-
|
7095
|
-
try:
|
7096
|
-
if i is not None:
|
7097
|
-
o = gen.send(i)
|
7098
|
-
else:
|
7099
|
-
o = next(gen)
|
7100
|
-
except StopIteration:
|
7101
|
-
break
|
7102
|
-
|
7103
|
-
|
7104
|
-
########################################
|
7105
|
-
# ../types.py
|
7106
|
-
|
7107
|
-
|
7108
|
-
##
|
7109
|
-
|
7110
|
-
|
7111
|
-
class ExitNow(Exception): # noqa
|
7112
|
-
pass
|
7113
|
-
|
7114
|
-
|
7115
|
-
ServerEpoch = ta.NewType('ServerEpoch', int)
|
7116
|
-
|
7117
|
-
|
7118
|
-
##
|
7119
|
-
|
7120
|
-
|
7121
|
-
@functools.total_ordering
|
7122
|
-
class ConfigPriorityOrdered(abc.ABC):
|
7123
|
-
@property
|
7124
|
-
@abc.abstractmethod
|
7125
|
-
def config(self) -> ta.Any:
|
7126
|
-
raise NotImplementedError
|
7127
|
-
|
7128
|
-
def __lt__(self, other):
|
7129
|
-
return self.config.priority < other.config.priority
|
7130
|
-
|
7131
|
-
def __eq__(self, other):
|
7132
|
-
return self.config.priority == other.config.priority
|
7133
|
-
|
7134
|
-
|
7135
|
-
##
|
7136
|
-
|
7137
|
-
|
7138
|
-
class SupervisorStateManager(abc.ABC):
|
7139
|
-
@property
|
7140
|
-
@abc.abstractmethod
|
7141
|
-
def state(self) -> SupervisorState:
|
7142
|
-
raise NotImplementedError
|
7143
|
-
|
7144
|
-
@abc.abstractmethod
|
7145
|
-
def set_state(self, state: SupervisorState) -> None:
|
7146
|
-
raise NotImplementedError
|
7147
|
-
|
7148
|
-
|
7149
|
-
##
|
7150
|
-
|
7151
|
-
|
7152
|
-
class HasDispatchers(abc.ABC):
|
7153
|
-
@abc.abstractmethod
|
7154
|
-
def get_dispatchers(self) -> 'Dispatchers':
|
7155
|
-
raise NotImplementedError
|
7156
|
-
|
7157
|
-
|
7158
|
-
class ProcessDispatcher(FdioHandler, abc.ABC):
|
7159
|
-
@property
|
7160
|
-
@abc.abstractmethod
|
7161
|
-
def channel(self) -> ProcessOutputChannel:
|
7162
|
-
raise NotImplementedError
|
7163
|
-
|
7164
|
-
@property
|
7165
|
-
@abc.abstractmethod
|
7166
|
-
def process(self) -> 'Process':
|
7167
|
-
raise NotImplementedError
|
7168
|
-
|
7169
|
-
|
7170
|
-
class ProcessOutputDispatcher(ProcessDispatcher, abc.ABC):
|
7171
|
-
@abc.abstractmethod
|
7172
|
-
def remove_logs(self) -> None:
|
7173
|
-
raise NotImplementedError
|
7174
|
-
|
7175
|
-
@abc.abstractmethod
|
7176
|
-
def reopen_logs(self) -> None:
|
7177
|
-
raise NotImplementedError
|
7178
|
-
|
7179
|
-
|
7180
|
-
class ProcessInputDispatcher(ProcessDispatcher, abc.ABC):
|
7181
|
-
@abc.abstractmethod
|
7182
|
-
def write(self, chars: ta.Union[bytes, str]) -> None:
|
7183
|
-
raise NotImplementedError
|
7184
|
-
|
7185
|
-
@abc.abstractmethod
|
7186
|
-
def flush(self) -> None:
|
7187
|
-
raise NotImplementedError
|
7188
|
-
|
7189
|
-
|
7190
|
-
##
|
7191
|
-
|
7192
|
-
|
7193
|
-
class Process(
|
7194
|
-
ConfigPriorityOrdered,
|
7195
|
-
HasDispatchers,
|
7196
|
-
abc.ABC,
|
7197
|
-
):
|
7198
|
-
@property
|
7199
|
-
@abc.abstractmethod
|
7200
|
-
def name(self) -> str:
|
7201
|
-
raise NotImplementedError
|
7202
|
-
|
7203
|
-
@property
|
7204
|
-
@abc.abstractmethod
|
7205
|
-
def config(self) -> ProcessConfig:
|
7206
|
-
raise NotImplementedError
|
7207
|
-
|
7208
|
-
@property
|
7209
|
-
@abc.abstractmethod
|
7210
|
-
def group(self) -> 'ProcessGroup':
|
7211
|
-
raise NotImplementedError
|
7212
|
-
|
7213
|
-
@property
|
7214
|
-
@abc.abstractmethod
|
7215
|
-
def pid(self) -> Pid:
|
7216
|
-
raise NotImplementedError
|
7217
|
-
|
7218
|
-
#
|
7219
|
-
|
7220
|
-
@abc.abstractmethod
|
7221
|
-
def finish(self, sts: Rc) -> None:
|
7222
|
-
raise NotImplementedError
|
7223
|
-
|
7224
|
-
@abc.abstractmethod
|
7225
|
-
def stop(self) -> ta.Optional[str]:
|
7226
|
-
raise NotImplementedError
|
7227
|
-
|
7228
|
-
@abc.abstractmethod
|
7229
|
-
def give_up(self) -> None:
|
7230
|
-
raise NotImplementedError
|
7231
|
-
|
7232
|
-
@abc.abstractmethod
|
7233
|
-
def transition(self) -> None:
|
7234
|
-
raise NotImplementedError
|
7235
|
-
|
7236
|
-
@property
|
7237
|
-
@abc.abstractmethod
|
7238
|
-
def state(self) -> ProcessState:
|
7239
|
-
raise NotImplementedError
|
7240
|
-
|
7241
|
-
@abc.abstractmethod
|
7242
|
-
def after_setuid(self) -> None:
|
7243
|
-
raise NotImplementedError
|
7244
|
-
|
7245
|
-
|
7246
|
-
##
|
7247
|
-
|
7248
|
-
|
7249
|
-
class ProcessGroup(
|
7250
|
-
ConfigPriorityOrdered,
|
7251
|
-
KeyedCollectionAccessors[str, Process],
|
7252
|
-
abc.ABC,
|
7253
|
-
):
|
7254
|
-
@property
|
7255
|
-
@abc.abstractmethod
|
7256
|
-
def name(self) -> str:
|
7257
|
-
raise NotImplementedError
|
7258
|
-
|
7259
|
-
@property
|
7260
|
-
@abc.abstractmethod
|
7261
|
-
def config(self) -> ProcessGroupConfig:
|
7262
|
-
raise NotImplementedError
|
7263
|
-
|
7264
|
-
@property
|
7265
|
-
@abc.abstractmethod
|
7266
|
-
def by_name(self) -> ta.Mapping[str, Process]:
|
7267
|
-
raise NotImplementedError
|
7268
|
-
|
7269
|
-
#
|
7270
|
-
|
7271
|
-
@abc.abstractmethod
|
7272
|
-
def stop_all(self) -> None:
|
7273
|
-
raise NotImplementedError
|
7274
|
-
|
7275
|
-
@abc.abstractmethod
|
7276
|
-
def get_unstopped_processes(self) -> ta.List[Process]:
|
7277
|
-
raise NotImplementedError
|
7278
|
-
|
7279
|
-
@abc.abstractmethod
|
7280
|
-
def before_remove(self) -> None:
|
7281
|
-
raise NotImplementedError
|
7282
|
-
|
7283
|
-
|
7284
|
-
########################################
|
7285
|
-
# ../../../omlish/http/coro/fdio.py
|
7286
|
-
|
7287
|
-
|
7288
|
-
class CoroHttpServerConnectionFdioHandler(SocketFdioHandler):
|
7601
|
+
class CoroHttpServerSocketHandler(SocketHandler):
|
7289
7602
|
def __init__(
|
7290
7603
|
self,
|
7291
|
-
|
7292
|
-
|
7293
|
-
|
7604
|
+
client_address: SocketAddress,
|
7605
|
+
rfile: ta.BinaryIO,
|
7606
|
+
wfile: ta.BinaryIO,
|
7294
7607
|
*,
|
7295
|
-
|
7296
|
-
write_size: int = 0x10000,
|
7608
|
+
server_factory: CoroHttpServerFactory,
|
7297
7609
|
log_handler: ta.Optional[ta.Callable[[CoroHttpServer, CoroHttpServer.AnyLogIo], None]] = None,
|
7298
7610
|
) -> None:
|
7299
|
-
|
7300
|
-
|
7301
|
-
|
7302
|
-
|
7303
|
-
self._handler = handler
|
7304
|
-
self._read_size = read_size
|
7305
|
-
self._write_size = write_size
|
7306
|
-
self._log_handler = log_handler
|
7307
|
-
|
7308
|
-
self._read_buf = ReadableListBuffer()
|
7309
|
-
self._write_buf: IncrementalWriteBuffer | None = None
|
7310
|
-
|
7311
|
-
self._coro_srv = CoroHttpServer(
|
7312
|
-
addr,
|
7313
|
-
handler=self._handler,
|
7611
|
+
super().__init__(
|
7612
|
+
client_address,
|
7613
|
+
rfile,
|
7614
|
+
wfile,
|
7314
7615
|
)
|
7315
|
-
self._srv_coro: ta.Optional[ta.Generator[CoroHttpServer.Io, ta.Optional[bytes], None]] = self._coro_srv.coro_handle() # noqa
|
7316
7616
|
|
7317
|
-
self.
|
7318
|
-
self.
|
7617
|
+
self._server_factory = server_factory
|
7618
|
+
self._log_handler = log_handler
|
7319
7619
|
|
7320
|
-
|
7620
|
+
def handle(self) -> None:
|
7621
|
+
server = self._server_factory(self._client_address)
|
7321
7622
|
|
7322
|
-
|
7323
|
-
coro = check.not_none(self._srv_coro)
|
7623
|
+
gen = server.coro_handle()
|
7324
7624
|
|
7325
|
-
|
7326
|
-
o = self._cur_io
|
7625
|
+
o = next(gen)
|
7327
7626
|
while True:
|
7328
|
-
if o is None:
|
7329
|
-
try:
|
7330
|
-
if d is not None:
|
7331
|
-
o = coro.send(d)
|
7332
|
-
d = None
|
7333
|
-
else:
|
7334
|
-
o = next(coro)
|
7335
|
-
except StopIteration:
|
7336
|
-
self.close()
|
7337
|
-
o = None
|
7338
|
-
break
|
7339
|
-
|
7340
7627
|
if isinstance(o, CoroHttpServer.AnyLogIo):
|
7628
|
+
i = None
|
7341
7629
|
if self._log_handler is not None:
|
7342
|
-
self._log_handler(
|
7343
|
-
o = None
|
7630
|
+
self._log_handler(server, o)
|
7344
7631
|
|
7345
7632
|
elif isinstance(o, CoroHttpServer.ReadIo):
|
7346
|
-
|
7347
|
-
break
|
7348
|
-
o = None
|
7633
|
+
i = self._rfile.read(o.sz)
|
7349
7634
|
|
7350
7635
|
elif isinstance(o, CoroHttpServer.ReadLineIo):
|
7351
|
-
|
7352
|
-
break
|
7353
|
-
o = None
|
7636
|
+
i = self._rfile.readline(o.sz)
|
7354
7637
|
|
7355
7638
|
elif isinstance(o, CoroHttpServer.WriteIo):
|
7356
|
-
|
7357
|
-
self.
|
7358
|
-
|
7639
|
+
i = None
|
7640
|
+
self._wfile.write(o.data)
|
7641
|
+
self._wfile.flush()
|
7359
7642
|
|
7360
7643
|
else:
|
7361
7644
|
raise TypeError(o)
|
7362
7645
|
|
7363
|
-
|
7364
|
-
|
7365
|
-
|
7366
|
-
|
7367
|
-
|
7368
|
-
|
7369
|
-
|
7370
|
-
def writable(self) -> bool:
|
7371
|
-
return self._write_buf is not None
|
7372
|
-
|
7373
|
-
#
|
7374
|
-
|
7375
|
-
def on_readable(self) -> None:
|
7376
|
-
try:
|
7377
|
-
buf = check.not_none(self._sock).recv(self._read_size)
|
7378
|
-
except BlockingIOError:
|
7379
|
-
return
|
7380
|
-
except ConnectionResetError:
|
7381
|
-
self.close()
|
7382
|
-
return
|
7383
|
-
if not buf:
|
7384
|
-
self.close()
|
7385
|
-
return
|
7386
|
-
|
7387
|
-
self._read_buf.feed(buf)
|
7388
|
-
|
7389
|
-
if isinstance(self._cur_io, CoroHttpServer.AnyReadIo):
|
7390
|
-
self._next_io()
|
7391
|
-
|
7392
|
-
def on_writable(self) -> None:
|
7393
|
-
check.isinstance(self._cur_io, CoroHttpServer.WriteIo)
|
7394
|
-
wb = check.not_none(self._write_buf)
|
7395
|
-
while wb.rem > 0:
|
7396
|
-
def send(d: bytes) -> int:
|
7397
|
-
try:
|
7398
|
-
return check.not_none(self._sock).send(d)
|
7399
|
-
except ConnectionResetError:
|
7400
|
-
self.close()
|
7401
|
-
return 0
|
7402
|
-
except BlockingIOError:
|
7403
|
-
return 0
|
7404
|
-
if not wb.write(send):
|
7646
|
+
try:
|
7647
|
+
if i is not None:
|
7648
|
+
o = gen.send(i)
|
7649
|
+
else:
|
7650
|
+
o = next(gen)
|
7651
|
+
except StopIteration:
|
7405
7652
|
break
|
7406
7653
|
|
7407
|
-
if wb.rem < 1:
|
7408
|
-
self._write_buf = None
|
7409
|
-
self._cur_io = None
|
7410
|
-
self._next_io()
|
7411
|
-
|
7412
7654
|
|
7413
7655
|
########################################
|
7414
7656
|
# ../dispatchers.py
|
@@ -8110,6 +8352,135 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
8110
8352
|
os.umask(self._config.umask)
|
8111
8353
|
|
8112
8354
|
|
8355
|
+
########################################
|
8356
|
+
# ../../../omlish/http/coro/fdio.py
|
8357
|
+
|
8358
|
+
|
8359
|
+
class CoroHttpServerConnectionFdioHandler(SocketFdioHandler):
|
8360
|
+
def __init__(
|
8361
|
+
self,
|
8362
|
+
addr: SocketAddress,
|
8363
|
+
sock: socket.socket,
|
8364
|
+
handler: HttpHandler,
|
8365
|
+
*,
|
8366
|
+
read_size: int = 0x10000,
|
8367
|
+
write_size: int = 0x10000,
|
8368
|
+
log_handler: ta.Optional[ta.Callable[[CoroHttpServer, CoroHttpServer.AnyLogIo], None]] = None,
|
8369
|
+
) -> None:
|
8370
|
+
check.state(not sock.getblocking())
|
8371
|
+
|
8372
|
+
super().__init__(addr, sock)
|
8373
|
+
|
8374
|
+
self._handler = handler
|
8375
|
+
self._read_size = read_size
|
8376
|
+
self._write_size = write_size
|
8377
|
+
self._log_handler = log_handler
|
8378
|
+
|
8379
|
+
self._read_buf = ReadableListBuffer()
|
8380
|
+
self._write_buf: IncrementalWriteBuffer | None = None
|
8381
|
+
|
8382
|
+
self._coro_srv = CoroHttpServer(
|
8383
|
+
addr,
|
8384
|
+
handler=self._handler,
|
8385
|
+
)
|
8386
|
+
self._srv_coro: ta.Optional[ta.Generator[CoroHttpServer.Io, ta.Optional[bytes], None]] = self._coro_srv.coro_handle() # noqa
|
8387
|
+
|
8388
|
+
self._cur_io: CoroHttpServer.Io | None = None
|
8389
|
+
self._next_io()
|
8390
|
+
|
8391
|
+
#
|
8392
|
+
|
8393
|
+
def _next_io(self) -> None: # noqa
|
8394
|
+
coro = check.not_none(self._srv_coro)
|
8395
|
+
|
8396
|
+
d: bytes | None = None
|
8397
|
+
o = self._cur_io
|
8398
|
+
while True:
|
8399
|
+
if o is None:
|
8400
|
+
try:
|
8401
|
+
if d is not None:
|
8402
|
+
o = coro.send(d)
|
8403
|
+
d = None
|
8404
|
+
else:
|
8405
|
+
o = next(coro)
|
8406
|
+
except StopIteration:
|
8407
|
+
self.close()
|
8408
|
+
o = None
|
8409
|
+
break
|
8410
|
+
|
8411
|
+
if isinstance(o, CoroHttpServer.AnyLogIo):
|
8412
|
+
if self._log_handler is not None:
|
8413
|
+
self._log_handler(self._coro_srv, o)
|
8414
|
+
o = None
|
8415
|
+
|
8416
|
+
elif isinstance(o, CoroHttpServer.ReadIo):
|
8417
|
+
if (d := self._read_buf.read(o.sz)) is None:
|
8418
|
+
break
|
8419
|
+
o = None
|
8420
|
+
|
8421
|
+
elif isinstance(o, CoroHttpServer.ReadLineIo):
|
8422
|
+
if (d := self._read_buf.read_until(b'\n')) is None:
|
8423
|
+
break
|
8424
|
+
o = None
|
8425
|
+
|
8426
|
+
elif isinstance(o, CoroHttpServer.WriteIo):
|
8427
|
+
check.none(self._write_buf)
|
8428
|
+
self._write_buf = IncrementalWriteBuffer(o.data, write_size=self._write_size)
|
8429
|
+
break
|
8430
|
+
|
8431
|
+
else:
|
8432
|
+
raise TypeError(o)
|
8433
|
+
|
8434
|
+
self._cur_io = o
|
8435
|
+
|
8436
|
+
#
|
8437
|
+
|
8438
|
+
def readable(self) -> bool:
|
8439
|
+
return True
|
8440
|
+
|
8441
|
+
def writable(self) -> bool:
|
8442
|
+
return self._write_buf is not None
|
8443
|
+
|
8444
|
+
#
|
8445
|
+
|
8446
|
+
def on_readable(self) -> None:
|
8447
|
+
try:
|
8448
|
+
buf = check.not_none(self._sock).recv(self._read_size)
|
8449
|
+
except BlockingIOError:
|
8450
|
+
return
|
8451
|
+
except ConnectionResetError:
|
8452
|
+
self.close()
|
8453
|
+
return
|
8454
|
+
if not buf:
|
8455
|
+
self.close()
|
8456
|
+
return
|
8457
|
+
|
8458
|
+
self._read_buf.feed(buf)
|
8459
|
+
|
8460
|
+
if isinstance(self._cur_io, CoroHttpServer.AnyReadIo):
|
8461
|
+
self._next_io()
|
8462
|
+
|
8463
|
+
def on_writable(self) -> None:
|
8464
|
+
check.isinstance(self._cur_io, CoroHttpServer.WriteIo)
|
8465
|
+
wb = check.not_none(self._write_buf)
|
8466
|
+
while wb.rem > 0:
|
8467
|
+
def send(d: bytes) -> int:
|
8468
|
+
try:
|
8469
|
+
return check.not_none(self._sock).send(d)
|
8470
|
+
except ConnectionResetError:
|
8471
|
+
self.close()
|
8472
|
+
return 0
|
8473
|
+
except BlockingIOError:
|
8474
|
+
return 0
|
8475
|
+
if not wb.write(send):
|
8476
|
+
break
|
8477
|
+
|
8478
|
+
if wb.rem < 1:
|
8479
|
+
self._write_buf = None
|
8480
|
+
self._cur_io = None
|
8481
|
+
self._next_io()
|
8482
|
+
|
8483
|
+
|
8113
8484
|
########################################
|
8114
8485
|
# ../groups.py
|
8115
8486
|
|
@@ -9685,7 +10056,7 @@ def main(
|
|
9685
10056
|
|
9686
10057
|
# if we hup, restart by making a new Supervisor()
|
9687
10058
|
for epoch in itertools.count():
|
9688
|
-
config =
|
10059
|
+
config = load_config_file_obj(
|
9689
10060
|
os.path.expanduser(cf),
|
9690
10061
|
ServerConfig,
|
9691
10062
|
prepare=prepare_server_config,
|