ominfra 0.0.0.dev157__py3-none-any.whl → 0.0.0.dev159__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/main.py +1 -1
- ominfra/journald/tailer.py +2 -2
- ominfra/manage/bootstrap_.py +1 -1
- ominfra/manage/commands/subprocess.py +4 -4
- ominfra/manage/deploy/apps.py +23 -21
- ominfra/manage/deploy/atomics.py +207 -0
- ominfra/manage/deploy/config.py +3 -0
- ominfra/manage/deploy/git.py +27 -47
- ominfra/manage/deploy/inject.py +11 -0
- ominfra/manage/deploy/paths.py +89 -51
- ominfra/manage/deploy/specs.py +42 -0
- ominfra/manage/deploy/tmp.py +46 -0
- ominfra/manage/deploy/types.py +1 -0
- ominfra/manage/deploy/venvs.py +16 -6
- ominfra/manage/remote/spawning.py +3 -3
- ominfra/manage/system/packages.py +1 -1
- ominfra/pyremote.py +26 -26
- ominfra/scripts/journald2aws.py +467 -354
- ominfra/scripts/manage.py +1426 -1037
- ominfra/scripts/supervisor.py +359 -336
- ominfra/supervisor/http.py +1 -1
- ominfra/supervisor/main.py +2 -2
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/RECORD +28 -25
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/top_level.txt +0 -0
ominfra/scripts/journald2aws.py
CHANGED
@@ -78,7 +78,7 @@ ConfigMapping = ta.Mapping[str, ta.Any]
|
|
78
78
|
# ../../../threadworkers.py
|
79
79
|
ThreadWorkerT = ta.TypeVar('ThreadWorkerT', bound='ThreadWorker')
|
80
80
|
|
81
|
-
# ../../../../omlish/
|
81
|
+
# ../../../../omlish/subprocesses.py
|
82
82
|
SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
|
83
83
|
|
84
84
|
|
@@ -1437,6 +1437,13 @@ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON
|
|
1437
1437
|
json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
|
1438
1438
|
|
1439
1439
|
|
1440
|
+
########################################
|
1441
|
+
# ../../../../../omlish/lite/logs.py
|
1442
|
+
|
1443
|
+
|
1444
|
+
log = logging.getLogger(__name__)
|
1445
|
+
|
1446
|
+
|
1440
1447
|
########################################
|
1441
1448
|
# ../../../../../omlish/lite/reflect.py
|
1442
1449
|
|
@@ -1559,6 +1566,116 @@ def format_num_bytes(num_bytes: int) -> str:
|
|
1559
1566
|
return f'{num_bytes / 1024 ** (len(FORMAT_NUM_BYTES_SUFFIXES) - 1):.2f}{FORMAT_NUM_BYTES_SUFFIXES[-1]}'
|
1560
1567
|
|
1561
1568
|
|
1569
|
+
########################################
|
1570
|
+
# ../../../../../omlish/logs/filters.py
|
1571
|
+
|
1572
|
+
|
1573
|
+
class TidLogFilter(logging.Filter):
|
1574
|
+
def filter(self, record):
|
1575
|
+
record.tid = threading.get_native_id()
|
1576
|
+
return True
|
1577
|
+
|
1578
|
+
|
1579
|
+
########################################
|
1580
|
+
# ../../../../../omlish/logs/proxy.py
|
1581
|
+
|
1582
|
+
|
1583
|
+
class ProxyLogFilterer(logging.Filterer):
|
1584
|
+
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
1585
|
+
self._underlying = underlying
|
1586
|
+
|
1587
|
+
@property
|
1588
|
+
def underlying(self) -> logging.Filterer:
|
1589
|
+
return self._underlying
|
1590
|
+
|
1591
|
+
@property
|
1592
|
+
def filters(self):
|
1593
|
+
return self._underlying.filters
|
1594
|
+
|
1595
|
+
@filters.setter
|
1596
|
+
def filters(self, filters):
|
1597
|
+
self._underlying.filters = filters
|
1598
|
+
|
1599
|
+
def addFilter(self, filter): # noqa
|
1600
|
+
self._underlying.addFilter(filter)
|
1601
|
+
|
1602
|
+
def removeFilter(self, filter): # noqa
|
1603
|
+
self._underlying.removeFilter(filter)
|
1604
|
+
|
1605
|
+
def filter(self, record):
|
1606
|
+
return self._underlying.filter(record)
|
1607
|
+
|
1608
|
+
|
1609
|
+
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
1610
|
+
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
1611
|
+
ProxyLogFilterer.__init__(self, underlying)
|
1612
|
+
|
1613
|
+
_underlying: logging.Handler
|
1614
|
+
|
1615
|
+
@property
|
1616
|
+
def underlying(self) -> logging.Handler:
|
1617
|
+
return self._underlying
|
1618
|
+
|
1619
|
+
def get_name(self):
|
1620
|
+
return self._underlying.get_name()
|
1621
|
+
|
1622
|
+
def set_name(self, name):
|
1623
|
+
self._underlying.set_name(name)
|
1624
|
+
|
1625
|
+
@property
|
1626
|
+
def name(self):
|
1627
|
+
return self._underlying.name
|
1628
|
+
|
1629
|
+
@property
|
1630
|
+
def level(self):
|
1631
|
+
return self._underlying.level
|
1632
|
+
|
1633
|
+
@level.setter
|
1634
|
+
def level(self, level):
|
1635
|
+
self._underlying.level = level
|
1636
|
+
|
1637
|
+
@property
|
1638
|
+
def formatter(self):
|
1639
|
+
return self._underlying.formatter
|
1640
|
+
|
1641
|
+
@formatter.setter
|
1642
|
+
def formatter(self, formatter):
|
1643
|
+
self._underlying.formatter = formatter
|
1644
|
+
|
1645
|
+
def createLock(self):
|
1646
|
+
self._underlying.createLock()
|
1647
|
+
|
1648
|
+
def acquire(self):
|
1649
|
+
self._underlying.acquire()
|
1650
|
+
|
1651
|
+
def release(self):
|
1652
|
+
self._underlying.release()
|
1653
|
+
|
1654
|
+
def setLevel(self, level):
|
1655
|
+
self._underlying.setLevel(level)
|
1656
|
+
|
1657
|
+
def format(self, record):
|
1658
|
+
return self._underlying.format(record)
|
1659
|
+
|
1660
|
+
def emit(self, record):
|
1661
|
+
self._underlying.emit(record)
|
1662
|
+
|
1663
|
+
def handle(self, record):
|
1664
|
+
return self._underlying.handle(record)
|
1665
|
+
|
1666
|
+
def setFormatter(self, fmt):
|
1667
|
+
self._underlying.setFormatter(fmt)
|
1668
|
+
|
1669
|
+
def flush(self):
|
1670
|
+
self._underlying.flush()
|
1671
|
+
|
1672
|
+
def close(self):
|
1673
|
+
self._underlying.close()
|
1674
|
+
|
1675
|
+
def handleError(self, record):
|
1676
|
+
self._underlying.handleError(record)
|
1677
|
+
|
1678
|
+
|
1562
1679
|
########################################
|
1563
1680
|
# ../../../../../omlish/os/pidfile.py
|
1564
1681
|
|
@@ -1988,6 +2105,52 @@ class AwsDataclassMeta:
|
|
1988
2105
|
return AwsDataclassMeta.Converters(d2a, a2d)
|
1989
2106
|
|
1990
2107
|
|
2108
|
+
########################################
|
2109
|
+
# ../cursor.py
|
2110
|
+
|
2111
|
+
|
2112
|
+
class JournalctlToAwsCursor:
|
2113
|
+
def __init__(
|
2114
|
+
self,
|
2115
|
+
cursor_file: ta.Optional[str] = None,
|
2116
|
+
*,
|
2117
|
+
ensure_locked: ta.Optional[ta.Callable[[], None]] = None,
|
2118
|
+
) -> None:
|
2119
|
+
super().__init__()
|
2120
|
+
self._cursor_file = cursor_file
|
2121
|
+
self._ensure_locked = ensure_locked
|
2122
|
+
|
2123
|
+
#
|
2124
|
+
|
2125
|
+
def get(self) -> ta.Optional[str]:
|
2126
|
+
if self._ensure_locked is not None:
|
2127
|
+
self._ensure_locked()
|
2128
|
+
|
2129
|
+
if not (cf := self._cursor_file):
|
2130
|
+
return None
|
2131
|
+
cf = os.path.expanduser(cf)
|
2132
|
+
|
2133
|
+
try:
|
2134
|
+
with open(cf) as f:
|
2135
|
+
return f.read().strip()
|
2136
|
+
except FileNotFoundError:
|
2137
|
+
return None
|
2138
|
+
|
2139
|
+
def set(self, cursor: str) -> None:
|
2140
|
+
if self._ensure_locked is not None:
|
2141
|
+
self._ensure_locked()
|
2142
|
+
|
2143
|
+
if not (cf := self._cursor_file):
|
2144
|
+
return
|
2145
|
+
cf = os.path.expanduser(cf)
|
2146
|
+
|
2147
|
+
log.info('Writing cursor file %s : %s', cf, cursor)
|
2148
|
+
with open(ncf := cf + '.next', 'w') as f:
|
2149
|
+
f.write(cursor)
|
2150
|
+
|
2151
|
+
os.rename(ncf, cf)
|
2152
|
+
|
2153
|
+
|
1991
2154
|
########################################
|
1992
2155
|
# ../../../../../omlish/io/buffers.py
|
1993
2156
|
|
@@ -2283,333 +2446,63 @@ class aclosing(contextlib.AbstractAsyncContextManager): # noqa
|
|
2283
2446
|
|
2284
2447
|
|
2285
2448
|
########################################
|
2286
|
-
# ../../../../../omlish/lite/
|
2449
|
+
# ../../../../../omlish/lite/marshal.py
|
2287
2450
|
"""
|
2288
2451
|
TODO:
|
2289
|
-
-
|
2290
|
-
-
|
2452
|
+
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
2453
|
+
- namedtuple
|
2454
|
+
- literals
|
2455
|
+
- newtypes?
|
2291
2456
|
"""
|
2292
2457
|
|
2293
2458
|
|
2294
|
-
log = logging.getLogger(__name__)
|
2295
|
-
|
2296
|
-
|
2297
2459
|
##
|
2298
2460
|
|
2299
2461
|
|
2300
|
-
|
2462
|
+
@dc.dataclass(frozen=True)
|
2463
|
+
class ObjMarshalOptions:
|
2464
|
+
raw_bytes: bool = False
|
2465
|
+
nonstrict_dataclasses: bool = False
|
2301
2466
|
|
2302
|
-
def filter(self, record):
|
2303
|
-
record.tid = threading.get_native_id()
|
2304
|
-
return True
|
2305
2467
|
|
2468
|
+
class ObjMarshaler(abc.ABC):
|
2469
|
+
@abc.abstractmethod
|
2470
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2471
|
+
raise NotImplementedError
|
2306
2472
|
|
2307
|
-
|
2473
|
+
@abc.abstractmethod
|
2474
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2475
|
+
raise NotImplementedError
|
2308
2476
|
|
2309
2477
|
|
2310
|
-
class
|
2478
|
+
class NopObjMarshaler(ObjMarshaler):
|
2479
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2480
|
+
return o
|
2311
2481
|
|
2312
|
-
|
2313
|
-
|
2314
|
-
'msg': False,
|
2315
|
-
'args': False,
|
2316
|
-
'levelname': False,
|
2317
|
-
'levelno': False,
|
2318
|
-
'pathname': False,
|
2319
|
-
'filename': False,
|
2320
|
-
'module': False,
|
2321
|
-
'exc_info': True,
|
2322
|
-
'exc_text': True,
|
2323
|
-
'stack_info': True,
|
2324
|
-
'lineno': False,
|
2325
|
-
'funcName': False,
|
2326
|
-
'created': False,
|
2327
|
-
'msecs': False,
|
2328
|
-
'relativeCreated': False,
|
2329
|
-
'thread': False,
|
2330
|
-
'threadName': False,
|
2331
|
-
'processName': False,
|
2332
|
-
'process': False,
|
2333
|
-
}
|
2482
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2483
|
+
return o
|
2334
2484
|
|
2335
|
-
def format(self, record: logging.LogRecord) -> str:
|
2336
|
-
dct = {
|
2337
|
-
k: v
|
2338
|
-
for k, o in self.KEYS.items()
|
2339
|
-
for v in [getattr(record, k)]
|
2340
|
-
if not (o and v is None)
|
2341
|
-
}
|
2342
|
-
return json_dumps_compact(dct)
|
2343
2485
|
|
2486
|
+
@dc.dataclass()
|
2487
|
+
class ProxyObjMarshaler(ObjMarshaler):
|
2488
|
+
m: ta.Optional[ObjMarshaler] = None
|
2344
2489
|
|
2345
|
-
|
2490
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2491
|
+
return check.not_none(self.m).marshal(o, ctx)
|
2346
2492
|
|
2493
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2494
|
+
return check.not_none(self.m).unmarshal(o, ctx)
|
2347
2495
|
|
2348
|
-
STANDARD_LOG_FORMAT_PARTS = [
|
2349
|
-
('asctime', '%(asctime)-15s'),
|
2350
|
-
('process', 'pid=%(process)-6s'),
|
2351
|
-
('thread', 'tid=%(thread)x'),
|
2352
|
-
('levelname', '%(levelname)s'),
|
2353
|
-
('name', '%(name)s'),
|
2354
|
-
('separator', '::'),
|
2355
|
-
('message', '%(message)s'),
|
2356
|
-
]
|
2357
2496
|
|
2497
|
+
@dc.dataclass(frozen=True)
|
2498
|
+
class CastObjMarshaler(ObjMarshaler):
|
2499
|
+
ty: type
|
2358
2500
|
|
2359
|
-
|
2501
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2502
|
+
return o
|
2360
2503
|
|
2361
|
-
|
2362
|
-
|
2363
|
-
return ' '.join(v for k, v in parts)
|
2364
|
-
|
2365
|
-
converter = datetime.datetime.fromtimestamp # type: ignore
|
2366
|
-
|
2367
|
-
def formatTime(self, record, datefmt=None):
|
2368
|
-
ct = self.converter(record.created) # type: ignore
|
2369
|
-
if datefmt:
|
2370
|
-
return ct.strftime(datefmt) # noqa
|
2371
|
-
else:
|
2372
|
-
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
2373
|
-
return '%s.%03d' % (t, record.msecs) # noqa
|
2374
|
-
|
2375
|
-
|
2376
|
-
##
|
2377
|
-
|
2378
|
-
|
2379
|
-
class ProxyLogFilterer(logging.Filterer):
|
2380
|
-
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
2381
|
-
self._underlying = underlying
|
2382
|
-
|
2383
|
-
@property
|
2384
|
-
def underlying(self) -> logging.Filterer:
|
2385
|
-
return self._underlying
|
2386
|
-
|
2387
|
-
@property
|
2388
|
-
def filters(self):
|
2389
|
-
return self._underlying.filters
|
2390
|
-
|
2391
|
-
@filters.setter
|
2392
|
-
def filters(self, filters):
|
2393
|
-
self._underlying.filters = filters
|
2394
|
-
|
2395
|
-
def addFilter(self, filter): # noqa
|
2396
|
-
self._underlying.addFilter(filter)
|
2397
|
-
|
2398
|
-
def removeFilter(self, filter): # noqa
|
2399
|
-
self._underlying.removeFilter(filter)
|
2400
|
-
|
2401
|
-
def filter(self, record):
|
2402
|
-
return self._underlying.filter(record)
|
2403
|
-
|
2404
|
-
|
2405
|
-
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
2406
|
-
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
2407
|
-
ProxyLogFilterer.__init__(self, underlying)
|
2408
|
-
|
2409
|
-
_underlying: logging.Handler
|
2410
|
-
|
2411
|
-
@property
|
2412
|
-
def underlying(self) -> logging.Handler:
|
2413
|
-
return self._underlying
|
2414
|
-
|
2415
|
-
def get_name(self):
|
2416
|
-
return self._underlying.get_name()
|
2417
|
-
|
2418
|
-
def set_name(self, name):
|
2419
|
-
self._underlying.set_name(name)
|
2420
|
-
|
2421
|
-
@property
|
2422
|
-
def name(self):
|
2423
|
-
return self._underlying.name
|
2424
|
-
|
2425
|
-
@property
|
2426
|
-
def level(self):
|
2427
|
-
return self._underlying.level
|
2428
|
-
|
2429
|
-
@level.setter
|
2430
|
-
def level(self, level):
|
2431
|
-
self._underlying.level = level
|
2432
|
-
|
2433
|
-
@property
|
2434
|
-
def formatter(self):
|
2435
|
-
return self._underlying.formatter
|
2436
|
-
|
2437
|
-
@formatter.setter
|
2438
|
-
def formatter(self, formatter):
|
2439
|
-
self._underlying.formatter = formatter
|
2440
|
-
|
2441
|
-
def createLock(self):
|
2442
|
-
self._underlying.createLock()
|
2443
|
-
|
2444
|
-
def acquire(self):
|
2445
|
-
self._underlying.acquire()
|
2446
|
-
|
2447
|
-
def release(self):
|
2448
|
-
self._underlying.release()
|
2449
|
-
|
2450
|
-
def setLevel(self, level):
|
2451
|
-
self._underlying.setLevel(level)
|
2452
|
-
|
2453
|
-
def format(self, record):
|
2454
|
-
return self._underlying.format(record)
|
2455
|
-
|
2456
|
-
def emit(self, record):
|
2457
|
-
self._underlying.emit(record)
|
2458
|
-
|
2459
|
-
def handle(self, record):
|
2460
|
-
return self._underlying.handle(record)
|
2461
|
-
|
2462
|
-
def setFormatter(self, fmt):
|
2463
|
-
self._underlying.setFormatter(fmt)
|
2464
|
-
|
2465
|
-
def flush(self):
|
2466
|
-
self._underlying.flush()
|
2467
|
-
|
2468
|
-
def close(self):
|
2469
|
-
self._underlying.close()
|
2470
|
-
|
2471
|
-
def handleError(self, record):
|
2472
|
-
self._underlying.handleError(record)
|
2473
|
-
|
2474
|
-
|
2475
|
-
##
|
2476
|
-
|
2477
|
-
|
2478
|
-
class StandardLogHandler(ProxyLogHandler):
|
2479
|
-
pass
|
2480
|
-
|
2481
|
-
|
2482
|
-
##
|
2483
|
-
|
2484
|
-
|
2485
|
-
@contextlib.contextmanager
|
2486
|
-
def _locking_logging_module_lock() -> ta.Iterator[None]:
|
2487
|
-
if hasattr(logging, '_acquireLock'):
|
2488
|
-
logging._acquireLock() # noqa
|
2489
|
-
try:
|
2490
|
-
yield
|
2491
|
-
finally:
|
2492
|
-
logging._releaseLock() # type: ignore # noqa
|
2493
|
-
|
2494
|
-
elif hasattr(logging, '_lock'):
|
2495
|
-
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
2496
|
-
with logging._lock: # noqa
|
2497
|
-
yield
|
2498
|
-
|
2499
|
-
else:
|
2500
|
-
raise Exception("Can't find lock in logging module")
|
2501
|
-
|
2502
|
-
|
2503
|
-
def configure_standard_logging(
|
2504
|
-
level: ta.Union[int, str] = logging.INFO,
|
2505
|
-
*,
|
2506
|
-
json: bool = False,
|
2507
|
-
target: ta.Optional[logging.Logger] = None,
|
2508
|
-
force: bool = False,
|
2509
|
-
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
2510
|
-
) -> ta.Optional[StandardLogHandler]:
|
2511
|
-
with _locking_logging_module_lock():
|
2512
|
-
if target is None:
|
2513
|
-
target = logging.root
|
2514
|
-
|
2515
|
-
#
|
2516
|
-
|
2517
|
-
if not force:
|
2518
|
-
if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
|
2519
|
-
return None
|
2520
|
-
|
2521
|
-
#
|
2522
|
-
|
2523
|
-
if handler_factory is not None:
|
2524
|
-
handler = handler_factory()
|
2525
|
-
else:
|
2526
|
-
handler = logging.StreamHandler()
|
2527
|
-
|
2528
|
-
#
|
2529
|
-
|
2530
|
-
formatter: logging.Formatter
|
2531
|
-
if json:
|
2532
|
-
formatter = JsonLogFormatter()
|
2533
|
-
else:
|
2534
|
-
formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
|
2535
|
-
handler.setFormatter(formatter)
|
2536
|
-
|
2537
|
-
#
|
2538
|
-
|
2539
|
-
handler.addFilter(TidLogFilter())
|
2540
|
-
|
2541
|
-
#
|
2542
|
-
|
2543
|
-
target.addHandler(handler)
|
2544
|
-
|
2545
|
-
#
|
2546
|
-
|
2547
|
-
if level is not None:
|
2548
|
-
target.setLevel(level)
|
2549
|
-
|
2550
|
-
#
|
2551
|
-
|
2552
|
-
return StandardLogHandler(handler)
|
2553
|
-
|
2554
|
-
|
2555
|
-
########################################
|
2556
|
-
# ../../../../../omlish/lite/marshal.py
|
2557
|
-
"""
|
2558
|
-
TODO:
|
2559
|
-
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
2560
|
-
- namedtuple
|
2561
|
-
- literals
|
2562
|
-
- newtypes?
|
2563
|
-
"""
|
2564
|
-
|
2565
|
-
|
2566
|
-
##
|
2567
|
-
|
2568
|
-
|
2569
|
-
@dc.dataclass(frozen=True)
|
2570
|
-
class ObjMarshalOptions:
|
2571
|
-
raw_bytes: bool = False
|
2572
|
-
nonstrict_dataclasses: bool = False
|
2573
|
-
|
2574
|
-
|
2575
|
-
class ObjMarshaler(abc.ABC):
|
2576
|
-
@abc.abstractmethod
|
2577
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2578
|
-
raise NotImplementedError
|
2579
|
-
|
2580
|
-
@abc.abstractmethod
|
2581
|
-
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2582
|
-
raise NotImplementedError
|
2583
|
-
|
2584
|
-
|
2585
|
-
class NopObjMarshaler(ObjMarshaler):
|
2586
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2587
|
-
return o
|
2588
|
-
|
2589
|
-
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2590
|
-
return o
|
2591
|
-
|
2592
|
-
|
2593
|
-
@dc.dataclass()
|
2594
|
-
class ProxyObjMarshaler(ObjMarshaler):
|
2595
|
-
m: ta.Optional[ObjMarshaler] = None
|
2596
|
-
|
2597
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2598
|
-
return check.not_none(self.m).marshal(o, ctx)
|
2599
|
-
|
2600
|
-
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2601
|
-
return check.not_none(self.m).unmarshal(o, ctx)
|
2602
|
-
|
2603
|
-
|
2604
|
-
@dc.dataclass(frozen=True)
|
2605
|
-
class CastObjMarshaler(ObjMarshaler):
|
2606
|
-
ty: type
|
2607
|
-
|
2608
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2609
|
-
return o
|
2610
|
-
|
2611
|
-
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2612
|
-
return self.ty(o)
|
2504
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2505
|
+
return self.ty(o)
|
2613
2506
|
|
2614
2507
|
|
2615
2508
|
class DynamicObjMarshaler(ObjMarshaler):
|
@@ -3006,58 +2899,66 @@ def is_debugger_attached() -> bool:
|
|
3006
2899
|
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
3007
2900
|
|
3008
2901
|
|
3009
|
-
|
2902
|
+
LITE_REQUIRED_PYTHON_VERSION = (3, 8)
|
3010
2903
|
|
3011
2904
|
|
3012
|
-
def
|
3013
|
-
if sys.version_info <
|
3014
|
-
raise OSError(f'Requires python {
|
2905
|
+
def check_lite_runtime_version() -> None:
|
2906
|
+
if sys.version_info < LITE_REQUIRED_PYTHON_VERSION:
|
2907
|
+
raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
|
3015
2908
|
|
3016
2909
|
|
3017
2910
|
########################################
|
3018
|
-
#
|
2911
|
+
# ../../../../../omlish/logs/json.py
|
2912
|
+
"""
|
2913
|
+
TODO:
|
2914
|
+
- translate json keys
|
2915
|
+
"""
|
3019
2916
|
|
3020
2917
|
|
3021
|
-
class
|
2918
|
+
class JsonLogFormatter(logging.Formatter):
|
2919
|
+
KEYS: ta.Mapping[str, bool] = {
|
2920
|
+
'name': False,
|
2921
|
+
'msg': False,
|
2922
|
+
'args': False,
|
2923
|
+
'levelname': False,
|
2924
|
+
'levelno': False,
|
2925
|
+
'pathname': False,
|
2926
|
+
'filename': False,
|
2927
|
+
'module': False,
|
2928
|
+
'exc_info': True,
|
2929
|
+
'exc_text': True,
|
2930
|
+
'stack_info': True,
|
2931
|
+
'lineno': False,
|
2932
|
+
'funcName': False,
|
2933
|
+
'created': False,
|
2934
|
+
'msecs': False,
|
2935
|
+
'relativeCreated': False,
|
2936
|
+
'thread': False,
|
2937
|
+
'threadName': False,
|
2938
|
+
'processName': False,
|
2939
|
+
'process': False,
|
2940
|
+
}
|
2941
|
+
|
3022
2942
|
def __init__(
|
3023
2943
|
self,
|
3024
|
-
|
3025
|
-
|
3026
|
-
|
2944
|
+
*args: ta.Any,
|
2945
|
+
json_dumps: ta.Optional[ta.Callable[[ta.Any], str]] = None,
|
2946
|
+
**kwargs: ta.Any,
|
3027
2947
|
) -> None:
|
3028
|
-
super().__init__()
|
3029
|
-
self._cursor_file = cursor_file
|
3030
|
-
self._ensure_locked = ensure_locked
|
3031
|
-
|
3032
|
-
#
|
3033
|
-
|
3034
|
-
def get(self) -> ta.Optional[str]:
|
3035
|
-
if self._ensure_locked is not None:
|
3036
|
-
self._ensure_locked()
|
3037
|
-
|
3038
|
-
if not (cf := self._cursor_file):
|
3039
|
-
return None
|
3040
|
-
cf = os.path.expanduser(cf)
|
3041
|
-
|
3042
|
-
try:
|
3043
|
-
with open(cf) as f:
|
3044
|
-
return f.read().strip()
|
3045
|
-
except FileNotFoundError:
|
3046
|
-
return None
|
3047
|
-
|
3048
|
-
def set(self, cursor: str) -> None:
|
3049
|
-
if self._ensure_locked is not None:
|
3050
|
-
self._ensure_locked()
|
3051
|
-
|
3052
|
-
if not (cf := self._cursor_file):
|
3053
|
-
return
|
3054
|
-
cf = os.path.expanduser(cf)
|
2948
|
+
super().__init__(*args, **kwargs)
|
3055
2949
|
|
3056
|
-
|
3057
|
-
|
3058
|
-
|
2950
|
+
if json_dumps is None:
|
2951
|
+
json_dumps = json_dumps_compact
|
2952
|
+
self._json_dumps = json_dumps
|
3059
2953
|
|
3060
|
-
|
2954
|
+
def format(self, record: logging.LogRecord) -> str:
|
2955
|
+
dct = {
|
2956
|
+
k: v
|
2957
|
+
for k, o in self.KEYS.items()
|
2958
|
+
for v in [getattr(record, k)]
|
2959
|
+
if not (o and v is None)
|
2960
|
+
}
|
2961
|
+
return self._json_dumps(dct)
|
3061
2962
|
|
3062
2963
|
|
3063
2964
|
########################################
|
@@ -3575,7 +3476,129 @@ class ThreadWorkerGroup:
|
|
3575
3476
|
|
3576
3477
|
|
3577
3478
|
########################################
|
3578
|
-
# ../../../../../omlish/
|
3479
|
+
# ../../../../../omlish/logs/standard.py
|
3480
|
+
"""
|
3481
|
+
TODO:
|
3482
|
+
- structured
|
3483
|
+
- prefixed
|
3484
|
+
- debug
|
3485
|
+
- optional noisy? noisy will never be lite - some kinda configure_standard callback mechanism?
|
3486
|
+
"""
|
3487
|
+
|
3488
|
+
|
3489
|
+
##
|
3490
|
+
|
3491
|
+
|
3492
|
+
STANDARD_LOG_FORMAT_PARTS = [
|
3493
|
+
('asctime', '%(asctime)-15s'),
|
3494
|
+
('process', 'pid=%(process)-6s'),
|
3495
|
+
('thread', 'tid=%(thread)x'),
|
3496
|
+
('levelname', '%(levelname)s'),
|
3497
|
+
('name', '%(name)s'),
|
3498
|
+
('separator', '::'),
|
3499
|
+
('message', '%(message)s'),
|
3500
|
+
]
|
3501
|
+
|
3502
|
+
|
3503
|
+
class StandardLogFormatter(logging.Formatter):
|
3504
|
+
@staticmethod
|
3505
|
+
def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
|
3506
|
+
return ' '.join(v for k, v in parts)
|
3507
|
+
|
3508
|
+
converter = datetime.datetime.fromtimestamp # type: ignore
|
3509
|
+
|
3510
|
+
def formatTime(self, record, datefmt=None):
|
3511
|
+
ct = self.converter(record.created) # type: ignore
|
3512
|
+
if datefmt:
|
3513
|
+
return ct.strftime(datefmt) # noqa
|
3514
|
+
else:
|
3515
|
+
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
3516
|
+
return '%s.%03d' % (t, record.msecs) # noqa
|
3517
|
+
|
3518
|
+
|
3519
|
+
##
|
3520
|
+
|
3521
|
+
|
3522
|
+
class StandardConfiguredLogHandler(ProxyLogHandler):
|
3523
|
+
def __init_subclass__(cls, **kwargs):
|
3524
|
+
raise TypeError('This class serves only as a marker and should not be subclassed.')
|
3525
|
+
|
3526
|
+
|
3527
|
+
##
|
3528
|
+
|
3529
|
+
|
3530
|
+
@contextlib.contextmanager
|
3531
|
+
def _locking_logging_module_lock() -> ta.Iterator[None]:
|
3532
|
+
if hasattr(logging, '_acquireLock'):
|
3533
|
+
logging._acquireLock() # noqa
|
3534
|
+
try:
|
3535
|
+
yield
|
3536
|
+
finally:
|
3537
|
+
logging._releaseLock() # type: ignore # noqa
|
3538
|
+
|
3539
|
+
elif hasattr(logging, '_lock'):
|
3540
|
+
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
3541
|
+
with logging._lock: # noqa
|
3542
|
+
yield
|
3543
|
+
|
3544
|
+
else:
|
3545
|
+
raise Exception("Can't find lock in logging module")
|
3546
|
+
|
3547
|
+
|
3548
|
+
def configure_standard_logging(
|
3549
|
+
level: ta.Union[int, str] = logging.INFO,
|
3550
|
+
*,
|
3551
|
+
json: bool = False,
|
3552
|
+
target: ta.Optional[logging.Logger] = None,
|
3553
|
+
force: bool = False,
|
3554
|
+
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
3555
|
+
) -> ta.Optional[StandardConfiguredLogHandler]:
|
3556
|
+
with _locking_logging_module_lock():
|
3557
|
+
if target is None:
|
3558
|
+
target = logging.root
|
3559
|
+
|
3560
|
+
#
|
3561
|
+
|
3562
|
+
if not force:
|
3563
|
+
if any(isinstance(h, StandardConfiguredLogHandler) for h in list(target.handlers)):
|
3564
|
+
return None
|
3565
|
+
|
3566
|
+
#
|
3567
|
+
|
3568
|
+
if handler_factory is not None:
|
3569
|
+
handler = handler_factory()
|
3570
|
+
else:
|
3571
|
+
handler = logging.StreamHandler()
|
3572
|
+
|
3573
|
+
#
|
3574
|
+
|
3575
|
+
formatter: logging.Formatter
|
3576
|
+
if json:
|
3577
|
+
formatter = JsonLogFormatter()
|
3578
|
+
else:
|
3579
|
+
formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
|
3580
|
+
handler.setFormatter(formatter)
|
3581
|
+
|
3582
|
+
#
|
3583
|
+
|
3584
|
+
handler.addFilter(TidLogFilter())
|
3585
|
+
|
3586
|
+
#
|
3587
|
+
|
3588
|
+
target.addHandler(handler)
|
3589
|
+
|
3590
|
+
#
|
3591
|
+
|
3592
|
+
if level is not None:
|
3593
|
+
target.setLevel(level)
|
3594
|
+
|
3595
|
+
#
|
3596
|
+
|
3597
|
+
return StandardConfiguredLogHandler(handler)
|
3598
|
+
|
3599
|
+
|
3600
|
+
########################################
|
3601
|
+
# ../../../../../omlish/subprocesses.py
|
3579
3602
|
|
3580
3603
|
|
3581
3604
|
##
|
@@ -3626,8 +3649,8 @@ def subprocess_close(
|
|
3626
3649
|
##
|
3627
3650
|
|
3628
3651
|
|
3629
|
-
class
|
3630
|
-
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] =
|
3652
|
+
class BaseSubprocesses(abc.ABC): # noqa
|
3653
|
+
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
|
3631
3654
|
|
3632
3655
|
def __init__(
|
3633
3656
|
self,
|
@@ -3640,6 +3663,9 @@ class AbstractSubprocesses(abc.ABC): # noqa
|
|
3640
3663
|
self._log = log if log is not None else self.DEFAULT_LOGGER
|
3641
3664
|
self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
|
3642
3665
|
|
3666
|
+
def set_logger(self, log: ta.Optional[logging.Logger]) -> None:
|
3667
|
+
self._log = log
|
3668
|
+
|
3643
3669
|
#
|
3644
3670
|
|
3645
3671
|
def prepare_args(
|
@@ -3751,23 +3777,25 @@ class AbstractSubprocesses(abc.ABC): # noqa
|
|
3751
3777
|
##
|
3752
3778
|
|
3753
3779
|
|
3754
|
-
class
|
3780
|
+
class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
|
3781
|
+
@abc.abstractmethod
|
3755
3782
|
def check_call(
|
3756
3783
|
self,
|
3757
3784
|
*cmd: str,
|
3758
3785
|
stdout: ta.Any = sys.stderr,
|
3759
3786
|
**kwargs: ta.Any,
|
3760
3787
|
) -> None:
|
3761
|
-
|
3762
|
-
subprocess.check_call(cmd, **kwargs)
|
3788
|
+
raise NotImplementedError
|
3763
3789
|
|
3790
|
+
@abc.abstractmethod
|
3764
3791
|
def check_output(
|
3765
3792
|
self,
|
3766
3793
|
*cmd: str,
|
3767
3794
|
**kwargs: ta.Any,
|
3768
3795
|
) -> bytes:
|
3769
|
-
|
3770
|
-
|
3796
|
+
raise NotImplementedError
|
3797
|
+
|
3798
|
+
#
|
3771
3799
|
|
3772
3800
|
def check_output_str(
|
3773
3801
|
self,
|
@@ -3809,9 +3837,94 @@ class Subprocesses(AbstractSubprocesses):
|
|
3809
3837
|
return ret.decode().strip()
|
3810
3838
|
|
3811
3839
|
|
3840
|
+
##
|
3841
|
+
|
3842
|
+
|
3843
|
+
class Subprocesses(AbstractSubprocesses):
|
3844
|
+
def check_call(
|
3845
|
+
self,
|
3846
|
+
*cmd: str,
|
3847
|
+
stdout: ta.Any = sys.stderr,
|
3848
|
+
**kwargs: ta.Any,
|
3849
|
+
) -> None:
|
3850
|
+
with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
|
3851
|
+
subprocess.check_call(cmd, **kwargs)
|
3852
|
+
|
3853
|
+
def check_output(
|
3854
|
+
self,
|
3855
|
+
*cmd: str,
|
3856
|
+
**kwargs: ta.Any,
|
3857
|
+
) -> bytes:
|
3858
|
+
with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
|
3859
|
+
return subprocess.check_output(cmd, **kwargs)
|
3860
|
+
|
3861
|
+
|
3812
3862
|
subprocesses = Subprocesses()
|
3813
3863
|
|
3814
3864
|
|
3865
|
+
##
|
3866
|
+
|
3867
|
+
|
3868
|
+
class AbstractAsyncSubprocesses(BaseSubprocesses):
|
3869
|
+
@abc.abstractmethod
|
3870
|
+
async def check_call(
|
3871
|
+
self,
|
3872
|
+
*cmd: str,
|
3873
|
+
stdout: ta.Any = sys.stderr,
|
3874
|
+
**kwargs: ta.Any,
|
3875
|
+
) -> None:
|
3876
|
+
raise NotImplementedError
|
3877
|
+
|
3878
|
+
@abc.abstractmethod
|
3879
|
+
async def check_output(
|
3880
|
+
self,
|
3881
|
+
*cmd: str,
|
3882
|
+
**kwargs: ta.Any,
|
3883
|
+
) -> bytes:
|
3884
|
+
raise NotImplementedError
|
3885
|
+
|
3886
|
+
#
|
3887
|
+
|
3888
|
+
async def check_output_str(
|
3889
|
+
self,
|
3890
|
+
*cmd: str,
|
3891
|
+
**kwargs: ta.Any,
|
3892
|
+
) -> str:
|
3893
|
+
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
3894
|
+
|
3895
|
+
#
|
3896
|
+
|
3897
|
+
async def try_call(
|
3898
|
+
self,
|
3899
|
+
*cmd: str,
|
3900
|
+
**kwargs: ta.Any,
|
3901
|
+
) -> bool:
|
3902
|
+
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
3903
|
+
return False
|
3904
|
+
else:
|
3905
|
+
return True
|
3906
|
+
|
3907
|
+
async def try_output(
|
3908
|
+
self,
|
3909
|
+
*cmd: str,
|
3910
|
+
**kwargs: ta.Any,
|
3911
|
+
) -> ta.Optional[bytes]:
|
3912
|
+
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
3913
|
+
return None
|
3914
|
+
else:
|
3915
|
+
return ret
|
3916
|
+
|
3917
|
+
async def try_output_str(
|
3918
|
+
self,
|
3919
|
+
*cmd: str,
|
3920
|
+
**kwargs: ta.Any,
|
3921
|
+
) -> ta.Optional[str]:
|
3922
|
+
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
3923
|
+
return None
|
3924
|
+
else:
|
3925
|
+
return ret.decode().strip()
|
3926
|
+
|
3927
|
+
|
3815
3928
|
########################################
|
3816
3929
|
# ../poster.py
|
3817
3930
|
"""
|