ominfra 0.0.0.dev156__py3-none-any.whl → 0.0.0.dev158__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/main.py +1 -1
- ominfra/journald/tailer.py +2 -2
- ominfra/manage/bootstrap_.py +1 -1
- ominfra/manage/commands/subprocess.py +4 -4
- ominfra/manage/deploy/apps.py +14 -15
- ominfra/manage/deploy/config.py +3 -0
- ominfra/manage/deploy/git.py +11 -27
- ominfra/manage/deploy/paths.py +48 -48
- ominfra/manage/deploy/specs.py +32 -0
- ominfra/manage/deploy/venvs.py +10 -5
- ominfra/manage/main.py +33 -4
- ominfra/manage/remote/spawning.py +4 -9
- ominfra/manage/system/packages.py +1 -1
- ominfra/pyremote.py +26 -26
- ominfra/scripts/journald2aws.py +469 -357
- ominfra/scripts/manage.py +2488 -1463
- ominfra/scripts/supervisor.py +385 -351
- ominfra/supervisor/configs.py +2 -0
- ominfra/supervisor/http.py +1 -1
- ominfra/supervisor/main.py +2 -2
- ominfra/supervisor/supervisor.py +2 -33
- ominfra/supervisor/utils/os.py +41 -0
- {ominfra-0.0.0.dev156.dist-info → ominfra-0.0.0.dev158.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev156.dist-info → ominfra-0.0.0.dev158.dist-info}/RECORD +28 -27
- {ominfra-0.0.0.dev156.dist-info → ominfra-0.0.0.dev158.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev156.dist-info → ominfra-0.0.0.dev158.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev156.dist-info → ominfra-0.0.0.dev158.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev156.dist-info → ominfra-0.0.0.dev158.dist-info}/top_level.txt +0 -0
ominfra/scripts/journald2aws.py
CHANGED
@@ -78,7 +78,7 @@ ConfigMapping = ta.Mapping[str, ta.Any]
|
|
78
78
|
# ../../../threadworkers.py
|
79
79
|
ThreadWorkerT = ta.TypeVar('ThreadWorkerT', bound='ThreadWorker')
|
80
80
|
|
81
|
-
# ../../../../omlish/
|
81
|
+
# ../../../../omlish/subprocesses.py
|
82
82
|
SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
|
83
83
|
|
84
84
|
|
@@ -932,8 +932,8 @@ class _CachedNullary(_AbstractCachedNullary):
|
|
932
932
|
return self._value
|
933
933
|
|
934
934
|
|
935
|
-
def cached_nullary(fn
|
936
|
-
return _CachedNullary(fn)
|
935
|
+
def cached_nullary(fn: CallableT) -> CallableT:
|
936
|
+
return _CachedNullary(fn) # type: ignore
|
937
937
|
|
938
938
|
|
939
939
|
def static_init(fn: CallableT) -> CallableT:
|
@@ -1437,6 +1437,13 @@ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON
|
|
1437
1437
|
json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
|
1438
1438
|
|
1439
1439
|
|
1440
|
+
########################################
|
1441
|
+
# ../../../../../omlish/lite/logs.py
|
1442
|
+
|
1443
|
+
|
1444
|
+
log = logging.getLogger(__name__)
|
1445
|
+
|
1446
|
+
|
1440
1447
|
########################################
|
1441
1448
|
# ../../../../../omlish/lite/reflect.py
|
1442
1449
|
|
@@ -1559,6 +1566,116 @@ def format_num_bytes(num_bytes: int) -> str:
|
|
1559
1566
|
return f'{num_bytes / 1024 ** (len(FORMAT_NUM_BYTES_SUFFIXES) - 1):.2f}{FORMAT_NUM_BYTES_SUFFIXES[-1]}'
|
1560
1567
|
|
1561
1568
|
|
1569
|
+
########################################
|
1570
|
+
# ../../../../../omlish/logs/filters.py
|
1571
|
+
|
1572
|
+
|
1573
|
+
class TidLogFilter(logging.Filter):
|
1574
|
+
def filter(self, record):
|
1575
|
+
record.tid = threading.get_native_id()
|
1576
|
+
return True
|
1577
|
+
|
1578
|
+
|
1579
|
+
########################################
|
1580
|
+
# ../../../../../omlish/logs/proxy.py
|
1581
|
+
|
1582
|
+
|
1583
|
+
class ProxyLogFilterer(logging.Filterer):
|
1584
|
+
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
1585
|
+
self._underlying = underlying
|
1586
|
+
|
1587
|
+
@property
|
1588
|
+
def underlying(self) -> logging.Filterer:
|
1589
|
+
return self._underlying
|
1590
|
+
|
1591
|
+
@property
|
1592
|
+
def filters(self):
|
1593
|
+
return self._underlying.filters
|
1594
|
+
|
1595
|
+
@filters.setter
|
1596
|
+
def filters(self, filters):
|
1597
|
+
self._underlying.filters = filters
|
1598
|
+
|
1599
|
+
def addFilter(self, filter): # noqa
|
1600
|
+
self._underlying.addFilter(filter)
|
1601
|
+
|
1602
|
+
def removeFilter(self, filter): # noqa
|
1603
|
+
self._underlying.removeFilter(filter)
|
1604
|
+
|
1605
|
+
def filter(self, record):
|
1606
|
+
return self._underlying.filter(record)
|
1607
|
+
|
1608
|
+
|
1609
|
+
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
1610
|
+
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
1611
|
+
ProxyLogFilterer.__init__(self, underlying)
|
1612
|
+
|
1613
|
+
_underlying: logging.Handler
|
1614
|
+
|
1615
|
+
@property
|
1616
|
+
def underlying(self) -> logging.Handler:
|
1617
|
+
return self._underlying
|
1618
|
+
|
1619
|
+
def get_name(self):
|
1620
|
+
return self._underlying.get_name()
|
1621
|
+
|
1622
|
+
def set_name(self, name):
|
1623
|
+
self._underlying.set_name(name)
|
1624
|
+
|
1625
|
+
@property
|
1626
|
+
def name(self):
|
1627
|
+
return self._underlying.name
|
1628
|
+
|
1629
|
+
@property
|
1630
|
+
def level(self):
|
1631
|
+
return self._underlying.level
|
1632
|
+
|
1633
|
+
@level.setter
|
1634
|
+
def level(self, level):
|
1635
|
+
self._underlying.level = level
|
1636
|
+
|
1637
|
+
@property
|
1638
|
+
def formatter(self):
|
1639
|
+
return self._underlying.formatter
|
1640
|
+
|
1641
|
+
@formatter.setter
|
1642
|
+
def formatter(self, formatter):
|
1643
|
+
self._underlying.formatter = formatter
|
1644
|
+
|
1645
|
+
def createLock(self):
|
1646
|
+
self._underlying.createLock()
|
1647
|
+
|
1648
|
+
def acquire(self):
|
1649
|
+
self._underlying.acquire()
|
1650
|
+
|
1651
|
+
def release(self):
|
1652
|
+
self._underlying.release()
|
1653
|
+
|
1654
|
+
def setLevel(self, level):
|
1655
|
+
self._underlying.setLevel(level)
|
1656
|
+
|
1657
|
+
def format(self, record):
|
1658
|
+
return self._underlying.format(record)
|
1659
|
+
|
1660
|
+
def emit(self, record):
|
1661
|
+
self._underlying.emit(record)
|
1662
|
+
|
1663
|
+
def handle(self, record):
|
1664
|
+
return self._underlying.handle(record)
|
1665
|
+
|
1666
|
+
def setFormatter(self, fmt):
|
1667
|
+
self._underlying.setFormatter(fmt)
|
1668
|
+
|
1669
|
+
def flush(self):
|
1670
|
+
self._underlying.flush()
|
1671
|
+
|
1672
|
+
def close(self):
|
1673
|
+
self._underlying.close()
|
1674
|
+
|
1675
|
+
def handleError(self, record):
|
1676
|
+
self._underlying.handleError(record)
|
1677
|
+
|
1678
|
+
|
1562
1679
|
########################################
|
1563
1680
|
# ../../../../../omlish/os/pidfile.py
|
1564
1681
|
|
@@ -1988,6 +2105,52 @@ class AwsDataclassMeta:
|
|
1988
2105
|
return AwsDataclassMeta.Converters(d2a, a2d)
|
1989
2106
|
|
1990
2107
|
|
2108
|
+
########################################
|
2109
|
+
# ../cursor.py
|
2110
|
+
|
2111
|
+
|
2112
|
+
class JournalctlToAwsCursor:
|
2113
|
+
def __init__(
|
2114
|
+
self,
|
2115
|
+
cursor_file: ta.Optional[str] = None,
|
2116
|
+
*,
|
2117
|
+
ensure_locked: ta.Optional[ta.Callable[[], None]] = None,
|
2118
|
+
) -> None:
|
2119
|
+
super().__init__()
|
2120
|
+
self._cursor_file = cursor_file
|
2121
|
+
self._ensure_locked = ensure_locked
|
2122
|
+
|
2123
|
+
#
|
2124
|
+
|
2125
|
+
def get(self) -> ta.Optional[str]:
|
2126
|
+
if self._ensure_locked is not None:
|
2127
|
+
self._ensure_locked()
|
2128
|
+
|
2129
|
+
if not (cf := self._cursor_file):
|
2130
|
+
return None
|
2131
|
+
cf = os.path.expanduser(cf)
|
2132
|
+
|
2133
|
+
try:
|
2134
|
+
with open(cf) as f:
|
2135
|
+
return f.read().strip()
|
2136
|
+
except FileNotFoundError:
|
2137
|
+
return None
|
2138
|
+
|
2139
|
+
def set(self, cursor: str) -> None:
|
2140
|
+
if self._ensure_locked is not None:
|
2141
|
+
self._ensure_locked()
|
2142
|
+
|
2143
|
+
if not (cf := self._cursor_file):
|
2144
|
+
return
|
2145
|
+
cf = os.path.expanduser(cf)
|
2146
|
+
|
2147
|
+
log.info('Writing cursor file %s : %s', cf, cursor)
|
2148
|
+
with open(ncf := cf + '.next', 'w') as f:
|
2149
|
+
f.write(cursor)
|
2150
|
+
|
2151
|
+
os.rename(ncf, cf)
|
2152
|
+
|
2153
|
+
|
1991
2154
|
########################################
|
1992
2155
|
# ../../../../../omlish/io/buffers.py
|
1993
2156
|
|
@@ -2283,340 +2446,71 @@ class aclosing(contextlib.AbstractAsyncContextManager): # noqa
|
|
2283
2446
|
|
2284
2447
|
|
2285
2448
|
########################################
|
2286
|
-
# ../../../../../omlish/lite/
|
2449
|
+
# ../../../../../omlish/lite/marshal.py
|
2287
2450
|
"""
|
2288
2451
|
TODO:
|
2289
|
-
-
|
2290
|
-
-
|
2452
|
+
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
2453
|
+
- namedtuple
|
2454
|
+
- literals
|
2455
|
+
- newtypes?
|
2291
2456
|
"""
|
2292
2457
|
|
2293
2458
|
|
2294
|
-
|
2459
|
+
##
|
2295
2460
|
|
2296
2461
|
|
2297
|
-
|
2462
|
+
@dc.dataclass(frozen=True)
|
2463
|
+
class ObjMarshalOptions:
|
2464
|
+
raw_bytes: bool = False
|
2465
|
+
nonstrict_dataclasses: bool = False
|
2298
2466
|
|
2299
2467
|
|
2300
|
-
class
|
2468
|
+
class ObjMarshaler(abc.ABC):
|
2469
|
+
@abc.abstractmethod
|
2470
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2471
|
+
raise NotImplementedError
|
2301
2472
|
|
2302
|
-
|
2303
|
-
|
2304
|
-
|
2473
|
+
@abc.abstractmethod
|
2474
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2475
|
+
raise NotImplementedError
|
2305
2476
|
|
2306
2477
|
|
2307
|
-
|
2478
|
+
class NopObjMarshaler(ObjMarshaler):
|
2479
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2480
|
+
return o
|
2308
2481
|
|
2482
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2483
|
+
return o
|
2309
2484
|
|
2310
|
-
class JsonLogFormatter(logging.Formatter):
|
2311
2485
|
|
2312
|
-
|
2313
|
-
|
2314
|
-
|
2315
|
-
'args': False,
|
2316
|
-
'levelname': False,
|
2317
|
-
'levelno': False,
|
2318
|
-
'pathname': False,
|
2319
|
-
'filename': False,
|
2320
|
-
'module': False,
|
2321
|
-
'exc_info': True,
|
2322
|
-
'exc_text': True,
|
2323
|
-
'stack_info': True,
|
2324
|
-
'lineno': False,
|
2325
|
-
'funcName': False,
|
2326
|
-
'created': False,
|
2327
|
-
'msecs': False,
|
2328
|
-
'relativeCreated': False,
|
2329
|
-
'thread': False,
|
2330
|
-
'threadName': False,
|
2331
|
-
'processName': False,
|
2332
|
-
'process': False,
|
2333
|
-
}
|
2486
|
+
@dc.dataclass()
|
2487
|
+
class ProxyObjMarshaler(ObjMarshaler):
|
2488
|
+
m: ta.Optional[ObjMarshaler] = None
|
2334
2489
|
|
2335
|
-
def
|
2336
|
-
|
2337
|
-
k: v
|
2338
|
-
for k, o in self.KEYS.items()
|
2339
|
-
for v in [getattr(record, k)]
|
2340
|
-
if not (o and v is None)
|
2341
|
-
}
|
2342
|
-
return json_dumps_compact(dct)
|
2490
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2491
|
+
return check.not_none(self.m).marshal(o, ctx)
|
2343
2492
|
|
2493
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2494
|
+
return check.not_none(self.m).unmarshal(o, ctx)
|
2344
2495
|
|
2345
|
-
##
|
2346
2496
|
|
2497
|
+
@dc.dataclass(frozen=True)
|
2498
|
+
class CastObjMarshaler(ObjMarshaler):
|
2499
|
+
ty: type
|
2347
2500
|
|
2348
|
-
|
2349
|
-
|
2350
|
-
('process', 'pid=%(process)-6s'),
|
2351
|
-
('thread', 'tid=%(thread)x'),
|
2352
|
-
('levelname', '%(levelname)s'),
|
2353
|
-
('name', '%(name)s'),
|
2354
|
-
('separator', '::'),
|
2355
|
-
('message', '%(message)s'),
|
2356
|
-
]
|
2501
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2502
|
+
return o
|
2357
2503
|
|
2504
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2505
|
+
return self.ty(o)
|
2358
2506
|
|
2359
|
-
class StandardLogFormatter(logging.Formatter):
|
2360
2507
|
|
2361
|
-
|
2362
|
-
def
|
2363
|
-
return
|
2508
|
+
class DynamicObjMarshaler(ObjMarshaler):
|
2509
|
+
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2510
|
+
return ctx.manager.marshal_obj(o, opts=ctx.options)
|
2364
2511
|
|
2365
|
-
|
2366
|
-
|
2367
|
-
def formatTime(self, record, datefmt=None):
|
2368
|
-
ct = self.converter(record.created) # type: ignore
|
2369
|
-
if datefmt:
|
2370
|
-
return ct.strftime(datefmt) # noqa
|
2371
|
-
else:
|
2372
|
-
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
2373
|
-
return '%s.%03d' % (t, record.msecs) # noqa
|
2374
|
-
|
2375
|
-
|
2376
|
-
##
|
2377
|
-
|
2378
|
-
|
2379
|
-
class ProxyLogFilterer(logging.Filterer):
|
2380
|
-
def __init__(self, underlying: logging.Filterer) -> None: # noqa
|
2381
|
-
self._underlying = underlying
|
2382
|
-
|
2383
|
-
@property
|
2384
|
-
def underlying(self) -> logging.Filterer:
|
2385
|
-
return self._underlying
|
2386
|
-
|
2387
|
-
@property
|
2388
|
-
def filters(self):
|
2389
|
-
return self._underlying.filters
|
2390
|
-
|
2391
|
-
@filters.setter
|
2392
|
-
def filters(self, filters):
|
2393
|
-
self._underlying.filters = filters
|
2394
|
-
|
2395
|
-
def addFilter(self, filter): # noqa
|
2396
|
-
self._underlying.addFilter(filter)
|
2397
|
-
|
2398
|
-
def removeFilter(self, filter): # noqa
|
2399
|
-
self._underlying.removeFilter(filter)
|
2400
|
-
|
2401
|
-
def filter(self, record):
|
2402
|
-
return self._underlying.filter(record)
|
2403
|
-
|
2404
|
-
|
2405
|
-
class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
|
2406
|
-
def __init__(self, underlying: logging.Handler) -> None: # noqa
|
2407
|
-
ProxyLogFilterer.__init__(self, underlying)
|
2408
|
-
|
2409
|
-
_underlying: logging.Handler
|
2410
|
-
|
2411
|
-
@property
|
2412
|
-
def underlying(self) -> logging.Handler:
|
2413
|
-
return self._underlying
|
2414
|
-
|
2415
|
-
def get_name(self):
|
2416
|
-
return self._underlying.get_name()
|
2417
|
-
|
2418
|
-
def set_name(self, name):
|
2419
|
-
self._underlying.set_name(name)
|
2420
|
-
|
2421
|
-
@property
|
2422
|
-
def name(self):
|
2423
|
-
return self._underlying.name
|
2424
|
-
|
2425
|
-
@property
|
2426
|
-
def level(self):
|
2427
|
-
return self._underlying.level
|
2428
|
-
|
2429
|
-
@level.setter
|
2430
|
-
def level(self, level):
|
2431
|
-
self._underlying.level = level
|
2432
|
-
|
2433
|
-
@property
|
2434
|
-
def formatter(self):
|
2435
|
-
return self._underlying.formatter
|
2436
|
-
|
2437
|
-
@formatter.setter
|
2438
|
-
def formatter(self, formatter):
|
2439
|
-
self._underlying.formatter = formatter
|
2440
|
-
|
2441
|
-
def createLock(self):
|
2442
|
-
self._underlying.createLock()
|
2443
|
-
|
2444
|
-
def acquire(self):
|
2445
|
-
self._underlying.acquire()
|
2446
|
-
|
2447
|
-
def release(self):
|
2448
|
-
self._underlying.release()
|
2449
|
-
|
2450
|
-
def setLevel(self, level):
|
2451
|
-
self._underlying.setLevel(level)
|
2452
|
-
|
2453
|
-
def format(self, record):
|
2454
|
-
return self._underlying.format(record)
|
2455
|
-
|
2456
|
-
def emit(self, record):
|
2457
|
-
self._underlying.emit(record)
|
2458
|
-
|
2459
|
-
def handle(self, record):
|
2460
|
-
return self._underlying.handle(record)
|
2461
|
-
|
2462
|
-
def setFormatter(self, fmt):
|
2463
|
-
self._underlying.setFormatter(fmt)
|
2464
|
-
|
2465
|
-
def flush(self):
|
2466
|
-
self._underlying.flush()
|
2467
|
-
|
2468
|
-
def close(self):
|
2469
|
-
self._underlying.close()
|
2470
|
-
|
2471
|
-
def handleError(self, record):
|
2472
|
-
self._underlying.handleError(record)
|
2473
|
-
|
2474
|
-
|
2475
|
-
##
|
2476
|
-
|
2477
|
-
|
2478
|
-
class StandardLogHandler(ProxyLogHandler):
|
2479
|
-
pass
|
2480
|
-
|
2481
|
-
|
2482
|
-
##
|
2483
|
-
|
2484
|
-
|
2485
|
-
@contextlib.contextmanager
|
2486
|
-
def _locking_logging_module_lock() -> ta.Iterator[None]:
|
2487
|
-
if hasattr(logging, '_acquireLock'):
|
2488
|
-
logging._acquireLock() # noqa
|
2489
|
-
try:
|
2490
|
-
yield
|
2491
|
-
finally:
|
2492
|
-
logging._releaseLock() # type: ignore # noqa
|
2493
|
-
|
2494
|
-
elif hasattr(logging, '_lock'):
|
2495
|
-
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
2496
|
-
with logging._lock: # noqa
|
2497
|
-
yield
|
2498
|
-
|
2499
|
-
else:
|
2500
|
-
raise Exception("Can't find lock in logging module")
|
2501
|
-
|
2502
|
-
|
2503
|
-
def configure_standard_logging(
|
2504
|
-
level: ta.Union[int, str] = logging.INFO,
|
2505
|
-
*,
|
2506
|
-
json: bool = False,
|
2507
|
-
target: ta.Optional[logging.Logger] = None,
|
2508
|
-
force: bool = False,
|
2509
|
-
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
2510
|
-
) -> ta.Optional[StandardLogHandler]:
|
2511
|
-
with _locking_logging_module_lock():
|
2512
|
-
if target is None:
|
2513
|
-
target = logging.root
|
2514
|
-
|
2515
|
-
#
|
2516
|
-
|
2517
|
-
if not force:
|
2518
|
-
if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
|
2519
|
-
return None
|
2520
|
-
|
2521
|
-
#
|
2522
|
-
|
2523
|
-
if handler_factory is not None:
|
2524
|
-
handler = handler_factory()
|
2525
|
-
else:
|
2526
|
-
handler = logging.StreamHandler()
|
2527
|
-
|
2528
|
-
#
|
2529
|
-
|
2530
|
-
formatter: logging.Formatter
|
2531
|
-
if json:
|
2532
|
-
formatter = JsonLogFormatter()
|
2533
|
-
else:
|
2534
|
-
formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
|
2535
|
-
handler.setFormatter(formatter)
|
2536
|
-
|
2537
|
-
#
|
2538
|
-
|
2539
|
-
handler.addFilter(TidLogFilter())
|
2540
|
-
|
2541
|
-
#
|
2542
|
-
|
2543
|
-
target.addHandler(handler)
|
2544
|
-
|
2545
|
-
#
|
2546
|
-
|
2547
|
-
if level is not None:
|
2548
|
-
target.setLevel(level)
|
2549
|
-
|
2550
|
-
#
|
2551
|
-
|
2552
|
-
return StandardLogHandler(handler)
|
2553
|
-
|
2554
|
-
|
2555
|
-
########################################
|
2556
|
-
# ../../../../../omlish/lite/marshal.py
|
2557
|
-
"""
|
2558
|
-
TODO:
|
2559
|
-
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
2560
|
-
- namedtuple
|
2561
|
-
- literals
|
2562
|
-
"""
|
2563
|
-
|
2564
|
-
|
2565
|
-
##
|
2566
|
-
|
2567
|
-
|
2568
|
-
@dc.dataclass(frozen=True)
|
2569
|
-
class ObjMarshalOptions:
|
2570
|
-
raw_bytes: bool = False
|
2571
|
-
nonstrict_dataclasses: bool = False
|
2572
|
-
|
2573
|
-
|
2574
|
-
class ObjMarshaler(abc.ABC):
|
2575
|
-
@abc.abstractmethod
|
2576
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2577
|
-
raise NotImplementedError
|
2578
|
-
|
2579
|
-
@abc.abstractmethod
|
2580
|
-
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2581
|
-
raise NotImplementedError
|
2582
|
-
|
2583
|
-
|
2584
|
-
class NopObjMarshaler(ObjMarshaler):
|
2585
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2586
|
-
return o
|
2587
|
-
|
2588
|
-
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2589
|
-
return o
|
2590
|
-
|
2591
|
-
|
2592
|
-
@dc.dataclass()
|
2593
|
-
class ProxyObjMarshaler(ObjMarshaler):
|
2594
|
-
m: ta.Optional[ObjMarshaler] = None
|
2595
|
-
|
2596
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2597
|
-
return check.not_none(self.m).marshal(o, ctx)
|
2598
|
-
|
2599
|
-
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2600
|
-
return check.not_none(self.m).unmarshal(o, ctx)
|
2601
|
-
|
2602
|
-
|
2603
|
-
@dc.dataclass(frozen=True)
|
2604
|
-
class CastObjMarshaler(ObjMarshaler):
|
2605
|
-
ty: type
|
2606
|
-
|
2607
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2608
|
-
return o
|
2609
|
-
|
2610
|
-
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2611
|
-
return self.ty(o)
|
2612
|
-
|
2613
|
-
|
2614
|
-
class DynamicObjMarshaler(ObjMarshaler):
|
2615
|
-
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2616
|
-
return ctx.manager.marshal_obj(o, opts=ctx.options)
|
2617
|
-
|
2618
|
-
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2619
|
-
return o
|
2512
|
+
def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
2513
|
+
return o
|
2620
2514
|
|
2621
2515
|
|
2622
2516
|
@dc.dataclass(frozen=True)
|
@@ -3014,49 +2908,57 @@ def check_runtime_version() -> None:
|
|
3014
2908
|
|
3015
2909
|
|
3016
2910
|
########################################
|
3017
|
-
#
|
2911
|
+
# ../../../../../omlish/logs/json.py
|
2912
|
+
"""
|
2913
|
+
TODO:
|
2914
|
+
- translate json keys
|
2915
|
+
"""
|
3018
2916
|
|
3019
2917
|
|
3020
|
-
class
|
2918
|
+
class JsonLogFormatter(logging.Formatter):
|
2919
|
+
KEYS: ta.Mapping[str, bool] = {
|
2920
|
+
'name': False,
|
2921
|
+
'msg': False,
|
2922
|
+
'args': False,
|
2923
|
+
'levelname': False,
|
2924
|
+
'levelno': False,
|
2925
|
+
'pathname': False,
|
2926
|
+
'filename': False,
|
2927
|
+
'module': False,
|
2928
|
+
'exc_info': True,
|
2929
|
+
'exc_text': True,
|
2930
|
+
'stack_info': True,
|
2931
|
+
'lineno': False,
|
2932
|
+
'funcName': False,
|
2933
|
+
'created': False,
|
2934
|
+
'msecs': False,
|
2935
|
+
'relativeCreated': False,
|
2936
|
+
'thread': False,
|
2937
|
+
'threadName': False,
|
2938
|
+
'processName': False,
|
2939
|
+
'process': False,
|
2940
|
+
}
|
2941
|
+
|
3021
2942
|
def __init__(
|
3022
2943
|
self,
|
3023
|
-
|
3024
|
-
|
3025
|
-
|
2944
|
+
*args: ta.Any,
|
2945
|
+
json_dumps: ta.Optional[ta.Callable[[ta.Any], str]] = None,
|
2946
|
+
**kwargs: ta.Any,
|
3026
2947
|
) -> None:
|
3027
|
-
super().__init__()
|
3028
|
-
self._cursor_file = cursor_file
|
3029
|
-
self._ensure_locked = ensure_locked
|
3030
|
-
|
3031
|
-
#
|
3032
|
-
|
3033
|
-
def get(self) -> ta.Optional[str]:
|
3034
|
-
if self._ensure_locked is not None:
|
3035
|
-
self._ensure_locked()
|
3036
|
-
|
3037
|
-
if not (cf := self._cursor_file):
|
3038
|
-
return None
|
3039
|
-
cf = os.path.expanduser(cf)
|
3040
|
-
|
3041
|
-
try:
|
3042
|
-
with open(cf) as f:
|
3043
|
-
return f.read().strip()
|
3044
|
-
except FileNotFoundError:
|
3045
|
-
return None
|
3046
|
-
|
3047
|
-
def set(self, cursor: str) -> None:
|
3048
|
-
if self._ensure_locked is not None:
|
3049
|
-
self._ensure_locked()
|
3050
|
-
|
3051
|
-
if not (cf := self._cursor_file):
|
3052
|
-
return
|
3053
|
-
cf = os.path.expanduser(cf)
|
2948
|
+
super().__init__(*args, **kwargs)
|
3054
2949
|
|
3055
|
-
|
3056
|
-
|
3057
|
-
|
2950
|
+
if json_dumps is None:
|
2951
|
+
json_dumps = json_dumps_compact
|
2952
|
+
self._json_dumps = json_dumps
|
3058
2953
|
|
3059
|
-
|
2954
|
+
def format(self, record: logging.LogRecord) -> str:
|
2955
|
+
dct = {
|
2956
|
+
k: v
|
2957
|
+
for k, o in self.KEYS.items()
|
2958
|
+
for v in [getattr(record, k)]
|
2959
|
+
if not (o and v is None)
|
2960
|
+
}
|
2961
|
+
return self._json_dumps(dct)
|
3060
2962
|
|
3061
2963
|
|
3062
2964
|
########################################
|
@@ -3574,7 +3476,127 @@ class ThreadWorkerGroup:
|
|
3574
3476
|
|
3575
3477
|
|
3576
3478
|
########################################
|
3577
|
-
# ../../../../../omlish/
|
3479
|
+
# ../../../../../omlish/logs/standard.py
|
3480
|
+
"""
|
3481
|
+
TODO:
|
3482
|
+
- structured
|
3483
|
+
- prefixed
|
3484
|
+
- debug
|
3485
|
+
"""
|
3486
|
+
|
3487
|
+
|
3488
|
+
##
|
3489
|
+
|
3490
|
+
|
3491
|
+
STANDARD_LOG_FORMAT_PARTS = [
|
3492
|
+
('asctime', '%(asctime)-15s'),
|
3493
|
+
('process', 'pid=%(process)-6s'),
|
3494
|
+
('thread', 'tid=%(thread)x'),
|
3495
|
+
('levelname', '%(levelname)s'),
|
3496
|
+
('name', '%(name)s'),
|
3497
|
+
('separator', '::'),
|
3498
|
+
('message', '%(message)s'),
|
3499
|
+
]
|
3500
|
+
|
3501
|
+
|
3502
|
+
class StandardLogFormatter(logging.Formatter):
|
3503
|
+
@staticmethod
|
3504
|
+
def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
|
3505
|
+
return ' '.join(v for k, v in parts)
|
3506
|
+
|
3507
|
+
converter = datetime.datetime.fromtimestamp # type: ignore
|
3508
|
+
|
3509
|
+
def formatTime(self, record, datefmt=None):
|
3510
|
+
ct = self.converter(record.created) # type: ignore
|
3511
|
+
if datefmt:
|
3512
|
+
return ct.strftime(datefmt) # noqa
|
3513
|
+
else:
|
3514
|
+
t = ct.strftime('%Y-%m-%d %H:%M:%S')
|
3515
|
+
return '%s.%03d' % (t, record.msecs) # noqa
|
3516
|
+
|
3517
|
+
|
3518
|
+
##
|
3519
|
+
|
3520
|
+
|
3521
|
+
class StandardLogHandler(ProxyLogHandler):
|
3522
|
+
pass
|
3523
|
+
|
3524
|
+
|
3525
|
+
##
|
3526
|
+
|
3527
|
+
|
3528
|
+
@contextlib.contextmanager
|
3529
|
+
def _locking_logging_module_lock() -> ta.Iterator[None]:
|
3530
|
+
if hasattr(logging, '_acquireLock'):
|
3531
|
+
logging._acquireLock() # noqa
|
3532
|
+
try:
|
3533
|
+
yield
|
3534
|
+
finally:
|
3535
|
+
logging._releaseLock() # type: ignore # noqa
|
3536
|
+
|
3537
|
+
elif hasattr(logging, '_lock'):
|
3538
|
+
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
3539
|
+
with logging._lock: # noqa
|
3540
|
+
yield
|
3541
|
+
|
3542
|
+
else:
|
3543
|
+
raise Exception("Can't find lock in logging module")
|
3544
|
+
|
3545
|
+
|
3546
|
+
def configure_standard_logging(
|
3547
|
+
level: ta.Union[int, str] = logging.INFO,
|
3548
|
+
*,
|
3549
|
+
json: bool = False,
|
3550
|
+
target: ta.Optional[logging.Logger] = None,
|
3551
|
+
force: bool = False,
|
3552
|
+
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
3553
|
+
) -> ta.Optional[StandardLogHandler]:
|
3554
|
+
with _locking_logging_module_lock():
|
3555
|
+
if target is None:
|
3556
|
+
target = logging.root
|
3557
|
+
|
3558
|
+
#
|
3559
|
+
|
3560
|
+
if not force:
|
3561
|
+
if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
|
3562
|
+
return None
|
3563
|
+
|
3564
|
+
#
|
3565
|
+
|
3566
|
+
if handler_factory is not None:
|
3567
|
+
handler = handler_factory()
|
3568
|
+
else:
|
3569
|
+
handler = logging.StreamHandler()
|
3570
|
+
|
3571
|
+
#
|
3572
|
+
|
3573
|
+
formatter: logging.Formatter
|
3574
|
+
if json:
|
3575
|
+
formatter = JsonLogFormatter()
|
3576
|
+
else:
|
3577
|
+
formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
|
3578
|
+
handler.setFormatter(formatter)
|
3579
|
+
|
3580
|
+
#
|
3581
|
+
|
3582
|
+
handler.addFilter(TidLogFilter())
|
3583
|
+
|
3584
|
+
#
|
3585
|
+
|
3586
|
+
target.addHandler(handler)
|
3587
|
+
|
3588
|
+
#
|
3589
|
+
|
3590
|
+
if level is not None:
|
3591
|
+
target.setLevel(level)
|
3592
|
+
|
3593
|
+
#
|
3594
|
+
|
3595
|
+
return StandardLogHandler(handler)
|
3596
|
+
|
3597
|
+
|
3598
|
+
########################################
|
3599
|
+
# ../../../../../omlish/subprocesses.py
|
3578
3600
|
|
3579
3601
|
|
3580
3602
|
##
|
@@ -3625,8 +3647,8 @@ def subprocess_close(
|
|
3625
3647
|
##
|
3626
3648
|
|
3627
3649
|
|
3628
|
-
class
|
3629
|
-
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] =
|
3650
|
+
class BaseSubprocesses(abc.ABC): # noqa
|
3651
|
+
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
|
3630
3652
|
|
3631
3653
|
def __init__(
|
3632
3654
|
self,
|
@@ -3639,6 +3661,9 @@ class AbstractSubprocesses(abc.ABC): # noqa
|
|
3639
3661
|
self._log = log if log is not None else self.DEFAULT_LOGGER
|
3640
3662
|
self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
|
3641
3663
|
|
3664
|
+
def set_logger(self, log: ta.Optional[logging.Logger]) -> None:
|
3665
|
+
self._log = log
|
3666
|
+
|
3642
3667
|
#
|
3643
3668
|
|
3644
3669
|
def prepare_args(
|
@@ -3750,23 +3775,25 @@ class AbstractSubprocesses(abc.ABC): # noqa
|
|
3750
3775
|
##
|
3751
3776
|
|
3752
3777
|
|
3753
|
-
class
|
3778
|
+
class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
|
3779
|
+
@abc.abstractmethod
|
3754
3780
|
def check_call(
|
3755
3781
|
self,
|
3756
3782
|
*cmd: str,
|
3757
3783
|
stdout: ta.Any = sys.stderr,
|
3758
3784
|
**kwargs: ta.Any,
|
3759
3785
|
) -> None:
|
3760
|
-
|
3761
|
-
subprocess.check_call(cmd, **kwargs)
|
3786
|
+
raise NotImplementedError
|
3762
3787
|
|
3788
|
+
@abc.abstractmethod
|
3763
3789
|
def check_output(
|
3764
3790
|
self,
|
3765
3791
|
*cmd: str,
|
3766
3792
|
**kwargs: ta.Any,
|
3767
3793
|
) -> bytes:
|
3768
|
-
|
3769
|
-
|
3794
|
+
raise NotImplementedError
|
3795
|
+
|
3796
|
+
#
|
3770
3797
|
|
3771
3798
|
def check_output_str(
|
3772
3799
|
self,
|
@@ -3808,9 +3835,94 @@ class Subprocesses(AbstractSubprocesses):
|
|
3808
3835
|
return ret.decode().strip()
|
3809
3836
|
|
3810
3837
|
|
3838
|
+
##
|
3839
|
+
|
3840
|
+
|
3841
|
+
class Subprocesses(AbstractSubprocesses):
|
3842
|
+
def check_call(
|
3843
|
+
self,
|
3844
|
+
*cmd: str,
|
3845
|
+
stdout: ta.Any = sys.stderr,
|
3846
|
+
**kwargs: ta.Any,
|
3847
|
+
) -> None:
|
3848
|
+
with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
|
3849
|
+
subprocess.check_call(cmd, **kwargs)
|
3850
|
+
|
3851
|
+
def check_output(
|
3852
|
+
self,
|
3853
|
+
*cmd: str,
|
3854
|
+
**kwargs: ta.Any,
|
3855
|
+
) -> bytes:
|
3856
|
+
with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
|
3857
|
+
return subprocess.check_output(cmd, **kwargs)
|
3858
|
+
|
3859
|
+
|
3811
3860
|
subprocesses = Subprocesses()
|
3812
3861
|
|
3813
3862
|
|
3863
|
+
##
|
3864
|
+
|
3865
|
+
|
3866
|
+
class AbstractAsyncSubprocesses(BaseSubprocesses):
|
3867
|
+
@abc.abstractmethod
|
3868
|
+
async def check_call(
|
3869
|
+
self,
|
3870
|
+
*cmd: str,
|
3871
|
+
stdout: ta.Any = sys.stderr,
|
3872
|
+
**kwargs: ta.Any,
|
3873
|
+
) -> None:
|
3874
|
+
raise NotImplementedError
|
3875
|
+
|
3876
|
+
@abc.abstractmethod
|
3877
|
+
async def check_output(
|
3878
|
+
self,
|
3879
|
+
*cmd: str,
|
3880
|
+
**kwargs: ta.Any,
|
3881
|
+
) -> bytes:
|
3882
|
+
raise NotImplementedError
|
3883
|
+
|
3884
|
+
#
|
3885
|
+
|
3886
|
+
async def check_output_str(
|
3887
|
+
self,
|
3888
|
+
*cmd: str,
|
3889
|
+
**kwargs: ta.Any,
|
3890
|
+
) -> str:
|
3891
|
+
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
3892
|
+
|
3893
|
+
#
|
3894
|
+
|
3895
|
+
async def try_call(
|
3896
|
+
self,
|
3897
|
+
*cmd: str,
|
3898
|
+
**kwargs: ta.Any,
|
3899
|
+
) -> bool:
|
3900
|
+
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
3901
|
+
return False
|
3902
|
+
else:
|
3903
|
+
return True
|
3904
|
+
|
3905
|
+
async def try_output(
|
3906
|
+
self,
|
3907
|
+
*cmd: str,
|
3908
|
+
**kwargs: ta.Any,
|
3909
|
+
) -> ta.Optional[bytes]:
|
3910
|
+
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
3911
|
+
return None
|
3912
|
+
else:
|
3913
|
+
return ret
|
3914
|
+
|
3915
|
+
async def try_output_str(
|
3916
|
+
self,
|
3917
|
+
*cmd: str,
|
3918
|
+
**kwargs: ta.Any,
|
3919
|
+
) -> ta.Optional[str]:
|
3920
|
+
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
3921
|
+
return None
|
3922
|
+
else:
|
3923
|
+
return ret.decode().strip()
|
3924
|
+
|
3925
|
+
|
3814
3926
|
########################################
|
3815
3927
|
# ../poster.py
|
3816
3928
|
"""
|