ominfra 0.0.0.dev90__py3-none-any.whl → 0.0.0.dev91__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/main.py +36 -19
- ominfra/clouds/aws/logs.py +2 -2
- ominfra/journald/__init__.py +0 -0
- ominfra/{clouds/aws/journald2aws/journald → journald}/messages.py +27 -15
- ominfra/journald/tailer.py +453 -0
- ominfra/scripts/journald2aws.py +548 -147
- ominfra/{clouds/aws/journald2aws/threadworker.py → threadworker.py} +6 -3
- {ominfra-0.0.0.dev90.dist-info → ominfra-0.0.0.dev91.dist-info}/METADATA +4 -4
- {ominfra-0.0.0.dev90.dist-info → ominfra-0.0.0.dev91.dist-info}/RECORD +14 -14
- ominfra/clouds/aws/journald2aws/journald/__init__.py +0 -1
- ominfra/clouds/aws/journald2aws/journald/tailer.py +0 -108
- /ominfra/{clouds/aws/journald2aws/journald → journald}/genmessages.py +0 -0
- {ominfra-0.0.0.dev90.dist-info → ominfra-0.0.0.dev91.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev90.dist-info → ominfra-0.0.0.dev91.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev90.dist-info → ominfra-0.0.0.dev91.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev90.dist-info → ominfra-0.0.0.dev91.dist-info}/top_level.txt +0 -0
ominfra/scripts/journald2aws.py
CHANGED
@@ -7,7 +7,7 @@
|
|
7
7
|
"""
|
8
8
|
TODO:
|
9
9
|
- create log group
|
10
|
-
- log stats - chunk sizes etc
|
10
|
+
- log stats - chunk sizes, byte count, num calls, etc
|
11
11
|
|
12
12
|
==
|
13
13
|
|
@@ -311,6 +311,9 @@ def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
|
|
311
311
|
# ../../../../../omlish/lite/strings.py
|
312
312
|
|
313
313
|
|
314
|
+
##
|
315
|
+
|
316
|
+
|
314
317
|
def camel_case(name: str, lower: bool = False) -> str:
|
315
318
|
if not name:
|
316
319
|
return ''
|
@@ -325,6 +328,9 @@ def snake_case(name: str) -> str:
|
|
325
328
|
return '_'.join([name[l:r].lower() for l, r in zip([None, *uppers], [*uppers, None])]).strip('_')
|
326
329
|
|
327
330
|
|
331
|
+
##
|
332
|
+
|
333
|
+
|
328
334
|
def is_dunder(name: str) -> bool:
|
329
335
|
return (
|
330
336
|
name[:2] == name[-2:] == '__' and
|
@@ -343,10 +349,31 @@ def is_sunder(name: str) -> bool:
|
|
343
349
|
)
|
344
350
|
|
345
351
|
|
352
|
+
##
|
353
|
+
|
354
|
+
|
346
355
|
def attr_repr(obj: ta.Any, *attrs: str) -> str:
|
347
356
|
return f'{type(obj).__name__}({", ".join(f"{attr}={getattr(obj, attr)!r}" for attr in attrs)})'
|
348
357
|
|
349
358
|
|
359
|
+
##
|
360
|
+
|
361
|
+
|
362
|
+
FORMAT_NUM_BYTES_SUFFIXES: ta.Sequence[str] = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB']
|
363
|
+
|
364
|
+
|
365
|
+
def format_num_bytes(num_bytes: int) -> str:
|
366
|
+
for i, suffix in enumerate(FORMAT_NUM_BYTES_SUFFIXES):
|
367
|
+
value = num_bytes / 1024 ** i
|
368
|
+
if num_bytes < 1024 ** (i + 1):
|
369
|
+
if value.is_integer():
|
370
|
+
return f'{int(value)}{suffix}'
|
371
|
+
else:
|
372
|
+
return f'{value:.2f}{suffix}'
|
373
|
+
|
374
|
+
return f'{num_bytes / 1024 ** (len(FORMAT_NUM_BYTES_SUFFIXES) - 1):.2f}{FORMAT_NUM_BYTES_SUFFIXES[-1]}'
|
375
|
+
|
376
|
+
|
350
377
|
########################################
|
351
378
|
# ../../auth.py
|
352
379
|
"""
|
@@ -1421,129 +1448,6 @@ def check_runtime_version() -> None:
|
|
1421
1448
|
f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
|
1422
1449
|
|
1423
1450
|
|
1424
|
-
########################################
|
1425
|
-
# ../journald/messages.py
|
1426
|
-
|
1427
|
-
|
1428
|
-
@dc.dataclass(frozen=True)
|
1429
|
-
class JournalctlMessage:
|
1430
|
-
raw: bytes
|
1431
|
-
dct: ta.Optional[ta.Mapping[str, ta.Any]] = None
|
1432
|
-
cursor: ta.Optional[str] = None
|
1433
|
-
ts_us: ta.Optional[int] = None # microseconds UTC
|
1434
|
-
|
1435
|
-
|
1436
|
-
class JournalctlMessageBuilder:
|
1437
|
-
def __init__(self) -> None:
|
1438
|
-
super().__init__()
|
1439
|
-
|
1440
|
-
self._buf = DelimitingBuffer(b'\n')
|
1441
|
-
|
1442
|
-
_cursor_field = '__CURSOR'
|
1443
|
-
_timestamp_field = '_SOURCE_REALTIME_TIMESTAMP'
|
1444
|
-
|
1445
|
-
def _make_message(self, raw: bytes) -> JournalctlMessage:
|
1446
|
-
dct = None
|
1447
|
-
cursor = None
|
1448
|
-
ts = None
|
1449
|
-
|
1450
|
-
try:
|
1451
|
-
dct = json.loads(raw.decode('utf-8', 'replace'))
|
1452
|
-
except Exception: # noqa
|
1453
|
-
log.exception('Failed to parse raw message: %r', raw)
|
1454
|
-
|
1455
|
-
else:
|
1456
|
-
cursor = dct.get(self._cursor_field)
|
1457
|
-
|
1458
|
-
if tsv := dct.get(self._timestamp_field):
|
1459
|
-
if isinstance(tsv, str):
|
1460
|
-
try:
|
1461
|
-
ts = int(tsv)
|
1462
|
-
except ValueError:
|
1463
|
-
try:
|
1464
|
-
ts = int(float(tsv))
|
1465
|
-
except ValueError:
|
1466
|
-
log.exception('Failed to parse timestamp: %r', tsv)
|
1467
|
-
elif isinstance(tsv, (int, float)):
|
1468
|
-
ts = int(tsv)
|
1469
|
-
else:
|
1470
|
-
log.exception('Invalid timestamp: %r', tsv)
|
1471
|
-
|
1472
|
-
return JournalctlMessage(
|
1473
|
-
raw=raw,
|
1474
|
-
dct=dct,
|
1475
|
-
cursor=cursor,
|
1476
|
-
ts_us=ts,
|
1477
|
-
)
|
1478
|
-
|
1479
|
-
def feed(self, data: bytes) -> ta.Sequence[JournalctlMessage]:
|
1480
|
-
ret: ta.List[JournalctlMessage] = []
|
1481
|
-
for line in self._buf.feed(data):
|
1482
|
-
ret.append(self._make_message(check_isinstance(line, bytes))) # type: ignore
|
1483
|
-
return ret
|
1484
|
-
|
1485
|
-
|
1486
|
-
########################################
|
1487
|
-
# ../threadworker.py
|
1488
|
-
|
1489
|
-
|
1490
|
-
class ThreadWorker(abc.ABC):
|
1491
|
-
def __init__(
|
1492
|
-
self,
|
1493
|
-
*,
|
1494
|
-
stop_event: ta.Optional[threading.Event] = None,
|
1495
|
-
) -> None:
|
1496
|
-
super().__init__()
|
1497
|
-
|
1498
|
-
if stop_event is None:
|
1499
|
-
stop_event = threading.Event()
|
1500
|
-
self._stop_event = stop_event
|
1501
|
-
|
1502
|
-
self._thread: ta.Optional[threading.Thread] = None
|
1503
|
-
|
1504
|
-
self._last_heartbeat: ta.Optional[float] = None
|
1505
|
-
|
1506
|
-
#
|
1507
|
-
|
1508
|
-
def should_stop(self) -> bool:
|
1509
|
-
return self._stop_event.is_set()
|
1510
|
-
|
1511
|
-
#
|
1512
|
-
|
1513
|
-
@property
|
1514
|
-
def last_heartbeat(self) -> ta.Optional[float]:
|
1515
|
-
return self._last_heartbeat
|
1516
|
-
|
1517
|
-
def _heartbeat(self) -> bool:
|
1518
|
-
self._last_heartbeat = time.time()
|
1519
|
-
|
1520
|
-
if self.should_stop():
|
1521
|
-
log.info('Stopping: %s', self)
|
1522
|
-
return False
|
1523
|
-
|
1524
|
-
return True
|
1525
|
-
|
1526
|
-
#
|
1527
|
-
|
1528
|
-
def is_alive(self) -> bool:
|
1529
|
-
return (thr := self._thread) is not None and thr.is_alive()
|
1530
|
-
|
1531
|
-
def start(self) -> None:
|
1532
|
-
thr = threading.Thread(target=self._run)
|
1533
|
-
self._thread = thr
|
1534
|
-
thr.start()
|
1535
|
-
|
1536
|
-
@abc.abstractmethod
|
1537
|
-
def _run(self) -> None:
|
1538
|
-
raise NotImplementedError
|
1539
|
-
|
1540
|
-
def stop(self) -> None:
|
1541
|
-
raise NotImplementedError
|
1542
|
-
|
1543
|
-
def cleanup(self) -> None: # noqa
|
1544
|
-
pass
|
1545
|
-
|
1546
|
-
|
1547
1451
|
########################################
|
1548
1452
|
# ../../logs.py
|
1549
1453
|
"""
|
@@ -1604,7 +1508,7 @@ class AwsLogMessagePoster:
|
|
1604
1508
|
- max_items
|
1605
1509
|
- max_bytes - manually build body
|
1606
1510
|
- flush_interval
|
1607
|
-
-
|
1511
|
+
- split sorted chunks if span over 24h
|
1608
1512
|
"""
|
1609
1513
|
|
1610
1514
|
DEFAULT_URL = 'https://logs.{region_name}.amazonaws.com/' # noqa
|
@@ -1677,7 +1581,7 @@ class AwsLogMessagePoster:
|
|
1677
1581
|
message=m.message,
|
1678
1582
|
timestamp=m.ts_ms,
|
1679
1583
|
)
|
1680
|
-
for m in messages
|
1584
|
+
for m in sorted(messages, key=lambda m: m.ts_ms)
|
1681
1585
|
],
|
1682
1586
|
)
|
1683
1587
|
|
@@ -1709,6 +1613,142 @@ class AwsLogMessagePoster:
|
|
1709
1613
|
return [post]
|
1710
1614
|
|
1711
1615
|
|
1616
|
+
########################################
|
1617
|
+
# ../../../../journald/messages.py
|
1618
|
+
|
1619
|
+
|
1620
|
+
@dc.dataclass(frozen=True)
|
1621
|
+
class JournalctlMessage:
|
1622
|
+
raw: bytes
|
1623
|
+
dct: ta.Optional[ta.Mapping[str, ta.Any]] = None
|
1624
|
+
cursor: ta.Optional[str] = None
|
1625
|
+
ts_us: ta.Optional[int] = None # microseconds UTC
|
1626
|
+
|
1627
|
+
|
1628
|
+
class JournalctlMessageBuilder:
|
1629
|
+
def __init__(self) -> None:
|
1630
|
+
super().__init__()
|
1631
|
+
|
1632
|
+
self._buf = DelimitingBuffer(b'\n')
|
1633
|
+
|
1634
|
+
_cursor_field = '__CURSOR'
|
1635
|
+
|
1636
|
+
_timestamp_fields: ta.Sequence[str] = [
|
1637
|
+
'_SOURCE_REALTIME_TIMESTAMP',
|
1638
|
+
'__REALTIME_TIMESTAMP',
|
1639
|
+
]
|
1640
|
+
|
1641
|
+
def _get_message_timestamp(self, dct: ta.Mapping[str, ta.Any]) -> ta.Optional[int]:
|
1642
|
+
for fld in self._timestamp_fields:
|
1643
|
+
if (tsv := dct.get(fld)) is None:
|
1644
|
+
continue
|
1645
|
+
|
1646
|
+
if isinstance(tsv, str):
|
1647
|
+
try:
|
1648
|
+
return int(tsv)
|
1649
|
+
except ValueError:
|
1650
|
+
try:
|
1651
|
+
return int(float(tsv))
|
1652
|
+
except ValueError:
|
1653
|
+
log.exception('Failed to parse timestamp: %r', tsv)
|
1654
|
+
|
1655
|
+
elif isinstance(tsv, (int, float)):
|
1656
|
+
return int(tsv)
|
1657
|
+
|
1658
|
+
log.error('Invalid timestamp: %r', dct)
|
1659
|
+
return None
|
1660
|
+
|
1661
|
+
def _make_message(self, raw: bytes) -> JournalctlMessage:
|
1662
|
+
dct = None
|
1663
|
+
cursor = None
|
1664
|
+
ts = None
|
1665
|
+
|
1666
|
+
try:
|
1667
|
+
dct = json.loads(raw.decode('utf-8', 'replace'))
|
1668
|
+
except Exception: # noqa
|
1669
|
+
log.exception('Failed to parse raw message: %r', raw)
|
1670
|
+
|
1671
|
+
else:
|
1672
|
+
cursor = dct.get(self._cursor_field)
|
1673
|
+
ts = self._get_message_timestamp(dct)
|
1674
|
+
|
1675
|
+
return JournalctlMessage(
|
1676
|
+
raw=raw,
|
1677
|
+
dct=dct,
|
1678
|
+
cursor=cursor,
|
1679
|
+
ts_us=ts,
|
1680
|
+
)
|
1681
|
+
|
1682
|
+
def feed(self, data: bytes) -> ta.Sequence[JournalctlMessage]:
|
1683
|
+
ret: ta.List[JournalctlMessage] = []
|
1684
|
+
for line in self._buf.feed(data):
|
1685
|
+
ret.append(self._make_message(check_isinstance(line, bytes))) # type: ignore
|
1686
|
+
return ret
|
1687
|
+
|
1688
|
+
|
1689
|
+
########################################
|
1690
|
+
# ../../../../threadworker.py
|
1691
|
+
"""
|
1692
|
+
TODO:
|
1693
|
+
- implement stop lol
|
1694
|
+
- collective heartbeat monitoring - ThreadWorkerGroups
|
1695
|
+
"""
|
1696
|
+
|
1697
|
+
|
1698
|
+
class ThreadWorker(abc.ABC):
|
1699
|
+
def __init__(
|
1700
|
+
self,
|
1701
|
+
*,
|
1702
|
+
stop_event: ta.Optional[threading.Event] = None,
|
1703
|
+
) -> None:
|
1704
|
+
super().__init__()
|
1705
|
+
|
1706
|
+
if stop_event is None:
|
1707
|
+
stop_event = threading.Event()
|
1708
|
+
self._stop_event = stop_event
|
1709
|
+
|
1710
|
+
self._thread: ta.Optional[threading.Thread] = None
|
1711
|
+
|
1712
|
+
self._last_heartbeat: ta.Optional[float] = None
|
1713
|
+
|
1714
|
+
#
|
1715
|
+
|
1716
|
+
def should_stop(self) -> bool:
|
1717
|
+
return self._stop_event.is_set()
|
1718
|
+
|
1719
|
+
#
|
1720
|
+
|
1721
|
+
@property
|
1722
|
+
def last_heartbeat(self) -> ta.Optional[float]:
|
1723
|
+
return self._last_heartbeat
|
1724
|
+
|
1725
|
+
def _heartbeat(self) -> bool:
|
1726
|
+
self._last_heartbeat = time.time()
|
1727
|
+
|
1728
|
+
if self.should_stop():
|
1729
|
+
log.info('Stopping: %s', self)
|
1730
|
+
return False
|
1731
|
+
|
1732
|
+
return True
|
1733
|
+
|
1734
|
+
#
|
1735
|
+
|
1736
|
+
def is_alive(self) -> bool:
|
1737
|
+
return (thr := self._thread) is not None and thr.is_alive()
|
1738
|
+
|
1739
|
+
def start(self) -> None:
|
1740
|
+
thr = threading.Thread(target=self._run)
|
1741
|
+
self._thread = thr
|
1742
|
+
thr.start()
|
1743
|
+
|
1744
|
+
@abc.abstractmethod
|
1745
|
+
def _run(self) -> None:
|
1746
|
+
raise NotImplementedError
|
1747
|
+
|
1748
|
+
def stop(self) -> None:
|
1749
|
+
raise NotImplementedError
|
1750
|
+
|
1751
|
+
|
1712
1752
|
########################################
|
1713
1753
|
# ../../../../../omlish/lite/subprocesses.py
|
1714
1754
|
|
@@ -1816,7 +1856,344 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
|
1816
1856
|
|
1817
1857
|
|
1818
1858
|
########################################
|
1819
|
-
#
|
1859
|
+
# ../../../../journald/tailer.py
|
1860
|
+
"""
|
1861
|
+
TODO:
|
1862
|
+
- https://www.rootusers.com/how-to-change-log-rate-limiting-in-linux/
|
1863
|
+
|
1864
|
+
==
|
1865
|
+
|
1866
|
+
https://www.freedesktop.org/software/systemd/man/latest/journalctl.html
|
1867
|
+
|
1868
|
+
Source Options
|
1869
|
+
--system, --user :: Show messages from system services and the kernel (with --system). Show messages from service of
|
1870
|
+
current user (with --user). If neither is specified, show all messages that the user can see. The
|
1871
|
+
--user option affects how --unit= arguments are treated. See --unit=. Note that --user only works if
|
1872
|
+
persistent logging is enabled, via the Storage= setting in journald.conf(5).
|
1873
|
+
-M, --machine= :: Show messages from a running, local container. Specify a container name to connect to.
|
1874
|
+
-m, --merge :: Show entries interleaved from all available journals, including remote ones.
|
1875
|
+
-D DIR, --directory=DIR :: Takes a directory path as argument. If specified, journalctl will operate on the specified
|
1876
|
+
journal directory DIR instead of the default runtime and system journal paths.
|
1877
|
+
-i GLOB, --file=GLOB :: Takes a file glob as an argument. If specified, journalctl will operate on the specified journal
|
1878
|
+
files matching GLOB instead of the default runtime and system journal paths. May be specified
|
1879
|
+
multiple times, in which case files will be suitably interleaved.
|
1880
|
+
--root=ROOT :: Takes a directory path as an argument. If specified, journalctl will operate on journal directories and
|
1881
|
+
catalog file hierarchy underneath the specified directory instead of the root directory (e.g.
|
1882
|
+
--update-catalog will create ROOT/var/lib/systemd/catalog/database, and journal files under
|
1883
|
+
ROOT/run/journal/ or ROOT/var/log/journal/ will be displayed).
|
1884
|
+
--image=IMAGE :: Takes a path to a disk image file or block device node. If specified, journalctl will operate on the
|
1885
|
+
file system in the indicated disk image. This option is similar to --root=, but operates on file
|
1886
|
+
systems stored in disk images or block devices, thus providing an easy way to extract log data from
|
1887
|
+
disk images. The disk image should either contain just a file system or a set of file systems within a
|
1888
|
+
GPT partition table, following the Discoverable Partitions Specification. For further information on
|
1889
|
+
supported disk images, see systemd-nspawn(1)'s switch of the same name.
|
1890
|
+
--image-policy=policy :: Takes an image policy string as argument, as per systemd.image-policy(7). The policy is
|
1891
|
+
enforced when operating on the disk image specified via --image=, see above. If not specified
|
1892
|
+
defaults to the "*" policy, i.e. all recognized file systems in the image are used.
|
1893
|
+
--namespace=NAMESPACE :: Takes a journal namespace identifier string as argument. If not specified the data collected by
|
1894
|
+
the default namespace is shown. If specified shows the log data of the specified namespace
|
1895
|
+
instead. If the namespace is specified as "*" data from all namespaces is shown, interleaved.
|
1896
|
+
If the namespace identifier is prefixed with "+" data from the specified namespace and the
|
1897
|
+
default namespace is shown, interleaved, but no other. For details about journal namespaces see
|
1898
|
+
systemd-journald.service(8).
|
1899
|
+
|
1900
|
+
Filtering Options
|
1901
|
+
-S, --since=, -U, --until= :: Start showing entries on or newer than the specified date, or on or older than the
|
1902
|
+
specified date, respectively. Date specifications should be of the format
|
1903
|
+
"2012-10-30 18:17:16". If the time part is omitted, "00:00:00" is assumed. If only the
|
1904
|
+
seconds component is omitted, ":00" is assumed. If the date component is omitted, the
|
1905
|
+
current day is assumed. Alternatively the strings "yesterday", "today", "tomorrow" are
|
1906
|
+
understood, which refer to 00:00:00 of the day before the current day, the current day, or
|
1907
|
+
the day after the current day, respectively. "now" refers to the current time. Finally,
|
1908
|
+
relative times may be specified, prefixed with "-" or "+", referring to times before or
|
1909
|
+
after the current time, respectively. For complete time and date specification, see
|
1910
|
+
systemd.time(7). Note that --output=short-full prints timestamps that follow precisely
|
1911
|
+
this format.
|
1912
|
+
-c, --cursor= :: Start showing entries from the location in the journal specified by the passed cursor.
|
1913
|
+
--after-cursor= :: Start showing entries from the location in the journal after the location specified by the passed
|
1914
|
+
cursor. The cursor is shown when the --show-cursor option is used.
|
1915
|
+
--cursor-file=FILE :: If FILE exists and contains a cursor, start showing entries after this location. Otherwise show
|
1916
|
+
entries according to the other given options. At the end, write the cursor of the last entry to
|
1917
|
+
FILE. Use this option to continually read the journal by sequentially calling journalctl.
|
1918
|
+
-b [[ID][±offset]|all], --boot[=[ID][±offset]|all] :: Show messages from a specific boot. This will add a match for
|
1919
|
+
"_BOOT_ID=". The argument may be empty, in which case logs for the
|
1920
|
+
current boot will be shown. If the boot ID is omitted, a positive
|
1921
|
+
offset will look up the boots starting from the beginning of the
|
1922
|
+
journal, and an equal-or-less-than zero offset will look up boots
|
1923
|
+
starting from the end of the journal. Thus, 1 means the first boot
|
1924
|
+
found in the journal in chronological order, 2 the second and so
|
1925
|
+
on; while -0 is the last boot, -1 the boot before last, and so on.
|
1926
|
+
An empty offset is equivalent to specifying -0, except when the
|
1927
|
+
current boot is not the last boot (e.g. because --directory= was
|
1928
|
+
specified to look at logs from a different machine). If the
|
1929
|
+
32-character ID is specified, it may optionally be followed by
|
1930
|
+
offset which identifies the boot relative to the one given by boot
|
1931
|
+
ID. Negative values mean earlier boots and positive values mean
|
1932
|
+
later boots. If offset is not specified, a value of zero is
|
1933
|
+
assumed, and the logs for the boot given by ID are shown. The
|
1934
|
+
special argument all can be used to negate the effect of an
|
1935
|
+
earlier use of -b.
|
1936
|
+
-u, --unit=UNIT|PATTERN :: Show messages for the specified systemd unit UNIT (such as a service unit), or for any of the
|
1937
|
+
units matched by PATTERN. If a pattern is specified, a list of unit names found in the
|
1938
|
+
journal is compared with the specified pattern and all that match are used. For each unit
|
1939
|
+
name, a match is added for messages from the unit ("_SYSTEMD_UNIT=UNIT"), along with
|
1940
|
+
additional matches for messages from systemd and messages about coredumps for the specified
|
1941
|
+
unit. A match is also added for "_SYSTEMD_SLICE=UNIT", such that if the provided UNIT is a
|
1942
|
+
systemd.slice(5) unit, all logs of children of the slice will be shown. With --user, all
|
1943
|
+
--unit= arguments will be converted to match user messages as if specified with --user-unit=.
|
1944
|
+
This parameter can be specified multiple times.
|
1945
|
+
--user-unit= :: Show messages for the specified user session unit. This will add a match for messages from the unit
|
1946
|
+
("_SYSTEMD_USER_UNIT=" and "_UID=") and additional matches for messages from session systemd and
|
1947
|
+
messages about coredumps for the specified unit. A match is also added for "_SYSTEMD_USER_SLICE=UNIT",
|
1948
|
+
such that if the provided UNIT is a systemd.slice(5) unit, all logs of children of the unit will be
|
1949
|
+
shown. This parameter can be specified multiple times.
|
1950
|
+
-t, --identifier=SYSLOG_IDENTIFIER :: Show messages for the specified syslog identifier SYSLOG_IDENTIFIER. This
|
1951
|
+
parameter can be specified multiple times.
|
1952
|
+
-T, --exclude-identifier=SYSLOG_IDENTIFIER :: Exclude messages for the specified syslog identifier SYSLOG_IDENTIFIER.
|
1953
|
+
This parameter can be specified multiple times.
|
1954
|
+
-p, --priority= :: Filter output by message priorities or priority ranges. Takes either a single numeric or textual log
|
1955
|
+
level (i.e. between 0/"emerg" and 7/"debug"), or a range of numeric/text log levels in the form
|
1956
|
+
FROM..TO. The log levels are the usual syslog log levels as documented in syslog(3), i.e. "emerg"
|
1957
|
+
(0), "alert" (1), "crit" (2), "err" (3), "warning" (4), "notice" (5), "info" (6), "debug" (7). If a
|
1958
|
+
single log level is specified, all messages with this log level or a lower (hence more important) log
|
1959
|
+
level are shown. If a range is specified, all messages within the range are shown, including both the
|
1960
|
+
start and the end value of the range. This will add "PRIORITY=" matches for the specified priorities.
|
1961
|
+
--facility= :: Filter output by syslog facility. Takes a comma-separated list of numbers or facility names. The names
|
1962
|
+
are the usual syslog facilities as documented in syslog(3). --facility=help may be used to display a list
|
1963
|
+
of known facility names and exit.
|
1964
|
+
-g, --grep= :: Filter output to entries where the MESSAGE= field matches the specified regular expression.
|
1965
|
+
PERL-compatible regular expressions are used, see pcre2pattern(3) for a detailed description of the
|
1966
|
+
syntax. If the pattern is all lowercase, matching is case insensitive. Otherwise, matching is case
|
1967
|
+
sensitive. This can be overridden with the --case-sensitive option, see below. When used with --lines=
|
1968
|
+
(not prefixed with "+"), --reverse is implied.
|
1969
|
+
--case-sensitive[=BOOLEAN] :: Make pattern matching case sensitive or case insensitive.
|
1970
|
+
-k, --dmesg :: Show only kernel messages. This implies -b and adds the match "_TRANSPORT=kernel".
|
1971
|
+
|
1972
|
+
Output Options
|
1973
|
+
-o, --output= :: Controls the formatting of the journal entries that are shown. Takes one of the following options:
|
1974
|
+
short :: is the default and generates an output that is mostly identical to the formatting of classic syslog files,
|
1975
|
+
showing one line per journal entry.
|
1976
|
+
short-full :: is very similar, but shows timestamps in the format the --since= and --until= options accept. Unlike the
|
1977
|
+
timestamp information shown in short output mode this mode includes weekday, year and timezone
|
1978
|
+
information in the output, and is locale-independent.
|
1979
|
+
short-iso :: is very similar, but shows timestamps in the RFC 3339 profile of ISO 8601.
|
1980
|
+
short-iso-precise :: as for short-iso but includes full microsecond precision.
|
1981
|
+
short-precise :: is very similar, but shows classic syslog timestamps with full microsecond precision.
|
1982
|
+
short-monotonic :: is very similar, but shows monotonic timestamps instead of wallclock timestamps.
|
1983
|
+
short-delta :: as for short-monotonic but includes the time difference to the previous entry. Maybe unreliable time
|
1984
|
+
differences are marked by a "*".
|
1985
|
+
short-unix :: is very similar, but shows seconds passed since January 1st 1970 UTC instead of wallclock timestamps
|
1986
|
+
("UNIX time"). The time is shown with microsecond accuracy.
|
1987
|
+
verbose :: shows the full-structured entry items with all fields.
|
1988
|
+
export :: serializes the journal into a binary (but mostly text-based) stream suitable for backups and network
|
1989
|
+
transfer (see Journal Export Format for more information). To import the binary stream back into native
|
1990
|
+
journald format use systemd-journal-remote(8).
|
1991
|
+
json :: formats entries as JSON objects, separated by newline characters (see Journal JSON Format for more
|
1992
|
+
information). Field values are generally encoded as JSON strings, with three exceptions: Fields larger than
|
1993
|
+
4096 bytes are encoded as null values. (This may be turned off by passing --all, but be aware that this may
|
1994
|
+
allocate overly long JSON objects.) Journal entries permit non-unique fields within the same log entry. JSON
|
1995
|
+
does not allow non-unique fields within objects. Due to this, if a non-unique field is encountered a JSON
|
1996
|
+
array is used as field value, listing all field values as elements. Fields containing non-printable or
|
1997
|
+
non-UTF8 bytes are encoded as arrays containing the raw bytes individually formatted as unsigned numbers. Note
|
1998
|
+
that this encoding is reversible (with the exception of the size limit).
|
1999
|
+
json-pretty :: formats entries as JSON data structures, but formats them in multiple lines in order to make them more
|
2000
|
+
readable by humans.
|
2001
|
+
json-sse :: formats entries as JSON data structures, but wraps them in a format suitable for Server-Sent Events.
|
2002
|
+
json-seq :: formats entries as JSON data structures, but prefixes them with an ASCII Record Separator character (0x1E)
|
2003
|
+
and suffixes them with an ASCII Line Feed character (0x0A), in accordance with JavaScript Object Notation
|
2004
|
+
(JSON) Text Sequences ("application/json-seq").
|
2005
|
+
cat :: generates a very terse output, only showing the actual message of each journal entry with no metadata, not even
|
2006
|
+
a timestamp. If combined with the --output-fields= option will output the listed fields for each log record,
|
2007
|
+
instead of the message.
|
2008
|
+
with-unit :: similar to short-full, but prefixes the unit and user unit names instead of the traditional syslog
|
2009
|
+
identifier. Useful when using templated instances, as it will include the arguments in the unit names.
|
2010
|
+
--truncate-newline :: Truncate each log message at the first newline character on output, so that only the first line of
|
2011
|
+
each message is displayed.
|
2012
|
+
--output-fields= :: A comma separated list of the fields which should be included in the output. This has an effect only
|
2013
|
+
for the output modes which would normally show all fields (verbose, export, json, json-pretty,
|
2014
|
+
json-sse and json-seq), as well as on cat. For the former, the "__CURSOR", "__REALTIME_TIMESTAMP",
|
2015
|
+
"__MONOTONIC_TIMESTAMP", and "_BOOT_ID" fields are always printed.
|
2016
|
+
-n, --lines= :: Show the most recent journal events and limit the number of events shown. The argument is a positive
|
2017
|
+
integer or "all" to disable the limit. Additionally, if the number is prefixed with "+", the oldest
|
2018
|
+
journal events are used instead. The default value is 10 if no argument is given. If --follow is used,
|
2019
|
+
this option is implied. When not prefixed with "+" and used with --grep=, --reverse is implied.
|
2020
|
+
-r, --reverse :: Reverse output so that the newest entries are displayed first.
|
2021
|
+
--show-cursor :: The cursor is shown after the last entry after two dashes:
|
2022
|
+
-- cursor: s=0639… :: The format of the cursor is private and subject to change.
|
2023
|
+
--utc :: Express time in Coordinated Universal Time (UTC).
|
2024
|
+
-x, --catalog :: Augment log lines with explanation texts from the message catalog. This will add explanatory help texts
|
2025
|
+
to log messages in the output where this is available. These short help texts will explain the context
|
2026
|
+
of an error or log event, possible solutions, as well as pointers to support forums, developer
|
2027
|
+
documentation, and any other relevant manuals. Note that help texts are not available for all messages,
|
2028
|
+
but only for selected ones. For more information on the message catalog, see Journal Message Catalogs.
|
2029
|
+
Note: when attaching journalctl output to bug reports, please do not use -x.
|
2030
|
+
--no-hostname :: Don't show the hostname field of log messages originating from the local host. This switch has an
|
2031
|
+
effect only on the short family of output modes (see above). Note: this option does not remove
|
2032
|
+
occurrences of the hostname from log entries themselves, so it does not prevent the hostname from being
|
2033
|
+
visible in the logs.
|
2034
|
+
--no-full, --full, -l :: Ellipsize fields when they do not fit in available columns. The default is to show full fields,
|
2035
|
+
allowing them to wrap or be truncated by the pager, if one is used. The old options -l/--full
|
2036
|
+
are not useful anymore, except to undo --no-full.
|
2037
|
+
-a, --all :: Show all fields in full, even if they include unprintable characters or are very long. By default, fields
|
2038
|
+
with unprintable characters are abbreviated as "blob data". (Note that the pager may escape unprintable
|
2039
|
+
characters again.)
|
2040
|
+
-f, --follow :: Show only the most recent journal entries, and continuously print new entries as they are appended to
|
2041
|
+
the journal.
|
2042
|
+
--no-tail :: Show all stored output lines, even in follow mode. Undoes the effect of --lines=.
|
2043
|
+
-q, --quiet :: Suppresses all informational messages (i.e. "-- Journal begins at …", "-- Reboot --"), any warning
|
2044
|
+
messages regarding inaccessible system journals when run as a normal user.
|
2045
|
+
|
2046
|
+
Pager Control Options
|
2047
|
+
--no-pager :: Do not pipe output into a pager.
|
2048
|
+
-e, --pager-end :: Immediately jump to the end of the journal inside the implied pager tool. This implies -n1000 to
|
2049
|
+
guarantee that the pager will not buffer logs of unbounded size. This may be overridden with an
|
2050
|
+
explicit -n with some other numeric value, while -nall will disable this cap. Note that this option
|
2051
|
+
is only supported for the less(1) pager.
|
2052
|
+
|
2053
|
+
Forward Secure Sealing (FSS) Options
|
2054
|
+
--interval= :: Specifies the change interval for the sealing key when generating an FSS key pair with --setup-keys.
|
2055
|
+
Shorter intervals increase CPU consumption but shorten the time range of undetectable journal
|
2056
|
+
alterations. Defaults to 15min.
|
2057
|
+
--verify-key= :: Specifies the FSS verification key to use for the --verify operation.
|
2058
|
+
--force :: When --setup-keys is passed and Forward Secure Sealing (FSS) has already been configured, recreate FSS keys.
|
2059
|
+
|
2060
|
+
Commands
|
2061
|
+
-N, --fields :: Print all field names currently used in all entries of the journal.
|
2062
|
+
-F, --field= :: Print all possible data values the specified field can take in all entries of the journal.
|
2063
|
+
--list-boots :: Show a tabular list of boot numbers (relative to the current boot), their IDs, and the timestamps of the
|
2064
|
+
first and last message pertaining to the boot. When specified with -n/--lines=[+]N option, only the
|
2065
|
+
first (when the number prefixed with "+") or the last (without prefix) N entries will be shown. When
|
2066
|
+
specified with -r/--reverse, the list will be shown in the reverse order.
|
2067
|
+
--disk-usage :: Shows the current disk usage of all journal files. This shows the sum of the disk usage of all archived
|
2068
|
+
and active journal files.
|
2069
|
+
--vacuum-size=, --vacuum-time=, --vacuum-files=
|
2070
|
+
--vacuum-size= :: removes the oldest archived journal files until the disk space they use falls below the specified
|
2071
|
+
size. Accepts the usual "K", "M", "G" and "T" suffixes (to the base of 1024).
|
2072
|
+
--vacuum-time= :: removes archived journal files older than the specified timespan. Accepts the usual "s" (default),
|
2073
|
+
"m", "h", "days", "weeks", "months", and "years" suffixes, see systemd.time(7) for details.
|
2074
|
+
--vacuum-files= :: leaves only the specified number of separate journal files.
|
2075
|
+
Note that running --vacuum-size= has only an indirect effect on the output shown by --disk-usage, as the latter
|
2076
|
+
includes active journal files, while the vacuuming operation only operates on archived journal files. Similarly,
|
2077
|
+
--vacuum-files= might not actually reduce the number of journal files to below the specified number, as it will not
|
2078
|
+
remove active journal files.
|
2079
|
+
--vacuum-size=, --vacuum-time= and --vacuum-files= may be combined in a single invocation to enforce any combination
|
2080
|
+
of a size, a time and a number of files limit on the archived journal files. Specifying any of these three parameters
|
2081
|
+
as zero is equivalent to not enforcing the specific limit, and is thus redundant.
|
2082
|
+
These three switches may also be combined with --rotate into one command. If so, all active files are rotated first,
|
2083
|
+
and the requested vacuuming operation is executed right after. The rotation has the effect that all currently active
|
2084
|
+
files are archived (and potentially new, empty journal files opened as replacement), and hence the vacuuming operation
|
2085
|
+
has the greatest effect as it can take all log data written so far into account.
|
2086
|
+
--verify :: Check the journal file for internal consistency. If the file has been generated with FSS enabled and the FSS
|
2087
|
+
verification key has been specified with --verify-key=, authenticity of the journal file is verified.
|
2088
|
+
--sync :: Asks the journal daemon to write all yet unwritten journal data to the backing file system and synchronize all
|
2089
|
+
journals. This call does not return until the synchronization operation is complete. This command guarantees
|
2090
|
+
that any log messages written before its invocation are safely stored on disk at the time it returns.
|
2091
|
+
--relinquish-var :: Asks the journal daemon for the reverse operation to --flush: if requested the daemon will write
|
2092
|
+
further log data to /run/log/journal/ and stops writing to /var/log/journal/. A subsequent call to
|
2093
|
+
--flush causes the log output to switch back to /var/log/journal/, see above.
|
2094
|
+
--smart-relinquish-var :: Similar to --relinquish-var, but executes no operation if the root file system and
|
2095
|
+
/var/log/journal/ reside on the same mount point. This operation is used during system
|
2096
|
+
shutdown in order to make the journal daemon stop writing data to /var/log/journal/ in case
|
2097
|
+
that directory is located on a mount point that needs to be unmounted.
|
2098
|
+
--flush :: Asks the journal daemon to flush any log data stored in /run/log/journal/ into /var/log/journal/, if
|
2099
|
+
persistent storage is enabled. This call does not return until the operation is complete. Note that this call
|
2100
|
+
is idempotent: the data is only flushed from /run/log/journal/ into /var/log/journal/ once during system
|
2101
|
+
runtime (but see --relinquish-var below), and this command exits cleanly without executing any operation if
|
2102
|
+
this has already happened. This command effectively guarantees that all data is flushed to /var/log/journal/
|
2103
|
+
at the time it returns.
|
2104
|
+
--rotate :: Asks the journal daemon to rotate journal files. This call does not return until the rotation operation is
|
2105
|
+
complete. Journal file rotation has the effect that all currently active journal files are marked as
|
2106
|
+
archived and renamed, so that they are never written to in future. New (empty) journal files are then
|
2107
|
+
created in their place. This operation may be combined with --vacuum-size=, --vacuum-time= and
|
2108
|
+
--vacuum-file= into a single command, see above.
|
2109
|
+
--header :: Instead of showing journal contents, show internal header information of the journal fields accessed. This
|
2110
|
+
option is particularly useful when trying to identify out-of-order journal entries, as happens for example
|
2111
|
+
when the machine is booted with the wrong system time.
|
2112
|
+
--list-catalog [128-bit-ID…] :: List the contents of the message catalog as a table of message IDs, plus their short
|
2113
|
+
description strings. If any 128-bit-IDs are specified, only those entries are shown.
|
2114
|
+
--dump-catalog [128-bit-ID…] :: Show the contents of the message catalog, with entries separated by a line consisting of
|
2115
|
+
two dashes and the ID (the format is the same as .catalog files). If any 128-bit-IDs are
|
2116
|
+
specified, only those entries are shown.
|
2117
|
+
--update-catalog :: Update the message catalog index. This command needs to be executed each time new catalog files are
|
2118
|
+
installed, removed, or updated to rebuild the binary catalog index.
|
2119
|
+
--setup-keys :: Instead of showing journal contents, generate a new key pair for Forward Secure Sealing (FSS). This will
|
2120
|
+
generate a sealing key and a verification key. The sealing key is stored in the journal data directory
|
2121
|
+
and shall remain on the host. The verification key should be stored externally. Refer to the Seal=
|
2122
|
+
option in journald.conf(5) for information on Forward Secure Sealing and for a link to a refereed
|
2123
|
+
scholarly paper detailing the cryptographic theory it is based on.
|
2124
|
+
-h, --help :: Print a short help text and exit.
|
2125
|
+
--version :: Print a short version string and exit.
|
2126
|
+
|
2127
|
+
Environment
|
2128
|
+
$SYSTEMD_LOG_LEVEL :: The maximum log level of emitted messages (messages with a higher log level, i.e. less important
|
2129
|
+
ones, will be suppressed). Takes a comma-separated list of values. A value may be either one of
|
2130
|
+
(in order of decreasing importance) emerg, alert, crit, err, warning, notice, info, debug, or an
|
2131
|
+
integer in the range 0…7. See syslog(3) for more information. Each value may optionally be
|
2132
|
+
prefixed with one of console, syslog, kmsg or journal followed by a colon to set the maximum log
|
2133
|
+
level for that specific log target (e.g. SYSTEMD_LOG_LEVEL=debug,console:info specifies to log at
|
2134
|
+
debug level except when logging to the console which should be at info level). Note that the
|
2135
|
+
global maximum log level takes priority over any per target maximum log levels.
|
2136
|
+
$SYSTEMD_LOG_COLOR :: A boolean. If true, messages written to the tty will be colored according to priority. This
|
2137
|
+
setting is only useful when messages are written directly to the terminal, because journalctl(1)
|
2138
|
+
and other tools that display logs will color messages based on the log level on their own.
|
2139
|
+
$SYSTEMD_LOG_TIME :: A boolean. If true, console log messages will be prefixed with a timestamp. This setting is only
|
2140
|
+
useful when messages are written directly to the terminal or a file, because journalctl(1) and
|
2141
|
+
other tools that display logs will attach timestamps based on the entry metadata on their own.
|
2142
|
+
$SYSTEMD_LOG_LOCATION :: A boolean. If true, messages will be prefixed with a filename and line number in the source
|
2143
|
+
code where the message originates. Note that the log location is often attached as metadata to
|
2144
|
+
journal entries anyway. Including it directly in the message text can nevertheless be
|
2145
|
+
convenient when debugging programs.
|
2146
|
+
$SYSTEMD_LOG_TID :: A boolean. If true, messages will be prefixed with the current numerical thread ID (TID). Note that
|
2147
|
+
the this information is attached as metadata to journal entries anyway. Including it directly in the
|
2148
|
+
message text can nevertheless be convenient when debugging programs.
|
2149
|
+
$SYSTEMD_LOG_TARGET :: The destination for log messages. One of console (log to the attached tty), console-prefixed (log
|
2150
|
+
to the attached tty but with prefixes encoding the log level and "facility", see syslog(3), kmsg
|
2151
|
+
(log to the kernel circular log buffer), journal (log to the journal), journal-or-kmsg (log to
|
2152
|
+
the journal if available, and to kmsg otherwise), auto (determine the appropriate log target
|
2153
|
+
automatically, the default), null (disable log output).
|
2154
|
+
$SYSTEMD_LOG_RATELIMIT_KMSG :: Whether to ratelimit kmsg or not. Takes a boolean. Defaults to "true". If disabled,
|
2155
|
+
systemd will not ratelimit messages written to kmsg.
|
2156
|
+
$SYSTEMD_PAGER :: Pager to use when --no-pager is not given; overrides $PAGER. If neither $SYSTEMD_PAGER nor $PAGER are
|
2157
|
+
set, a set of well-known pager implementations are tried in turn, including less(1) and more(1), until
|
2158
|
+
one is found. If no pager implementation is discovered no pager is invoked. Setting this environment
|
2159
|
+
variable to an empty string or the value "cat" is equivalent to passing --no-pager. Note: if
|
2160
|
+
$SYSTEMD_PAGERSECURE is not set, $SYSTEMD_PAGER (as well as $PAGER) will be silently ignored.
|
2161
|
+
$SYSTEMD_LESS :: Override the options passed to less (by default "FRSXMK"). Users might want to change two options in
|
2162
|
+
particular:
|
2163
|
+
K :: This option instructs the pager to exit immediately when Ctrl+C is pressed. To allow less to handle Ctrl+C itself
|
2164
|
+
to switch back to the pager command prompt, unset this option. If the value of $SYSTEMD_LESS does not include
|
2165
|
+
"K", and the pager that is invoked is less, Ctrl+C will be ignored by the executable, and needs to be handled by
|
2166
|
+
the pager.
|
2167
|
+
X :: This option instructs the pager to not send termcap initialization and deinitialization strings to the terminal.
|
2168
|
+
It is set by default to allow command output to remain visible in the terminal even after the pager exits.
|
2169
|
+
Nevertheless, this prevents some pager functionality from working, in particular paged output cannot be scrolled
|
2170
|
+
with the mouse. Note that setting the regular $LESS environment variable has no effect for less invocations by
|
2171
|
+
systemd tools.
|
2172
|
+
$SYSTEMD_LESSCHARSET :: Override the charset passed to less (by default "utf-8", if the invoking terminal is determined
|
2173
|
+
to be UTF-8 compatible). Note that setting the regular $LESSCHARSET environment variable has no
|
2174
|
+
effect for less invocations by systemd tools.
|
2175
|
+
$SYSTEMD_PAGERSECURE :: Takes a boolean argument. When true, the "secure" mode of the pager is enabled; if false,
|
2176
|
+
disabled. If $SYSTEMD_PAGERSECURE is not set at all, secure mode is enabled if the effective UID
|
2177
|
+
is not the same as the owner of the login session, see geteuid(2) and sd_pid_get_owner_uid(3).
|
2178
|
+
In secure mode, LESSSECURE=1 will be set when invoking the pager, and the pager shall disable
|
2179
|
+
commands that open or create new files or start new subprocesses. When $SYSTEMD_PAGERSECURE is
|
2180
|
+
not set at all, pagers which are not known to implement secure mode will not be used. (Currently
|
2181
|
+
only less(1) implements secure mode.) Note: when commands are invoked with elevated privileges,
|
2182
|
+
for example under sudo(8) or pkexec(1), care must be taken to ensure that unintended interactive
|
2183
|
+
features are not enabled. "Secure" mode for the pager may be enabled automatically as describe
|
2184
|
+
above. Setting SYSTEMD_PAGERSECURE=0 or not removing it from the inherited environment allows
|
2185
|
+
the user to invoke arbitrary commands. Note that if the $SYSTEMD_PAGER or $PAGER variables are
|
2186
|
+
to be honoured, $SYSTEMD_PAGERSECURE must be set too. It might be reasonable to completely
|
2187
|
+
disable the pager using --no-pager instead.
|
2188
|
+
$SYSTEMD_COLORS :: Takes a boolean argument. When true, systemd and related utilities will use colors in their output,
|
2189
|
+
otherwise the output will be monochrome. Additionally, the variable can take one of the following
|
2190
|
+
special values: "16", "256" to restrict the use of colors to the base 16 or 256 ANSI colors,
|
2191
|
+
respectively. This can be specified to override the automatic decision based on $TERM and what the
|
2192
|
+
console is connected to.
|
2193
|
+
$SYSTEMD_URLIFY :: The value must be a boolean. Controls whether clickable links should be generated in the output for
|
2194
|
+
terminal emulators supporting this. This can be specified to override the decision that systemd makes
|
2195
|
+
based on $TERM and other conditions.
|
2196
|
+
"""
|
1820
2197
|
|
1821
2198
|
|
1822
2199
|
class JournalctlTailerWorker(ThreadWorker):
|
@@ -1887,11 +2264,11 @@ class JournalctlTailerWorker(ThreadWorker):
|
|
1887
2264
|
|
1888
2265
|
while True:
|
1889
2266
|
if not self._heartbeat():
|
1890
|
-
|
2267
|
+
return
|
1891
2268
|
|
1892
2269
|
while stdout.readable():
|
1893
2270
|
if not self._heartbeat():
|
1894
|
-
|
2271
|
+
return
|
1895
2272
|
|
1896
2273
|
buf = stdout.read(self._read_size)
|
1897
2274
|
if not buf:
|
@@ -1901,11 +2278,18 @@ class JournalctlTailerWorker(ThreadWorker):
|
|
1901
2278
|
log.debug('Journalctl read buffer: %r', buf)
|
1902
2279
|
msgs = self._mb.feed(buf)
|
1903
2280
|
if msgs:
|
1904
|
-
|
2281
|
+
while True:
|
2282
|
+
try:
|
2283
|
+
self._output.put(msgs, timeout=1.)
|
2284
|
+
except queue.Full:
|
2285
|
+
if not self._heartbeat():
|
2286
|
+
return
|
2287
|
+
else:
|
2288
|
+
break
|
1905
2289
|
|
1906
2290
|
if self._proc.poll() is not None:
|
1907
2291
|
log.critical('Journalctl process terminated')
|
1908
|
-
|
2292
|
+
return
|
1909
2293
|
|
1910
2294
|
log.debug('Journalctl readable')
|
1911
2295
|
time.sleep(self._sleep_s)
|
@@ -2039,15 +2423,18 @@ class JournalctlToAws:
|
|
2039
2423
|
|
2040
2424
|
@cached_nullary
|
2041
2425
|
def _journalctl_tailer_worker(self) -> JournalctlTailerWorker:
|
2042
|
-
ac: ta.Optional[str] =
|
2043
|
-
if ac is None:
|
2044
|
-
ac = self._read_cursor_file()
|
2045
|
-
if ac is not None:
|
2046
|
-
log.info('Starting from cursor %s', ac)
|
2426
|
+
ac: ta.Optional[str] = None
|
2047
2427
|
|
2048
2428
|
if (since := self._config.journalctl_since):
|
2049
2429
|
log.info('Starting since %s', since)
|
2050
2430
|
|
2431
|
+
else:
|
2432
|
+
ac = self._config.journalctl_after_cursor
|
2433
|
+
if ac is None:
|
2434
|
+
ac = self._read_cursor_file()
|
2435
|
+
if ac is not None:
|
2436
|
+
log.info('Starting from cursor %s', ac)
|
2437
|
+
|
2051
2438
|
return JournalctlTailerWorker(
|
2052
2439
|
self._journalctl_message_queue(),
|
2053
2440
|
|
@@ -2063,9 +2450,9 @@ class JournalctlToAws:
|
|
2063
2450
|
def run(self) -> None:
|
2064
2451
|
self._ensure_locked()
|
2065
2452
|
|
2066
|
-
q = self._journalctl_message_queue()
|
2067
|
-
jtw = self._journalctl_tailer_worker()
|
2068
|
-
mp = self._aws_log_message_poster()
|
2453
|
+
q = self._journalctl_message_queue() # type: queue.Queue[ta.Sequence[JournalctlMessage]]
|
2454
|
+
jtw = self._journalctl_tailer_worker() # type: JournalctlTailerWorker
|
2455
|
+
mp = self._aws_log_message_poster() # type: AwsLogMessagePoster
|
2069
2456
|
|
2070
2457
|
jtw.start()
|
2071
2458
|
|
@@ -2075,7 +2462,13 @@ class JournalctlToAws:
|
|
2075
2462
|
log.critical('Journalctl tailer worker died')
|
2076
2463
|
break
|
2077
2464
|
|
2078
|
-
|
2465
|
+
try:
|
2466
|
+
msgs: ta.Sequence[JournalctlMessage] = q.get(timeout=1.)
|
2467
|
+
except queue.Empty:
|
2468
|
+
msgs = []
|
2469
|
+
if not msgs:
|
2470
|
+
continue
|
2471
|
+
|
2079
2472
|
log.debug('%r', msgs)
|
2080
2473
|
|
2081
2474
|
cur_cursor: ta.Optional[str] = None
|
@@ -2088,10 +2481,14 @@ class JournalctlToAws:
|
|
2088
2481
|
log.warning('Empty queue chunk')
|
2089
2482
|
continue
|
2090
2483
|
|
2091
|
-
|
2092
|
-
|
2093
|
-
|
2094
|
-
|
2484
|
+
feed_msgs = []
|
2485
|
+
for m in msgs:
|
2486
|
+
feed_msgs.append(mp.Message(
|
2487
|
+
message=json.dumps(m.dct, sort_keys=True),
|
2488
|
+
ts_ms=int((m.ts_us / 1000.) if m.ts_us is not None else (time.time() * 1000.)),
|
2489
|
+
))
|
2490
|
+
|
2491
|
+
[post] = mp.feed(feed_msgs)
|
2095
2492
|
log.debug('%r', post)
|
2096
2493
|
|
2097
2494
|
if not self._config.dry_run:
|
@@ -2158,9 +2555,13 @@ def _main() -> None:
|
|
2158
2555
|
|
2159
2556
|
#
|
2160
2557
|
|
2161
|
-
for
|
2162
|
-
|
2163
|
-
|
2558
|
+
for ca, pa in [
|
2559
|
+
('journalctl_after_cursor', 'after_cursor'),
|
2560
|
+
('journalctl_since', 'since'),
|
2561
|
+
('dry_run', 'dry_run'),
|
2562
|
+
]:
|
2563
|
+
if (av := getattr(args, pa)):
|
2564
|
+
config = dc.replace(config, **{ca: av})
|
2164
2565
|
|
2165
2566
|
#
|
2166
2567
|
|