ominfra 0.0.0.dev102__py3-none-any.whl → 0.0.0.dev103__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/cursor.py +47 -0
- ominfra/clouds/aws/journald2aws/driver.py +210 -0
- ominfra/clouds/aws/journald2aws/main.py +9 -262
- ominfra/clouds/aws/journald2aws/poster.py +89 -0
- ominfra/clouds/aws/logs.py +19 -13
- ominfra/deploy/_executor.py +25 -3
- ominfra/deploy/poly/_main.py +26 -2
- ominfra/journald/genmessages.py +14 -2
- ominfra/journald/tailer.py +39 -33
- ominfra/pyremote/_runcommands.py +25 -3
- ominfra/scripts/journald2aws.py +428 -195
- ominfra/scripts/supervisor.py +1 -1
- ominfra/threadworkers.py +139 -0
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/RECORD +19 -16
- ominfra/threadworker.py +0 -67
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,47 @@
|
|
1
|
+
# ruff: noqa: UP007
|
2
|
+
import os.path
|
3
|
+
import typing as ta
|
4
|
+
|
5
|
+
from omlish.lite.logs import log
|
6
|
+
|
7
|
+
|
8
|
+
class JournalctlToAwsCursor:
|
9
|
+
def __init__(
|
10
|
+
self,
|
11
|
+
cursor_file: ta.Optional[str] = None,
|
12
|
+
*,
|
13
|
+
ensure_locked: ta.Optional[ta.Callable[[], None]] = None,
|
14
|
+
) -> None:
|
15
|
+
super().__init__()
|
16
|
+
self._cursor_file = cursor_file
|
17
|
+
self._ensure_locked = ensure_locked
|
18
|
+
|
19
|
+
#
|
20
|
+
|
21
|
+
def get(self) -> ta.Optional[str]:
|
22
|
+
if self._ensure_locked is not None:
|
23
|
+
self._ensure_locked()
|
24
|
+
|
25
|
+
if not (cf := self._cursor_file):
|
26
|
+
return None
|
27
|
+
cf = os.path.expanduser(cf)
|
28
|
+
|
29
|
+
try:
|
30
|
+
with open(cf) as f:
|
31
|
+
return f.read().strip()
|
32
|
+
except FileNotFoundError:
|
33
|
+
return None
|
34
|
+
|
35
|
+
def set(self, cursor: str) -> None:
|
36
|
+
if self._ensure_locked is not None:
|
37
|
+
self._ensure_locked()
|
38
|
+
|
39
|
+
if not (cf := self._cursor_file):
|
40
|
+
return
|
41
|
+
cf = os.path.expanduser(cf)
|
42
|
+
|
43
|
+
log.info('Writing cursor file %s : %s', cf, cursor)
|
44
|
+
with open(ncf := cf + '.next', 'w') as f:
|
45
|
+
f.write(cursor)
|
46
|
+
|
47
|
+
os.rename(ncf, cf)
|
@@ -0,0 +1,210 @@
|
|
1
|
+
# ruff: noqa: UP007
|
2
|
+
"""
|
3
|
+
TODO:
|
4
|
+
- create log group
|
5
|
+
- log stats - chunk sizes, byte count, num calls, etc
|
6
|
+
|
7
|
+
==
|
8
|
+
|
9
|
+
https://www.freedesktop.org/software/systemd/man/latest/journalctl.html
|
10
|
+
|
11
|
+
journalctl:
|
12
|
+
-o json
|
13
|
+
--show-cursor
|
14
|
+
|
15
|
+
--since "2012-10-30 18:17:16"
|
16
|
+
--until "2012-10-30 18:17:16"
|
17
|
+
|
18
|
+
--after-cursor <cursor>
|
19
|
+
|
20
|
+
==
|
21
|
+
|
22
|
+
https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html
|
23
|
+
|
24
|
+
==
|
25
|
+
|
26
|
+
@dc.dataclass(frozen=True)
|
27
|
+
class Journald2AwsConfig:
|
28
|
+
log_group_name: str
|
29
|
+
log_stream_name: str
|
30
|
+
|
31
|
+
aws_batch_size: int = 1_000
|
32
|
+
aws_flush_interval_s: float = 1.
|
33
|
+
"""
|
34
|
+
import dataclasses as dc
|
35
|
+
import os.path
|
36
|
+
import queue
|
37
|
+
import time
|
38
|
+
import typing as ta
|
39
|
+
|
40
|
+
from omlish.lite.cached import cached_nullary
|
41
|
+
from omlish.lite.check import check_non_empty_str
|
42
|
+
from omlish.lite.contextmanagers import ExitStacked
|
43
|
+
from omlish.lite.logs import log
|
44
|
+
from omlish.lite.pidfile import Pidfile
|
45
|
+
from omlish.lite.runtime import is_debugger_attached
|
46
|
+
|
47
|
+
from ....journald.messages import JournalctlMessage # noqa
|
48
|
+
from ....journald.tailer import JournalctlTailerWorker
|
49
|
+
from ..auth import AwsSigner
|
50
|
+
from ..logs import AwsLogMessageBuilder
|
51
|
+
from .cursor import JournalctlToAwsCursor
|
52
|
+
from .poster import JournalctlToAwsPosterWorker
|
53
|
+
|
54
|
+
|
55
|
+
##
|
56
|
+
|
57
|
+
|
58
|
+
class JournalctlToAwsDriver(ExitStacked):
|
59
|
+
@dc.dataclass(frozen=True)
|
60
|
+
class Config:
|
61
|
+
pid_file: ta.Optional[str] = None
|
62
|
+
|
63
|
+
cursor_file: ta.Optional[str] = None
|
64
|
+
|
65
|
+
runtime_limit: ta.Optional[float] = None
|
66
|
+
|
67
|
+
#
|
68
|
+
|
69
|
+
aws_log_group_name: str = 'omlish'
|
70
|
+
aws_log_stream_name: ta.Optional[str] = None
|
71
|
+
|
72
|
+
aws_access_key_id: ta.Optional[str] = None
|
73
|
+
aws_secret_access_key: ta.Optional[str] = dc.field(default=None, repr=False)
|
74
|
+
|
75
|
+
aws_region_name: str = 'us-west-1'
|
76
|
+
|
77
|
+
aws_dry_run: bool = False
|
78
|
+
|
79
|
+
#
|
80
|
+
|
81
|
+
journalctl_cmd: ta.Optional[ta.Sequence[str]] = None
|
82
|
+
|
83
|
+
journalctl_after_cursor: ta.Optional[str] = None
|
84
|
+
journalctl_since: ta.Optional[str] = None
|
85
|
+
|
86
|
+
def __init__(self, config: Config) -> None:
|
87
|
+
super().__init__()
|
88
|
+
|
89
|
+
self._config = config
|
90
|
+
|
91
|
+
#
|
92
|
+
|
93
|
+
@cached_nullary
|
94
|
+
def _pidfile(self) -> ta.Optional[Pidfile]:
|
95
|
+
if self._config.pid_file is None:
|
96
|
+
return None
|
97
|
+
|
98
|
+
pfp = os.path.expanduser(self._config.pid_file)
|
99
|
+
|
100
|
+
log.info('Opening pidfile %s', pfp)
|
101
|
+
|
102
|
+
pf = self._enter_context(Pidfile(pfp))
|
103
|
+
pf.write()
|
104
|
+
return pf
|
105
|
+
|
106
|
+
def _ensure_locked(self) -> None:
|
107
|
+
if (pf := self._pidfile()) is not None:
|
108
|
+
pf.ensure_locked()
|
109
|
+
|
110
|
+
#
|
111
|
+
|
112
|
+
@cached_nullary
|
113
|
+
def _cursor(self) -> JournalctlToAwsCursor:
|
114
|
+
return JournalctlToAwsCursor(
|
115
|
+
self._config.cursor_file,
|
116
|
+
ensure_locked=self._ensure_locked,
|
117
|
+
)
|
118
|
+
|
119
|
+
#
|
120
|
+
|
121
|
+
@cached_nullary
|
122
|
+
def _aws_credentials(self) -> ta.Optional[AwsSigner.Credentials]:
|
123
|
+
if self._config.aws_access_key_id is None and self._config.aws_secret_access_key is None:
|
124
|
+
return None
|
125
|
+
|
126
|
+
return AwsSigner.Credentials(
|
127
|
+
access_key_id=check_non_empty_str(self._config.aws_access_key_id),
|
128
|
+
secret_access_key=check_non_empty_str(self._config.aws_secret_access_key),
|
129
|
+
)
|
130
|
+
|
131
|
+
@cached_nullary
|
132
|
+
def _aws_log_message_builder(self) -> AwsLogMessageBuilder:
|
133
|
+
return AwsLogMessageBuilder(
|
134
|
+
log_group_name=self._config.aws_log_group_name,
|
135
|
+
log_stream_name=check_non_empty_str(self._config.aws_log_stream_name),
|
136
|
+
region_name=self._config.aws_region_name,
|
137
|
+
credentials=self._aws_credentials(),
|
138
|
+
)
|
139
|
+
|
140
|
+
#
|
141
|
+
|
142
|
+
@cached_nullary
|
143
|
+
def _journalctl_message_queue(self): # type: () -> queue.Queue[ta.Sequence[JournalctlMessage]]
|
144
|
+
return queue.Queue()
|
145
|
+
|
146
|
+
@cached_nullary
|
147
|
+
def _journalctl_tailer_worker(self) -> JournalctlTailerWorker:
|
148
|
+
ac: ta.Optional[str] = None
|
149
|
+
|
150
|
+
if (since := self._config.journalctl_since):
|
151
|
+
log.info('Starting since %s', since)
|
152
|
+
|
153
|
+
else:
|
154
|
+
ac = self._config.journalctl_after_cursor
|
155
|
+
if ac is None:
|
156
|
+
ac = self._cursor().get()
|
157
|
+
if ac is not None:
|
158
|
+
log.info('Starting from cursor %s', ac)
|
159
|
+
|
160
|
+
return JournalctlTailerWorker(
|
161
|
+
self._journalctl_message_queue(),
|
162
|
+
|
163
|
+
since=since,
|
164
|
+
after_cursor=ac,
|
165
|
+
|
166
|
+
cmd=self._config.journalctl_cmd,
|
167
|
+
shell_wrap=is_debugger_attached(),
|
168
|
+
)
|
169
|
+
|
170
|
+
#
|
171
|
+
|
172
|
+
@cached_nullary
|
173
|
+
def _aws_poster_worker(self) -> JournalctlToAwsPosterWorker:
|
174
|
+
return JournalctlToAwsPosterWorker(
|
175
|
+
self._journalctl_message_queue(),
|
176
|
+
self._aws_log_message_builder(),
|
177
|
+
self._cursor(),
|
178
|
+
|
179
|
+
ensure_locked=self._ensure_locked,
|
180
|
+
dry_run=self._config.aws_dry_run,
|
181
|
+
)
|
182
|
+
|
183
|
+
#
|
184
|
+
|
185
|
+
def run(self) -> None:
|
186
|
+
pw: JournalctlToAwsPosterWorker = self._aws_poster_worker()
|
187
|
+
tw: JournalctlTailerWorker = self._journalctl_tailer_worker()
|
188
|
+
|
189
|
+
ws = [pw, tw]
|
190
|
+
|
191
|
+
for w in ws:
|
192
|
+
w.start()
|
193
|
+
|
194
|
+
start = time.time()
|
195
|
+
|
196
|
+
while True:
|
197
|
+
for w in ws:
|
198
|
+
if not w.is_alive():
|
199
|
+
log.critical('Worker died: %r', w)
|
200
|
+
break
|
201
|
+
|
202
|
+
if (rl := self._config.runtime_limit) is not None and time.time() - start >= rl:
|
203
|
+
log.warning('Runtime limit reached')
|
204
|
+
break
|
205
|
+
|
206
|
+
time.sleep(1.)
|
207
|
+
|
208
|
+
for w in reversed(ws):
|
209
|
+
w.stop()
|
210
|
+
w.join()
|
@@ -1,270 +1,15 @@
|
|
1
1
|
#!/usr/bin/env python3
|
2
|
-
# ruff: noqa: UP007
|
3
2
|
# @omlish-amalg ../../../scripts/journald2aws.py
|
4
|
-
"""
|
5
|
-
TODO:
|
6
|
-
- create log group
|
7
|
-
- log stats - chunk sizes, byte count, num calls, etc
|
8
|
-
|
9
|
-
==
|
10
|
-
|
11
|
-
https://www.freedesktop.org/software/systemd/man/latest/journalctl.html
|
12
|
-
|
13
|
-
journalctl:
|
14
|
-
-o json
|
15
|
-
--show-cursor
|
16
|
-
|
17
|
-
--since "2012-10-30 18:17:16"
|
18
|
-
--until "2012-10-30 18:17:16"
|
19
|
-
|
20
|
-
--after-cursor <cursor>
|
21
|
-
|
22
|
-
==
|
23
|
-
|
24
|
-
https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html
|
25
|
-
|
26
|
-
==
|
27
|
-
|
28
|
-
@dc.dataclass(frozen=True)
|
29
|
-
class Journald2AwsConfig:
|
30
|
-
log_group_name: str
|
31
|
-
log_stream_name: str
|
32
|
-
|
33
|
-
aws_batch_size: int = 1_000
|
34
|
-
aws_flush_interval_s: float = 1.
|
35
|
-
"""
|
36
3
|
import argparse
|
37
|
-
import contextlib
|
38
4
|
import dataclasses as dc
|
39
5
|
import json
|
40
6
|
import os.path
|
41
|
-
import queue
|
42
7
|
import sys
|
43
|
-
import time
|
44
|
-
import typing as ta
|
45
|
-
import urllib.request
|
46
8
|
|
47
|
-
from omlish.lite.cached import cached_nullary
|
48
|
-
from omlish.lite.check import check_non_empty_str
|
49
|
-
from omlish.lite.check import check_not_none
|
50
9
|
from omlish.lite.logs import configure_standard_logging
|
51
|
-
from omlish.lite.logs import log
|
52
10
|
from omlish.lite.marshal import unmarshal_obj
|
53
|
-
from omlish.lite.pidfile import Pidfile
|
54
|
-
from omlish.lite.runtime import is_debugger_attached
|
55
|
-
|
56
|
-
from ....journald.messages import JournalctlMessage # noqa
|
57
|
-
from ....journald.tailer import JournalctlTailerWorker
|
58
|
-
from ..auth import AwsSigner
|
59
|
-
from ..logs import AwsLogMessagePoster
|
60
|
-
from ..logs import AwsPutLogEventsResponse
|
61
|
-
|
62
|
-
|
63
|
-
@dc.dataclass(frozen=True)
|
64
|
-
class JournalctlOpts:
|
65
|
-
after_cursor: ta.Optional[str] = None
|
66
|
-
|
67
|
-
since: ta.Optional[str] = None
|
68
|
-
until: ta.Optional[str] = None
|
69
|
-
|
70
|
-
|
71
|
-
class JournalctlToAws:
|
72
|
-
@dc.dataclass(frozen=True)
|
73
|
-
class Config:
|
74
|
-
pid_file: ta.Optional[str] = None
|
75
|
-
|
76
|
-
cursor_file: ta.Optional[str] = None
|
77
|
-
|
78
|
-
#
|
79
|
-
|
80
|
-
aws_log_group_name: str = 'omlish'
|
81
|
-
aws_log_stream_name: ta.Optional[str] = None
|
82
|
-
|
83
|
-
aws_access_key_id: ta.Optional[str] = None
|
84
|
-
aws_secret_access_key: ta.Optional[str] = dc.field(default=None, repr=False)
|
85
|
-
|
86
|
-
aws_region_name: str = 'us-west-1'
|
87
|
-
|
88
|
-
#
|
89
|
-
|
90
|
-
journalctl_cmd: ta.Optional[ta.Sequence[str]] = None
|
91
|
-
|
92
|
-
journalctl_after_cursor: ta.Optional[str] = None
|
93
|
-
journalctl_since: ta.Optional[str] = None
|
94
|
-
|
95
|
-
#
|
96
|
-
|
97
|
-
dry_run: bool = False
|
98
|
-
|
99
|
-
def __init__(self, config: Config) -> None:
|
100
|
-
super().__init__()
|
101
|
-
self._config = config
|
102
|
-
|
103
|
-
#
|
104
|
-
|
105
|
-
_es: contextlib.ExitStack
|
106
|
-
|
107
|
-
def __enter__(self) -> 'JournalctlToAws':
|
108
|
-
self._es = contextlib.ExitStack().__enter__()
|
109
|
-
return self
|
110
|
-
|
111
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
112
|
-
return self._es.__exit__(exc_type, exc_val, exc_tb)
|
113
|
-
|
114
|
-
#
|
115
|
-
|
116
|
-
@cached_nullary
|
117
|
-
def _pidfile(self) -> ta.Optional[Pidfile]:
|
118
|
-
if self._config.pid_file is None:
|
119
|
-
return None
|
120
|
-
|
121
|
-
pfp = os.path.expanduser(self._config.pid_file)
|
122
|
-
|
123
|
-
log.info('Opening pidfile %s', pfp)
|
124
|
-
|
125
|
-
pf = self._es.enter_context(Pidfile(pfp))
|
126
|
-
pf.write()
|
127
|
-
return pf
|
128
|
-
|
129
|
-
def _ensure_locked(self) -> None:
|
130
|
-
if (pf := self._pidfile()) is not None:
|
131
|
-
pf.ensure_locked()
|
132
|
-
|
133
|
-
#
|
134
|
-
|
135
|
-
def _read_cursor_file(self) -> ta.Optional[str]:
|
136
|
-
self._ensure_locked()
|
137
|
-
|
138
|
-
if not (cf := self._config.cursor_file):
|
139
|
-
return None
|
140
|
-
cf = os.path.expanduser(cf)
|
141
|
-
|
142
|
-
try:
|
143
|
-
with open(cf) as f:
|
144
|
-
return f.read().strip()
|
145
|
-
except FileNotFoundError:
|
146
|
-
return None
|
147
|
-
|
148
|
-
def _write_cursor_file(self, cursor: str) -> None:
|
149
|
-
self._ensure_locked()
|
150
|
-
|
151
|
-
if not (cf := self._config.cursor_file):
|
152
|
-
return
|
153
|
-
cf = os.path.expanduser(cf)
|
154
|
-
|
155
|
-
log.info('Writing cursor file %s : %s', cf, cursor)
|
156
|
-
with open(ncf := cf + '.next', 'w') as f:
|
157
|
-
f.write(cursor)
|
158
|
-
|
159
|
-
os.rename(ncf, cf)
|
160
|
-
|
161
|
-
#
|
162
|
-
|
163
|
-
@cached_nullary
|
164
|
-
def _aws_credentials(self) -> AwsSigner.Credentials:
|
165
|
-
return AwsSigner.Credentials(
|
166
|
-
access_key_id=check_non_empty_str(self._config.aws_access_key_id),
|
167
|
-
secret_access_key=check_non_empty_str(self._config.aws_secret_access_key),
|
168
|
-
)
|
169
|
-
|
170
|
-
@cached_nullary
|
171
|
-
def _aws_log_message_poster(self) -> AwsLogMessagePoster:
|
172
|
-
return AwsLogMessagePoster(
|
173
|
-
log_group_name=self._config.aws_log_group_name,
|
174
|
-
log_stream_name=check_non_empty_str(self._config.aws_log_stream_name),
|
175
|
-
region_name=self._config.aws_region_name,
|
176
|
-
credentials=check_not_none(self._aws_credentials()),
|
177
|
-
)
|
178
|
-
|
179
|
-
#
|
180
|
-
|
181
|
-
@cached_nullary
|
182
|
-
def _journalctl_message_queue(self): # type: () -> queue.Queue[ta.Sequence[JournalctlMessage]]
|
183
|
-
return queue.Queue()
|
184
|
-
|
185
|
-
@cached_nullary
|
186
|
-
def _journalctl_tailer_worker(self) -> JournalctlTailerWorker:
|
187
|
-
ac: ta.Optional[str] = None
|
188
|
-
|
189
|
-
if (since := self._config.journalctl_since):
|
190
|
-
log.info('Starting since %s', since)
|
191
|
-
|
192
|
-
else:
|
193
|
-
ac = self._config.journalctl_after_cursor
|
194
|
-
if ac is None:
|
195
|
-
ac = self._read_cursor_file()
|
196
|
-
if ac is not None:
|
197
|
-
log.info('Starting from cursor %s', ac)
|
198
|
-
|
199
|
-
return JournalctlTailerWorker(
|
200
|
-
self._journalctl_message_queue(),
|
201
|
-
|
202
|
-
since=since,
|
203
|
-
after_cursor=ac,
|
204
|
-
|
205
|
-
cmd=self._config.journalctl_cmd,
|
206
|
-
shell_wrap=is_debugger_attached(),
|
207
|
-
)
|
208
|
-
|
209
|
-
#
|
210
|
-
|
211
|
-
def run(self) -> None:
|
212
|
-
self._ensure_locked()
|
213
|
-
|
214
|
-
q = self._journalctl_message_queue() # type: queue.Queue[ta.Sequence[JournalctlMessage]]
|
215
|
-
jtw = self._journalctl_tailer_worker() # type: JournalctlTailerWorker
|
216
|
-
mp = self._aws_log_message_poster() # type: AwsLogMessagePoster
|
217
|
-
|
218
|
-
jtw.start()
|
219
|
-
|
220
|
-
last_cursor: ta.Optional[str] = None # noqa
|
221
|
-
while True:
|
222
|
-
if not jtw.is_alive():
|
223
|
-
log.critical('Journalctl tailer worker died')
|
224
|
-
break
|
225
|
-
|
226
|
-
try:
|
227
|
-
msgs: ta.Sequence[JournalctlMessage] = q.get(timeout=1.)
|
228
|
-
except queue.Empty:
|
229
|
-
msgs = []
|
230
|
-
if not msgs:
|
231
|
-
continue
|
232
|
-
|
233
|
-
log.debug('%r', msgs)
|
234
|
-
|
235
|
-
cur_cursor: ta.Optional[str] = None
|
236
|
-
for m in reversed(msgs):
|
237
|
-
if m.cursor is not None:
|
238
|
-
cur_cursor = m.cursor
|
239
|
-
break
|
240
|
-
|
241
|
-
if not msgs:
|
242
|
-
log.warning('Empty queue chunk')
|
243
|
-
continue
|
244
|
-
|
245
|
-
feed_msgs = []
|
246
|
-
for m in msgs:
|
247
|
-
feed_msgs.append(mp.Message(
|
248
|
-
message=json.dumps(m.dct, sort_keys=True),
|
249
|
-
ts_ms=int((m.ts_us / 1000.) if m.ts_us is not None else (time.time() * 1000.)),
|
250
|
-
))
|
251
|
-
|
252
|
-
[post] = mp.feed(feed_msgs)
|
253
|
-
log.debug('%r', post)
|
254
|
-
|
255
|
-
if not self._config.dry_run:
|
256
|
-
with urllib.request.urlopen(urllib.request.Request( # noqa
|
257
|
-
post.url,
|
258
|
-
method='POST',
|
259
|
-
headers=dict(post.headers),
|
260
|
-
data=post.data,
|
261
|
-
)) as resp:
|
262
|
-
response = AwsPutLogEventsResponse.from_aws(json.loads(resp.read().decode('utf-8')))
|
263
|
-
log.debug('%r', response)
|
264
11
|
|
265
|
-
|
266
|
-
self._write_cursor_file(cur_cursor)
|
267
|
-
last_cursor = cur_cursor # noqa
|
12
|
+
from .driver import JournalctlToAwsDriver
|
268
13
|
|
269
14
|
|
270
15
|
def _main() -> None:
|
@@ -279,6 +24,8 @@ def _main() -> None:
|
|
279
24
|
|
280
25
|
parser.add_argument('--message', nargs='?')
|
281
26
|
parser.add_argument('--real', action='store_true')
|
27
|
+
parser.add_argument('--num-messages', type=int)
|
28
|
+
parser.add_argument('--runtime-limit', type=float)
|
282
29
|
|
283
30
|
args = parser.parse_args()
|
284
31
|
|
@@ -288,13 +35,13 @@ def _main() -> None:
|
|
288
35
|
|
289
36
|
#
|
290
37
|
|
291
|
-
config:
|
38
|
+
config: JournalctlToAwsDriver.Config
|
292
39
|
if args.config_file:
|
293
40
|
with open(os.path.expanduser(args.config_file)) as cf:
|
294
41
|
config_dct = json.load(cf)
|
295
|
-
config = unmarshal_obj(config_dct,
|
42
|
+
config = unmarshal_obj(config_dct, JournalctlToAwsDriver.Config)
|
296
43
|
else:
|
297
|
-
config =
|
44
|
+
config = JournalctlToAwsDriver.Config()
|
298
45
|
|
299
46
|
#
|
300
47
|
|
@@ -311,7 +58,7 @@ def _main() -> None:
|
|
311
58
|
'--sleep-n', '2',
|
312
59
|
'--sleep-s', '.5',
|
313
60
|
*(['--message', args.message] if args.message else []),
|
314
|
-
|
61
|
+
str(args.num_messages or 100_000),
|
315
62
|
])
|
316
63
|
|
317
64
|
#
|
@@ -319,14 +66,14 @@ def _main() -> None:
|
|
319
66
|
for ca, pa in [
|
320
67
|
('journalctl_after_cursor', 'after_cursor'),
|
321
68
|
('journalctl_since', 'since'),
|
322
|
-
('
|
69
|
+
('aws_dry_run', 'dry_run'),
|
323
70
|
]:
|
324
71
|
if (av := getattr(args, pa)):
|
325
72
|
config = dc.replace(config, **{ca: av})
|
326
73
|
|
327
74
|
#
|
328
75
|
|
329
|
-
with
|
76
|
+
with JournalctlToAwsDriver(config) as jta:
|
330
77
|
jta.run()
|
331
78
|
|
332
79
|
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# ruff: noqa: UP007
|
2
|
+
"""
|
3
|
+
TODO:
|
4
|
+
- retries
|
5
|
+
"""
|
6
|
+
import json
|
7
|
+
import queue
|
8
|
+
import time
|
9
|
+
import typing as ta
|
10
|
+
import urllib.request
|
11
|
+
|
12
|
+
from omlish.lite.logs import log
|
13
|
+
|
14
|
+
from ....journald.messages import JournalctlMessage # noqa
|
15
|
+
from ....threadworkers import ThreadWorker
|
16
|
+
from ..logs import AwsLogMessageBuilder
|
17
|
+
from ..logs import AwsPutLogEventsResponse
|
18
|
+
from .cursor import JournalctlToAwsCursor
|
19
|
+
|
20
|
+
|
21
|
+
class JournalctlToAwsPosterWorker(ThreadWorker):
|
22
|
+
def __init__(
|
23
|
+
self,
|
24
|
+
queue, # type: queue.Queue[ta.Sequence[JournalctlMessage]] # noqa
|
25
|
+
builder: AwsLogMessageBuilder,
|
26
|
+
cursor: JournalctlToAwsCursor,
|
27
|
+
*,
|
28
|
+
ensure_locked: ta.Optional[ta.Callable[[], None]] = None,
|
29
|
+
dry_run: bool = False,
|
30
|
+
queue_timeout_s: float = 1.,
|
31
|
+
**kwargs: ta.Any,
|
32
|
+
) -> None:
|
33
|
+
super().__init__(**kwargs)
|
34
|
+
self._queue = queue
|
35
|
+
self._builder = builder
|
36
|
+
self._cursor = cursor
|
37
|
+
self._ensure_locked = ensure_locked
|
38
|
+
self._dry_run = dry_run
|
39
|
+
self._queue_timeout_s = queue_timeout_s
|
40
|
+
#
|
41
|
+
|
42
|
+
def _run(self) -> None:
|
43
|
+
if self._ensure_locked is not None:
|
44
|
+
self._ensure_locked()
|
45
|
+
|
46
|
+
last_cursor: ta.Optional[str] = None # noqa
|
47
|
+
while True:
|
48
|
+
self._heartbeat()
|
49
|
+
|
50
|
+
try:
|
51
|
+
msgs: ta.Sequence[JournalctlMessage] = self._queue.get(timeout=self._queue_timeout_s)
|
52
|
+
except queue.Empty:
|
53
|
+
msgs = []
|
54
|
+
|
55
|
+
if not msgs:
|
56
|
+
log.debug('Empty queue chunk')
|
57
|
+
continue
|
58
|
+
|
59
|
+
log.debug('%r', msgs)
|
60
|
+
|
61
|
+
cur_cursor: ta.Optional[str] = None
|
62
|
+
for m in reversed(msgs):
|
63
|
+
if m.cursor is not None:
|
64
|
+
cur_cursor = m.cursor
|
65
|
+
break
|
66
|
+
|
67
|
+
feed_msgs = []
|
68
|
+
for m in msgs:
|
69
|
+
feed_msgs.append(AwsLogMessageBuilder.Message(
|
70
|
+
message=json.dumps(m.dct, sort_keys=True),
|
71
|
+
ts_ms=int((m.ts_us / 1000.) if m.ts_us is not None else (time.time() * 1000.)),
|
72
|
+
))
|
73
|
+
|
74
|
+
for post in self._builder.feed(feed_msgs):
|
75
|
+
log.debug('%r', post)
|
76
|
+
|
77
|
+
if not self._dry_run:
|
78
|
+
with urllib.request.urlopen(urllib.request.Request( # noqa
|
79
|
+
post.url,
|
80
|
+
method='POST',
|
81
|
+
headers=dict(post.headers),
|
82
|
+
data=post.data,
|
83
|
+
)) as resp:
|
84
|
+
response = AwsPutLogEventsResponse.from_aws(json.loads(resp.read().decode('utf-8')))
|
85
|
+
log.debug('%r', response)
|
86
|
+
|
87
|
+
if cur_cursor is not None:
|
88
|
+
self._cursor.set(cur_cursor)
|
89
|
+
last_cursor = cur_cursor # noqa
|