ominfra 0.0.0.dev102__py3-none-any.whl → 0.0.0.dev103__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,47 @@
1
+ # ruff: noqa: UP007
2
+ import os.path
3
+ import typing as ta
4
+
5
+ from omlish.lite.logs import log
6
+
7
+
8
+ class JournalctlToAwsCursor:
9
+ def __init__(
10
+ self,
11
+ cursor_file: ta.Optional[str] = None,
12
+ *,
13
+ ensure_locked: ta.Optional[ta.Callable[[], None]] = None,
14
+ ) -> None:
15
+ super().__init__()
16
+ self._cursor_file = cursor_file
17
+ self._ensure_locked = ensure_locked
18
+
19
+ #
20
+
21
+ def get(self) -> ta.Optional[str]:
22
+ if self._ensure_locked is not None:
23
+ self._ensure_locked()
24
+
25
+ if not (cf := self._cursor_file):
26
+ return None
27
+ cf = os.path.expanduser(cf)
28
+
29
+ try:
30
+ with open(cf) as f:
31
+ return f.read().strip()
32
+ except FileNotFoundError:
33
+ return None
34
+
35
+ def set(self, cursor: str) -> None:
36
+ if self._ensure_locked is not None:
37
+ self._ensure_locked()
38
+
39
+ if not (cf := self._cursor_file):
40
+ return
41
+ cf = os.path.expanduser(cf)
42
+
43
+ log.info('Writing cursor file %s : %s', cf, cursor)
44
+ with open(ncf := cf + '.next', 'w') as f:
45
+ f.write(cursor)
46
+
47
+ os.rename(ncf, cf)
@@ -0,0 +1,210 @@
1
+ # ruff: noqa: UP007
2
+ """
3
+ TODO:
4
+ - create log group
5
+ - log stats - chunk sizes, byte count, num calls, etc
6
+
7
+ ==
8
+
9
+ https://www.freedesktop.org/software/systemd/man/latest/journalctl.html
10
+
11
+ journalctl:
12
+ -o json
13
+ --show-cursor
14
+
15
+ --since "2012-10-30 18:17:16"
16
+ --until "2012-10-30 18:17:16"
17
+
18
+ --after-cursor <cursor>
19
+
20
+ ==
21
+
22
+ https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html
23
+
24
+ ==
25
+
26
+ @dc.dataclass(frozen=True)
27
+ class Journald2AwsConfig:
28
+ log_group_name: str
29
+ log_stream_name: str
30
+
31
+ aws_batch_size: int = 1_000
32
+ aws_flush_interval_s: float = 1.
33
+ """
34
+ import dataclasses as dc
35
+ import os.path
36
+ import queue
37
+ import time
38
+ import typing as ta
39
+
40
+ from omlish.lite.cached import cached_nullary
41
+ from omlish.lite.check import check_non_empty_str
42
+ from omlish.lite.contextmanagers import ExitStacked
43
+ from omlish.lite.logs import log
44
+ from omlish.lite.pidfile import Pidfile
45
+ from omlish.lite.runtime import is_debugger_attached
46
+
47
+ from ....journald.messages import JournalctlMessage # noqa
48
+ from ....journald.tailer import JournalctlTailerWorker
49
+ from ..auth import AwsSigner
50
+ from ..logs import AwsLogMessageBuilder
51
+ from .cursor import JournalctlToAwsCursor
52
+ from .poster import JournalctlToAwsPosterWorker
53
+
54
+
55
+ ##
56
+
57
+
58
+ class JournalctlToAwsDriver(ExitStacked):
59
+ @dc.dataclass(frozen=True)
60
+ class Config:
61
+ pid_file: ta.Optional[str] = None
62
+
63
+ cursor_file: ta.Optional[str] = None
64
+
65
+ runtime_limit: ta.Optional[float] = None
66
+
67
+ #
68
+
69
+ aws_log_group_name: str = 'omlish'
70
+ aws_log_stream_name: ta.Optional[str] = None
71
+
72
+ aws_access_key_id: ta.Optional[str] = None
73
+ aws_secret_access_key: ta.Optional[str] = dc.field(default=None, repr=False)
74
+
75
+ aws_region_name: str = 'us-west-1'
76
+
77
+ aws_dry_run: bool = False
78
+
79
+ #
80
+
81
+ journalctl_cmd: ta.Optional[ta.Sequence[str]] = None
82
+
83
+ journalctl_after_cursor: ta.Optional[str] = None
84
+ journalctl_since: ta.Optional[str] = None
85
+
86
+ def __init__(self, config: Config) -> None:
87
+ super().__init__()
88
+
89
+ self._config = config
90
+
91
+ #
92
+
93
+ @cached_nullary
94
+ def _pidfile(self) -> ta.Optional[Pidfile]:
95
+ if self._config.pid_file is None:
96
+ return None
97
+
98
+ pfp = os.path.expanduser(self._config.pid_file)
99
+
100
+ log.info('Opening pidfile %s', pfp)
101
+
102
+ pf = self._enter_context(Pidfile(pfp))
103
+ pf.write()
104
+ return pf
105
+
106
+ def _ensure_locked(self) -> None:
107
+ if (pf := self._pidfile()) is not None:
108
+ pf.ensure_locked()
109
+
110
+ #
111
+
112
+ @cached_nullary
113
+ def _cursor(self) -> JournalctlToAwsCursor:
114
+ return JournalctlToAwsCursor(
115
+ self._config.cursor_file,
116
+ ensure_locked=self._ensure_locked,
117
+ )
118
+
119
+ #
120
+
121
+ @cached_nullary
122
+ def _aws_credentials(self) -> ta.Optional[AwsSigner.Credentials]:
123
+ if self._config.aws_access_key_id is None and self._config.aws_secret_access_key is None:
124
+ return None
125
+
126
+ return AwsSigner.Credentials(
127
+ access_key_id=check_non_empty_str(self._config.aws_access_key_id),
128
+ secret_access_key=check_non_empty_str(self._config.aws_secret_access_key),
129
+ )
130
+
131
+ @cached_nullary
132
+ def _aws_log_message_builder(self) -> AwsLogMessageBuilder:
133
+ return AwsLogMessageBuilder(
134
+ log_group_name=self._config.aws_log_group_name,
135
+ log_stream_name=check_non_empty_str(self._config.aws_log_stream_name),
136
+ region_name=self._config.aws_region_name,
137
+ credentials=self._aws_credentials(),
138
+ )
139
+
140
+ #
141
+
142
+ @cached_nullary
143
+ def _journalctl_message_queue(self): # type: () -> queue.Queue[ta.Sequence[JournalctlMessage]]
144
+ return queue.Queue()
145
+
146
+ @cached_nullary
147
+ def _journalctl_tailer_worker(self) -> JournalctlTailerWorker:
148
+ ac: ta.Optional[str] = None
149
+
150
+ if (since := self._config.journalctl_since):
151
+ log.info('Starting since %s', since)
152
+
153
+ else:
154
+ ac = self._config.journalctl_after_cursor
155
+ if ac is None:
156
+ ac = self._cursor().get()
157
+ if ac is not None:
158
+ log.info('Starting from cursor %s', ac)
159
+
160
+ return JournalctlTailerWorker(
161
+ self._journalctl_message_queue(),
162
+
163
+ since=since,
164
+ after_cursor=ac,
165
+
166
+ cmd=self._config.journalctl_cmd,
167
+ shell_wrap=is_debugger_attached(),
168
+ )
169
+
170
+ #
171
+
172
+ @cached_nullary
173
+ def _aws_poster_worker(self) -> JournalctlToAwsPosterWorker:
174
+ return JournalctlToAwsPosterWorker(
175
+ self._journalctl_message_queue(),
176
+ self._aws_log_message_builder(),
177
+ self._cursor(),
178
+
179
+ ensure_locked=self._ensure_locked,
180
+ dry_run=self._config.aws_dry_run,
181
+ )
182
+
183
+ #
184
+
185
+ def run(self) -> None:
186
+ pw: JournalctlToAwsPosterWorker = self._aws_poster_worker()
187
+ tw: JournalctlTailerWorker = self._journalctl_tailer_worker()
188
+
189
+ ws = [pw, tw]
190
+
191
+ for w in ws:
192
+ w.start()
193
+
194
+ start = time.time()
195
+
196
+ while True:
197
+ for w in ws:
198
+ if not w.is_alive():
199
+ log.critical('Worker died: %r', w)
200
+ break
201
+
202
+ if (rl := self._config.runtime_limit) is not None and time.time() - start >= rl:
203
+ log.warning('Runtime limit reached')
204
+ break
205
+
206
+ time.sleep(1.)
207
+
208
+ for w in reversed(ws):
209
+ w.stop()
210
+ w.join()
@@ -1,270 +1,15 @@
1
1
  #!/usr/bin/env python3
2
- # ruff: noqa: UP007
3
2
  # @omlish-amalg ../../../scripts/journald2aws.py
4
- """
5
- TODO:
6
- - create log group
7
- - log stats - chunk sizes, byte count, num calls, etc
8
-
9
- ==
10
-
11
- https://www.freedesktop.org/software/systemd/man/latest/journalctl.html
12
-
13
- journalctl:
14
- -o json
15
- --show-cursor
16
-
17
- --since "2012-10-30 18:17:16"
18
- --until "2012-10-30 18:17:16"
19
-
20
- --after-cursor <cursor>
21
-
22
- ==
23
-
24
- https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html
25
-
26
- ==
27
-
28
- @dc.dataclass(frozen=True)
29
- class Journald2AwsConfig:
30
- log_group_name: str
31
- log_stream_name: str
32
-
33
- aws_batch_size: int = 1_000
34
- aws_flush_interval_s: float = 1.
35
- """
36
3
  import argparse
37
- import contextlib
38
4
  import dataclasses as dc
39
5
  import json
40
6
  import os.path
41
- import queue
42
7
  import sys
43
- import time
44
- import typing as ta
45
- import urllib.request
46
8
 
47
- from omlish.lite.cached import cached_nullary
48
- from omlish.lite.check import check_non_empty_str
49
- from omlish.lite.check import check_not_none
50
9
  from omlish.lite.logs import configure_standard_logging
51
- from omlish.lite.logs import log
52
10
  from omlish.lite.marshal import unmarshal_obj
53
- from omlish.lite.pidfile import Pidfile
54
- from omlish.lite.runtime import is_debugger_attached
55
-
56
- from ....journald.messages import JournalctlMessage # noqa
57
- from ....journald.tailer import JournalctlTailerWorker
58
- from ..auth import AwsSigner
59
- from ..logs import AwsLogMessagePoster
60
- from ..logs import AwsPutLogEventsResponse
61
-
62
-
63
- @dc.dataclass(frozen=True)
64
- class JournalctlOpts:
65
- after_cursor: ta.Optional[str] = None
66
-
67
- since: ta.Optional[str] = None
68
- until: ta.Optional[str] = None
69
-
70
-
71
- class JournalctlToAws:
72
- @dc.dataclass(frozen=True)
73
- class Config:
74
- pid_file: ta.Optional[str] = None
75
-
76
- cursor_file: ta.Optional[str] = None
77
-
78
- #
79
-
80
- aws_log_group_name: str = 'omlish'
81
- aws_log_stream_name: ta.Optional[str] = None
82
-
83
- aws_access_key_id: ta.Optional[str] = None
84
- aws_secret_access_key: ta.Optional[str] = dc.field(default=None, repr=False)
85
-
86
- aws_region_name: str = 'us-west-1'
87
-
88
- #
89
-
90
- journalctl_cmd: ta.Optional[ta.Sequence[str]] = None
91
-
92
- journalctl_after_cursor: ta.Optional[str] = None
93
- journalctl_since: ta.Optional[str] = None
94
-
95
- #
96
-
97
- dry_run: bool = False
98
-
99
- def __init__(self, config: Config) -> None:
100
- super().__init__()
101
- self._config = config
102
-
103
- #
104
-
105
- _es: contextlib.ExitStack
106
-
107
- def __enter__(self) -> 'JournalctlToAws':
108
- self._es = contextlib.ExitStack().__enter__()
109
- return self
110
-
111
- def __exit__(self, exc_type, exc_val, exc_tb):
112
- return self._es.__exit__(exc_type, exc_val, exc_tb)
113
-
114
- #
115
-
116
- @cached_nullary
117
- def _pidfile(self) -> ta.Optional[Pidfile]:
118
- if self._config.pid_file is None:
119
- return None
120
-
121
- pfp = os.path.expanduser(self._config.pid_file)
122
-
123
- log.info('Opening pidfile %s', pfp)
124
-
125
- pf = self._es.enter_context(Pidfile(pfp))
126
- pf.write()
127
- return pf
128
-
129
- def _ensure_locked(self) -> None:
130
- if (pf := self._pidfile()) is not None:
131
- pf.ensure_locked()
132
-
133
- #
134
-
135
- def _read_cursor_file(self) -> ta.Optional[str]:
136
- self._ensure_locked()
137
-
138
- if not (cf := self._config.cursor_file):
139
- return None
140
- cf = os.path.expanduser(cf)
141
-
142
- try:
143
- with open(cf) as f:
144
- return f.read().strip()
145
- except FileNotFoundError:
146
- return None
147
-
148
- def _write_cursor_file(self, cursor: str) -> None:
149
- self._ensure_locked()
150
-
151
- if not (cf := self._config.cursor_file):
152
- return
153
- cf = os.path.expanduser(cf)
154
-
155
- log.info('Writing cursor file %s : %s', cf, cursor)
156
- with open(ncf := cf + '.next', 'w') as f:
157
- f.write(cursor)
158
-
159
- os.rename(ncf, cf)
160
-
161
- #
162
-
163
- @cached_nullary
164
- def _aws_credentials(self) -> AwsSigner.Credentials:
165
- return AwsSigner.Credentials(
166
- access_key_id=check_non_empty_str(self._config.aws_access_key_id),
167
- secret_access_key=check_non_empty_str(self._config.aws_secret_access_key),
168
- )
169
-
170
- @cached_nullary
171
- def _aws_log_message_poster(self) -> AwsLogMessagePoster:
172
- return AwsLogMessagePoster(
173
- log_group_name=self._config.aws_log_group_name,
174
- log_stream_name=check_non_empty_str(self._config.aws_log_stream_name),
175
- region_name=self._config.aws_region_name,
176
- credentials=check_not_none(self._aws_credentials()),
177
- )
178
-
179
- #
180
-
181
- @cached_nullary
182
- def _journalctl_message_queue(self): # type: () -> queue.Queue[ta.Sequence[JournalctlMessage]]
183
- return queue.Queue()
184
-
185
- @cached_nullary
186
- def _journalctl_tailer_worker(self) -> JournalctlTailerWorker:
187
- ac: ta.Optional[str] = None
188
-
189
- if (since := self._config.journalctl_since):
190
- log.info('Starting since %s', since)
191
-
192
- else:
193
- ac = self._config.journalctl_after_cursor
194
- if ac is None:
195
- ac = self._read_cursor_file()
196
- if ac is not None:
197
- log.info('Starting from cursor %s', ac)
198
-
199
- return JournalctlTailerWorker(
200
- self._journalctl_message_queue(),
201
-
202
- since=since,
203
- after_cursor=ac,
204
-
205
- cmd=self._config.journalctl_cmd,
206
- shell_wrap=is_debugger_attached(),
207
- )
208
-
209
- #
210
-
211
- def run(self) -> None:
212
- self._ensure_locked()
213
-
214
- q = self._journalctl_message_queue() # type: queue.Queue[ta.Sequence[JournalctlMessage]]
215
- jtw = self._journalctl_tailer_worker() # type: JournalctlTailerWorker
216
- mp = self._aws_log_message_poster() # type: AwsLogMessagePoster
217
-
218
- jtw.start()
219
-
220
- last_cursor: ta.Optional[str] = None # noqa
221
- while True:
222
- if not jtw.is_alive():
223
- log.critical('Journalctl tailer worker died')
224
- break
225
-
226
- try:
227
- msgs: ta.Sequence[JournalctlMessage] = q.get(timeout=1.)
228
- except queue.Empty:
229
- msgs = []
230
- if not msgs:
231
- continue
232
-
233
- log.debug('%r', msgs)
234
-
235
- cur_cursor: ta.Optional[str] = None
236
- for m in reversed(msgs):
237
- if m.cursor is not None:
238
- cur_cursor = m.cursor
239
- break
240
-
241
- if not msgs:
242
- log.warning('Empty queue chunk')
243
- continue
244
-
245
- feed_msgs = []
246
- for m in msgs:
247
- feed_msgs.append(mp.Message(
248
- message=json.dumps(m.dct, sort_keys=True),
249
- ts_ms=int((m.ts_us / 1000.) if m.ts_us is not None else (time.time() * 1000.)),
250
- ))
251
-
252
- [post] = mp.feed(feed_msgs)
253
- log.debug('%r', post)
254
-
255
- if not self._config.dry_run:
256
- with urllib.request.urlopen(urllib.request.Request( # noqa
257
- post.url,
258
- method='POST',
259
- headers=dict(post.headers),
260
- data=post.data,
261
- )) as resp:
262
- response = AwsPutLogEventsResponse.from_aws(json.loads(resp.read().decode('utf-8')))
263
- log.debug('%r', response)
264
11
 
265
- if cur_cursor is not None:
266
- self._write_cursor_file(cur_cursor)
267
- last_cursor = cur_cursor # noqa
12
+ from .driver import JournalctlToAwsDriver
268
13
 
269
14
 
270
15
  def _main() -> None:
@@ -279,6 +24,8 @@ def _main() -> None:
279
24
 
280
25
  parser.add_argument('--message', nargs='?')
281
26
  parser.add_argument('--real', action='store_true')
27
+ parser.add_argument('--num-messages', type=int)
28
+ parser.add_argument('--runtime-limit', type=float)
282
29
 
283
30
  args = parser.parse_args()
284
31
 
@@ -288,13 +35,13 @@ def _main() -> None:
288
35
 
289
36
  #
290
37
 
291
- config: JournalctlToAws.Config
38
+ config: JournalctlToAwsDriver.Config
292
39
  if args.config_file:
293
40
  with open(os.path.expanduser(args.config_file)) as cf:
294
41
  config_dct = json.load(cf)
295
- config = unmarshal_obj(config_dct, JournalctlToAws.Config)
42
+ config = unmarshal_obj(config_dct, JournalctlToAwsDriver.Config)
296
43
  else:
297
- config = JournalctlToAws.Config()
44
+ config = JournalctlToAwsDriver.Config()
298
45
 
299
46
  #
300
47
 
@@ -311,7 +58,7 @@ def _main() -> None:
311
58
  '--sleep-n', '2',
312
59
  '--sleep-s', '.5',
313
60
  *(['--message', args.message] if args.message else []),
314
- '100000',
61
+ str(args.num_messages or 100_000),
315
62
  ])
316
63
 
317
64
  #
@@ -319,14 +66,14 @@ def _main() -> None:
319
66
  for ca, pa in [
320
67
  ('journalctl_after_cursor', 'after_cursor'),
321
68
  ('journalctl_since', 'since'),
322
- ('dry_run', 'dry_run'),
69
+ ('aws_dry_run', 'dry_run'),
323
70
  ]:
324
71
  if (av := getattr(args, pa)):
325
72
  config = dc.replace(config, **{ca: av})
326
73
 
327
74
  #
328
75
 
329
- with JournalctlToAws(config) as jta:
76
+ with JournalctlToAwsDriver(config) as jta:
330
77
  jta.run()
331
78
 
332
79
 
@@ -0,0 +1,89 @@
1
+ # ruff: noqa: UP007
2
+ """
3
+ TODO:
4
+ - retries
5
+ """
6
+ import json
7
+ import queue
8
+ import time
9
+ import typing as ta
10
+ import urllib.request
11
+
12
+ from omlish.lite.logs import log
13
+
14
+ from ....journald.messages import JournalctlMessage # noqa
15
+ from ....threadworkers import ThreadWorker
16
+ from ..logs import AwsLogMessageBuilder
17
+ from ..logs import AwsPutLogEventsResponse
18
+ from .cursor import JournalctlToAwsCursor
19
+
20
+
21
+ class JournalctlToAwsPosterWorker(ThreadWorker):
22
+ def __init__(
23
+ self,
24
+ queue, # type: queue.Queue[ta.Sequence[JournalctlMessage]] # noqa
25
+ builder: AwsLogMessageBuilder,
26
+ cursor: JournalctlToAwsCursor,
27
+ *,
28
+ ensure_locked: ta.Optional[ta.Callable[[], None]] = None,
29
+ dry_run: bool = False,
30
+ queue_timeout_s: float = 1.,
31
+ **kwargs: ta.Any,
32
+ ) -> None:
33
+ super().__init__(**kwargs)
34
+ self._queue = queue
35
+ self._builder = builder
36
+ self._cursor = cursor
37
+ self._ensure_locked = ensure_locked
38
+ self._dry_run = dry_run
39
+ self._queue_timeout_s = queue_timeout_s
40
+ #
41
+
42
+ def _run(self) -> None:
43
+ if self._ensure_locked is not None:
44
+ self._ensure_locked()
45
+
46
+ last_cursor: ta.Optional[str] = None # noqa
47
+ while True:
48
+ self._heartbeat()
49
+
50
+ try:
51
+ msgs: ta.Sequence[JournalctlMessage] = self._queue.get(timeout=self._queue_timeout_s)
52
+ except queue.Empty:
53
+ msgs = []
54
+
55
+ if not msgs:
56
+ log.debug('Empty queue chunk')
57
+ continue
58
+
59
+ log.debug('%r', msgs)
60
+
61
+ cur_cursor: ta.Optional[str] = None
62
+ for m in reversed(msgs):
63
+ if m.cursor is not None:
64
+ cur_cursor = m.cursor
65
+ break
66
+
67
+ feed_msgs = []
68
+ for m in msgs:
69
+ feed_msgs.append(AwsLogMessageBuilder.Message(
70
+ message=json.dumps(m.dct, sort_keys=True),
71
+ ts_ms=int((m.ts_us / 1000.) if m.ts_us is not None else (time.time() * 1000.)),
72
+ ))
73
+
74
+ for post in self._builder.feed(feed_msgs):
75
+ log.debug('%r', post)
76
+
77
+ if not self._dry_run:
78
+ with urllib.request.urlopen(urllib.request.Request( # noqa
79
+ post.url,
80
+ method='POST',
81
+ headers=dict(post.headers),
82
+ data=post.data,
83
+ )) as resp:
84
+ response = AwsPutLogEventsResponse.from_aws(json.loads(resp.read().decode('utf-8')))
85
+ log.debug('%r', response)
86
+
87
+ if cur_cursor is not None:
88
+ self._cursor.set(cur_cursor)
89
+ last_cursor = cur_cursor # noqa