ominfra 0.0.0.dev90__py3-none-any.whl → 0.0.0.dev92__py3-none-any.whl

ominfra/clouds/aws/journald2aws/main.py

@@ -4,7 +4,7 @@
 """
 TODO:
  - create log group
- - log stats - chunk sizes etc
+ - log stats - chunk sizes, byte count, num calls, etc
 
 ==
 
@@ -53,11 +53,11 @@ from omlish.lite.marshal import unmarshal_obj
 from omlish.lite.pidfile import Pidfile
 from omlish.lite.runtime import is_debugger_attached
 
+from ....journald.messages import JournalctlMessage  # noqa
+from ....journald.tailer import JournalctlTailerWorker
 from ..auth import AwsSigner
 from ..logs import AwsLogMessagePoster
 from ..logs import AwsPutLogEventsResponse
-from .journald.messages import JournalctlMessage  # noqa
-from .journald.tailer import JournalctlTailerWorker
 
 
 @dc.dataclass(frozen=True)
@@ -184,15 +184,18 @@ class JournalctlToAws:
 
     @cached_nullary
     def _journalctl_tailer_worker(self) -> JournalctlTailerWorker:
-        ac: ta.Optional[str] = self._config.journalctl_after_cursor
-        if ac is None:
-            ac = self._read_cursor_file()
-        if ac is not None:
-            log.info('Starting from cursor %s', ac)
+        ac: ta.Optional[str] = None
 
         if (since := self._config.journalctl_since):
             log.info('Starting since %s', since)
 
+        else:
+            ac = self._config.journalctl_after_cursor
+            if ac is None:
+                ac = self._read_cursor_file()
+            if ac is not None:
+                log.info('Starting from cursor %s', ac)
+
         return JournalctlTailerWorker(
             self._journalctl_message_queue(),
 
@@ -208,9 +211,9 @@ class JournalctlToAws:
     def run(self) -> None:
         self._ensure_locked()
 
-        q = self._journalctl_message_queue()
-        jtw = self._journalctl_tailer_worker()
-        mp = self._aws_log_message_poster()
+        q = self._journalctl_message_queue()  # type: queue.Queue[ta.Sequence[JournalctlMessage]]
+        jtw = self._journalctl_tailer_worker()  # type: JournalctlTailerWorker
+        mp = self._aws_log_message_poster()  # type: AwsLogMessagePoster
 
         jtw.start()
 
@@ -220,7 +223,13 @@ class JournalctlToAws:
                 log.critical('Journalctl tailer worker died')
                 break
 
-            msgs: ta.Sequence[JournalctlMessage] = q.get()
+            try:
+                msgs: ta.Sequence[JournalctlMessage] = q.get(timeout=1.)
+            except queue.Empty:
+                msgs = []
+            if not msgs:
+                continue
+
             log.debug('%r', msgs)
 
             cur_cursor: ta.Optional[str] = None
@@ -233,10 +242,14 @@ class JournalctlToAws:
                 log.warning('Empty queue chunk')
                 continue
 
-            [post] = mp.feed([mp.Message(
-                message=json.dumps(m.dct),
-                ts_ms=int(time.time() * 1000.),
-            ) for m in msgs])
+            feed_msgs = []
+            for m in msgs:
+                feed_msgs.append(mp.Message(
+                    message=json.dumps(m.dct, sort_keys=True),
+                    ts_ms=int((m.ts_us / 1000.) if m.ts_us is not None else (time.time() * 1000.)),
+                ))
+
+            [post] = mp.feed(feed_msgs)
             log.debug('%r', post)
 
             if not self._config.dry_run:
@@ -294,7 +307,7 @@ def _main() -> None:
     if not args.real:
         config = dc.replace(config, journalctl_cmd=[
             sys.executable,
-            os.path.join(os.path.dirname(__file__), 'journald', 'genmessages.py'),
+            os.path.join(os.path.dirname(__file__), '..', '..', '..', 'journald', 'genmessages.py'),
             '--sleep-n', '2',
             '--sleep-s', '.5',
             *(['--message', args.message] if args.message else []),
@@ -303,9 +316,13 @@ def _main() -> None:
 
     #
 
-    for a in ['after_cursor', 'since', 'dry_run']:
-        if (pa := getattr(args, a)):
-            config = dc.replace(config, **{a: pa})
+    for ca, pa in [
+        ('journalctl_after_cursor', 'after_cursor'),
+        ('journalctl_since', 'since'),
+        ('dry_run', 'dry_run'),
+    ]:
+        if (av := getattr(args, pa)):
+            config = dc.replace(config, **{ca: av})
 
     #
 

ominfra/clouds/aws/logs.py

@@ -68,7 +68,7 @@ class AwsLogMessagePoster:
      - max_items
      - max_bytes - manually build body
      - flush_interval
-     - !! sort by timestamp
+     - split sorted chunks if span over 24h
     """
 
     DEFAULT_URL = 'https://logs.{region_name}.amazonaws.com/'  # noqa
@@ -141,7 +141,7 @@ class AwsLogMessagePoster:
                     message=m.message,
                     timestamp=m.ts_ms,
                 )
-                for m in messages
+                for m in sorted(messages, key=lambda m: m.ts_ms)
             ],
         )
 

ominfra/{clouds/aws/journald2aws/journald → journald}/messages.py

@@ -1,4 +1,5 @@
 # ruff: noqa: UP006 UP007
+# @omlish-lite
 import dataclasses as dc
 import json
 import typing as ta
@@ -23,7 +24,31 @@ class JournalctlMessageBuilder:
         self._buf = DelimitingBuffer(b'\n')
 
     _cursor_field = '__CURSOR'
-    _timestamp_field = '_SOURCE_REALTIME_TIMESTAMP'
+
+    _timestamp_fields: ta.Sequence[str] = [
+        '_SOURCE_REALTIME_TIMESTAMP',
+        '__REALTIME_TIMESTAMP',
+    ]
+
+    def _get_message_timestamp(self, dct: ta.Mapping[str, ta.Any]) -> ta.Optional[int]:
+        for fld in self._timestamp_fields:
+            if (tsv := dct.get(fld)) is None:
+                continue
+
+            if isinstance(tsv, str):
+                try:
+                    return int(tsv)
+                except ValueError:
+                    try:
+                        return int(float(tsv))
+                    except ValueError:
+                        log.exception('Failed to parse timestamp: %r', tsv)
+
+            elif isinstance(tsv, (int, float)):
+                return int(tsv)
+
+        log.error('Invalid timestamp: %r', dct)
+        return None
 
     def _make_message(self, raw: bytes) -> JournalctlMessage:
         dct = None
@@ -37,20 +62,7 @@ class JournalctlMessageBuilder:
 
         else:
             cursor = dct.get(self._cursor_field)
-
-            if tsv := dct.get(self._timestamp_field):
-                if isinstance(tsv, str):
-                    try:
-                        ts = int(tsv)
-                    except ValueError:
-                        try:
-                            ts = int(float(tsv))
-                        except ValueError:
-                            log.exception('Failed to parse timestamp: %r', tsv)
-                elif isinstance(tsv, (int, float)):
-                    ts = int(tsv)
-                else:
-                    log.exception('Invalid timestamp: %r', tsv)
+            ts = self._get_message_timestamp(dct)
 
         return JournalctlMessage(
             raw=raw,