ominfra 0.0.0.dev120__py3-none-any.whl → 0.0.0.dev122__py3-none-any.whl

ominfra/clouds/aws/journald2aws/__main__.py

@@ -0,0 +1,4 @@
+if __name__ == '__main__':
+    from .main import _main
+
+    _main()

ominfra/clouds/aws/journald2aws/driver.py

@@ -46,6 +46,7 @@ from omlish.lite.runtime import is_debugger_attached
 
 from ....journald.messages import JournalctlMessage  # noqa
 from ....journald.tailer import JournalctlTailerWorker
+from ....threadworkers import ThreadWorkerGroup
 from ..auth import AwsSigner
 from ..logs import AwsLogMessageBuilder
 from .cursor import JournalctlToAwsCursor
@@ -63,6 +64,7 @@ class JournalctlToAwsDriver(ExitStacked):
         cursor_file: ta.Optional[str] = None
 
         runtime_limit: ta.Optional[float] = None
+        heartbeat_age_limit: ta.Optional[float] = 60.
 
         #
 
@@ -139,6 +141,12 @@ class JournalctlToAwsDriver(ExitStacked):
 
     #
 
+    @cached_nullary
+    def _worker_group(self) -> ThreadWorkerGroup:
+        return ThreadWorkerGroup()
+
+    #
+
     @cached_nullary
     def _journalctl_message_queue(self):  # type: () -> queue.Queue[ta.Sequence[JournalctlMessage]]
         return queue.Queue()
@@ -165,6 +173,8 @@ class JournalctlToAwsDriver(ExitStacked):
 
             cmd=self._config.journalctl_cmd,
             shell_wrap=is_debugger_attached(),
+
+            worker_groups=[self._worker_group()],
         )
 
     #
@@ -178,26 +188,38 @@ class JournalctlToAwsDriver(ExitStacked):
 
             ensure_locked=self._ensure_locked,
             dry_run=self._config.aws_dry_run,
+
+            worker_groups=[self._worker_group()],
         )
 
     #
 
-    def run(self) -> None:
-        pw: JournalctlToAwsPosterWorker = self._aws_poster_worker()
-        tw: JournalctlTailerWorker = self._journalctl_tailer_worker()
+    def _exit_contexts(self) -> None:
+        wg = self._worker_group()
+        wg.stop_all()
+        wg.join_all()
 
-        ws = [pw, tw]
+    def run(self) -> None:
+        self._aws_poster_worker()
+        self._journalctl_tailer_worker()
 
-        for w in ws:
-            w.start()
+        wg = self._worker_group()
+        wg.start_all()
 
         start = time.time()
 
         while True:
-            for w in ws:
-                if not w.is_alive():
-                    log.critical('Worker died: %r', w)
-                    break
+            for w in wg.get_dead():
+                log.critical('Worker died: %r', w)
+                break
+
+            if (al := self._config.heartbeat_age_limit) is not None:
+                hbs = wg.check_heartbeats()
+                log.debug('Worker heartbeats: %r', hbs)
+                for w, age in hbs.items():
+                    if age > al:
+                        log.critical('Worker heartbeat age limit exceeded: %r %f > %f', w, age, al)
+                        break
 
             if (rl := self._config.runtime_limit) is not None and time.time() - start >= rl:
                 log.warning('Runtime limit reached')
@@ -205,6 +227,5 @@ class JournalctlToAwsDriver(ExitStacked):
 
             time.sleep(1.)
 
-        for w in reversed(ws):
-            w.stop()
-            w.join()
+        wg.stop_all()
+        wg.join_all()

ominfra/clouds/aws/journald2aws/main.py

@@ -2,13 +2,12 @@
 # @omlish-amalg ../../../scripts/journald2aws.py
 import argparse
 import dataclasses as dc
-import json
 import os.path
 import sys
 
 from omlish.lite.logs import configure_standard_logging
-from omlish.lite.marshal import unmarshal_obj
 
+from ....configs import read_config_file
 from .driver import JournalctlToAwsDriver
 
 
@@ -37,9 +36,7 @@ def _main() -> None:
 
     config: JournalctlToAwsDriver.Config
     if args.config_file:
-        with open(os.path.expanduser(args.config_file)) as cf:
-            config_dct = json.load(cf)
-        config = unmarshal_obj(config_dct, JournalctlToAwsDriver.Config)
+        config = read_config_file(os.path.expanduser(args.config_file), JournalctlToAwsDriver.Config)
     else:
         config = JournalctlToAwsDriver.Config()
 

ominfra/configs.py

@@ -0,0 +1,70 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import json
+import typing as ta
+
+from omdev.toml.parser import toml_loads
+from omlish.lite.check import check_isinstance
+from omlish.lite.check import check_not_isinstance
+from omlish.lite.marshal import unmarshal_obj
+
+
+T = ta.TypeVar('T')
+
+ConfigMapping = ta.Mapping[str, ta.Any]
+
+
+def read_config_file(
+        path: str,
+        cls: ta.Type[T],
+        *,
+        prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
+) -> T:
+    with open(path) as cf:
+        if path.endswith('.toml'):
+            config_dct = toml_loads(cf.read())
+        else:
+            config_dct = json.loads(cf.read())
+
+    if prepare is not None:
+        config_dct = prepare(config_dct)  # type: ignore
+
+    return unmarshal_obj(config_dct, cls)
+
+
+def build_config_named_children(
+        o: ta.Union[
+            ta.Sequence[ConfigMapping],
+            ta.Mapping[str, ConfigMapping],
+            None,
+        ],
+        *,
+        name_key: str = 'name',
+) -> ta.Optional[ta.Sequence[ConfigMapping]]:
+    if o is None:
+        return None
+
+    lst: ta.List[ConfigMapping] = []
+    if isinstance(o, ta.Mapping):
+        for k, v in o.items():
+            check_isinstance(v, ta.Mapping)
+            if name_key in v:
+                n = v[name_key]
+                if k != n:
+                    raise KeyError(f'Given names do not match: {n} != {k}')
+                lst.append(v)
+            else:
+                lst.append({name_key: k, **v})
+
+    else:
+        check_not_isinstance(o, str)
+        lst.extend(o)
+
+    seen = set()
+    for d in lst:
+        n = d['name']
+        if n in d:
+            raise KeyError(f'Duplicate name: {n}')
+        seen.add(n)
+
+    return lst

ominfra/deploy/_executor.py

@@ -66,6 +66,7 @@ import subprocess
 import sys
 import textwrap
 import threading
+import types
 import typing as ta
 import uuid
 import weakref  # noqa
@@ -129,7 +130,7 @@ class _cached_nullary:  # noqa
         return bound
 
 
-def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
+def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
     return _cached_nullary(fn)
 
 
@@ -255,6 +256,14 @@ def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
     return it
 
 
+def is_new_type(spec: ta.Any) -> bool:
+    if isinstance(ta.NewType, type):
+        return isinstance(spec, ta.NewType)
+    else:
+        # Before https://github.com/python/cpython/commit/c2f33dfc83ab270412bf243fb21f724037effa1a
+        return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1]  # type: ignore  # noqa
+
+
 def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
     seen = set()
     todo = list(reversed(cls.__subclasses__()))

ominfra/deploy/poly/_main.py

@@ -103,7 +103,7 @@ class _cached_nullary:  # noqa
         return bound
 
 
-def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
+def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
     return _cached_nullary(fn)
 
 

ominfra/pyremote/_runcommands.py

@@ -24,6 +24,7 @@ import subprocess
 import sys
 import textwrap
 import threading
+import types
 import typing as ta
 import uuid
 import weakref  # noqa
@@ -212,7 +213,7 @@ class _cached_nullary:  # noqa
         return bound
 
 
-def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
+def cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
     return _cached_nullary(fn)
 
 
@@ -338,6 +339,14 @@ def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
     return it
 
 
+def is_new_type(spec: ta.Any) -> bool:
+    if isinstance(ta.NewType, type):
+        return isinstance(spec, ta.NewType)
+    else:
+        # Before https://github.com/python/cpython/commit/c2f33dfc83ab270412bf243fb21f724037effa1a
+        return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1]  # type: ignore  # noqa
+
+
 def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
     seen = set()
     todo = list(reversed(cls.__subclasses__()))