ominfra 0.0.0.dev88__py3-none-any.whl → 0.0.0.dev90__py3-none-any.whl

ominfra/scripts/journald2aws.py

@@ -0,0 +1,2172 @@
+#!/usr/bin/env python3
+# noinspection DuplicatedCode
+# @omlish-lite
+# @omlish-script
+# @omlish-amalg-output ../clouds/aws/journald2aws/main.py
+# ruff: noqa: N802 UP006 UP007 UP036
+"""
+TODO:
+ - create log group
+ - log stats - chunk sizes etc
+
+==
+
+https://www.freedesktop.org/software/systemd/man/latest/journalctl.html
+
+journalctl:
+  -o json
+  --show-cursor
+
+  --since "2012-10-30 18:17:16"
+  --until "2012-10-30 18:17:16"
+
+  --after-cursor <cursor>
+
+==
+
+https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html
+
+==
+
+@dc.dataclass(frozen=True)
+class Journald2AwsConfig:
+    log_group_name: str
+    log_stream_name: str
+
+    aws_batch_size: int = 1_000
+    aws_flush_interval_s: float = 1.
+"""
+import abc
+import argparse
+import base64
+import collections.abc
+import contextlib
+import dataclasses as dc
+import datetime
+import decimal
+import enum
+import fcntl
+import fractions
+import functools
+import hashlib
+import hmac
+import inspect
+import io
+import json
+import logging
+import os
+import os.path
+import queue
+import shlex
+import signal
+import subprocess
+import sys
+import threading
+import time
+import typing as ta
+import urllib.parse
+import urllib.request
+import uuid
+import weakref  # noqa
+
+
+########################################
+
+
+if sys.version_info < (3, 8):
+    raise OSError(
+        f'Requires python (3, 8), got {sys.version_info} from {sys.executable}')  # noqa
+
+
+########################################
+
+
+# ../../../../../omlish/lite/check.py
+T = ta.TypeVar('T')
+
+
+########################################
+# ../../../../../omlish/lite/cached.py
+
+
+class cached_nullary:  # noqa
+    def __init__(self, fn):
+        super().__init__()
+        self._fn = fn
+        self._value = self._missing = object()
+        functools.update_wrapper(self, fn)
+
+    def __call__(self, *args, **kwargs):  # noqa
+        if self._value is self._missing:
+            self._value = self._fn()
+        return self._value
+
+    def __get__(self, instance, owner):  # noqa
+        bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
+        return bound
+
+
+########################################
+# ../../../../../omlish/lite/check.py
+
+
+def check_isinstance(v: T, spec: ta.Union[ta.Type[T], tuple]) -> T:
+    if not isinstance(v, spec):
+        raise TypeError(v)
+    return v
+
+
+def check_not_isinstance(v: T, spec: ta.Union[type, tuple]) -> T:
+    if isinstance(v, spec):
+        raise TypeError(v)
+    return v
+
+
+def check_not_none(v: ta.Optional[T]) -> T:
+    if v is None:
+        raise ValueError
+    return v
+
+
+def check_not(v: ta.Any) -> None:
+    if v:
+        raise ValueError(v)
+    return v
+
+
+def check_non_empty_str(v: ta.Optional[str]) -> str:
+    if not v:
+        raise ValueError
+    return v
+
+
+def check_state(v: bool, msg: str = 'Illegal state') -> None:
+    if not v:
+        raise ValueError(msg)
+
+
+def check_equal(l: T, r: T) -> T:
+    if l != r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_not_equal(l: T, r: T) -> T:
+    if l == r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_single(vs: ta.Iterable[T]) -> T:
+    [v] = vs
+    return v
+
+
+########################################
+# ../../../../../omlish/lite/json.py
+
+
+##
+
+
+JSON_PRETTY_INDENT = 2
+
+JSON_PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=JSON_PRETTY_INDENT,
+)
+
+json_dump_pretty: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_PRETTY_KWARGS)  # type: ignore
+json_dumps_pretty: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_PRETTY_KWARGS)
+
+
+##
+
+
+JSON_COMPACT_SEPARATORS = (',', ':')
+
+JSON_COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=None,
+    separators=JSON_COMPACT_SEPARATORS,
+)
+
+json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_COMPACT_KWARGS)  # type: ignore
+json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
+
+
+########################################
+# ../../../../../omlish/lite/pidfile.py
+
+
+class Pidfile:
+    def __init__(self, path: str) -> None:
+        super().__init__()
+        self._path = path
+
+    _f: ta.TextIO
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({self._path!r})'
+
+    def __enter__(self) -> 'Pidfile':
+        fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o600)
+        try:
+            os.set_inheritable(fd, True)
+            f = os.fdopen(fd, 'r+')
+        except Exception:
+            try:
+                os.close(fd)
+            except Exception:  # noqa
+                pass
+            raise
+        self._f = f
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._f is not None:
+            self._f.close()
+            del self._f
+
+    def try_lock(self) -> bool:
+        try:
+            fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            return True
+        except OSError:
+            return False
+
+    def ensure_locked(self) -> None:
+        if not self.try_lock():
+            raise RuntimeError('Could not get lock')
+
+    def write(self, pid: ta.Optional[int] = None) -> None:
+        self.ensure_locked()
+        if pid is None:
+            pid = os.getpid()
+        self._f.write(f'{pid}\n')
+        self._f.flush()
+
+    def clear(self) -> None:
+        self.ensure_locked()
+        self._f.seek(0)
+        self._f.truncate()
+
+    def read(self) -> int:
+        if self.try_lock():
+            raise RuntimeError('Got lock')
+        self._f.seek(0)
+        return int(self._f.read())
+
+    def kill(self, sig: int = signal.SIGTERM) -> None:
+        pid = self.read()
+        os.kill(pid, sig)  # FIXME: Still racy
+
+
+########################################
+# ../../../../../omlish/lite/reflect.py
+
+
+_GENERIC_ALIAS_TYPES = (
+    ta._GenericAlias,  # type: ignore  # noqa
+    *([ta._SpecialGenericAlias] if hasattr(ta, '_SpecialGenericAlias') else []),  # noqa
+)
+
+
+def is_generic_alias(obj, *, origin: ta.Any = None) -> bool:
+    return (
+        isinstance(obj, _GENERIC_ALIAS_TYPES) and
+        (origin is None or ta.get_origin(obj) is origin)
+    )
+
+
+is_union_alias = functools.partial(is_generic_alias, origin=ta.Union)
+is_callable_alias = functools.partial(is_generic_alias, origin=ta.Callable)
+
+
+def is_optional_alias(spec: ta.Any) -> bool:
+    return (
+        isinstance(spec, _GENERIC_ALIAS_TYPES) and  # noqa
+        ta.get_origin(spec) is ta.Union and
+        len(ta.get_args(spec)) == 2 and
+        any(a in (None, type(None)) for a in ta.get_args(spec))
+    )
+
+
+def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
+    [it] = [it for it in ta.get_args(spec) if it not in (None, type(None))]
+    return it
+
+
+def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
+    seen = set()
+    todo = list(reversed(cls.__subclasses__()))
+    while todo:
+        cur = todo.pop()
+        if cur in seen:
+            continue
+        seen.add(cur)
+        yield cur
+        todo.extend(reversed(cur.__subclasses__()))
+
+
+########################################
+# ../../../../../omlish/lite/strings.py
+
+
+def camel_case(name: str, lower: bool = False) -> str:
+    if not name:
+        return ''
+    s = ''.join(map(str.capitalize, name.split('_')))  # noqa
+    if lower:
+        s = s[0].lower() + s[1:]
+    return s
+
+
+def snake_case(name: str) -> str:
+    uppers: list[int | None] = [i for i, c in enumerate(name) if c.isupper()]
+    return '_'.join([name[l:r].lower() for l, r in zip([None, *uppers], [*uppers, None])]).strip('_')
+
+
+def is_dunder(name: str) -> bool:
+    return (
+        name[:2] == name[-2:] == '__' and
+        name[2:3] != '_' and
+        name[-3:-2] != '_' and
+        len(name) > 4
+    )
+
+
+def is_sunder(name: str) -> bool:
+    return (
+        name[0] == name[-1] == '_' and
+        name[1:2] != '_' and
+        name[-2:-1] != '_' and
+        len(name) > 2
+    )
+
+
+def attr_repr(obj: ta.Any, *attrs: str) -> str:
+    return f'{type(obj).__name__}({", ".join(f"{attr}={getattr(obj, attr)!r}" for attr in attrs)})'
+
+
+########################################
+# ../../auth.py
+"""
+https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html
+
+TODO:
+ - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+  - boto / s3transfer upload_fileobj doesn't stream either lol - eagerly calcs Content-MD5
+ - sts tokens
+ - !! fix canonical_qs - sort params
+ - secrets
+"""
+
+
+##
+
+
+class AwsSigner:
+    def __init__(
+            self,
+            creds: 'AwsSigner.Credentials',
+            region_name: str,
+            service_name: str,
+    ) -> None:
+        super().__init__()
+        self._creds = creds
+        self._region_name = region_name
+        self._service_name = service_name
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Credentials:
+        access_key_id: str
+        secret_access_key: str = dc.field(repr=False)
+
+    @dc.dataclass(frozen=True)
+    class Request:
+        method: str
+        url: str
+        headers: ta.Mapping[str, ta.Sequence[str]] = dc.field(default_factory=dict)
+        payload: bytes = b''
+
+    #
+
+    ISO8601 = '%Y%m%dT%H%M%SZ'
+
+    #
+
+    @staticmethod
+    def _host_from_url(url: str) -> str:
+        url_parts = urllib.parse.urlsplit(url)
+        host = check_non_empty_str(url_parts.hostname)
+        default_ports = {
+            'http': 80,
+            'https': 443,
+        }
+        if url_parts.port is not None:
+            if url_parts.port != default_ports.get(url_parts.scheme):
+                host = '%s:%d' % (host, url_parts.port)
+        return host
+
+    @staticmethod
+    def _lower_case_http_map(d: ta.Mapping[str, ta.Sequence[str]]) -> ta.Mapping[str, ta.Sequence[str]]:
+        o: ta.Dict[str, ta.List[str]] = {}
+        for k, vs in d.items():
+            o.setdefault(k.lower(), []).extend(check_not_isinstance(vs, str))
+        return o
+
+    #
+
+    @staticmethod
+    def _as_bytes(data: ta.Union[str, bytes]) -> bytes:
+        return data if isinstance(data, bytes) else data.encode('utf-8')
+
+    @staticmethod
+    def _sha256(data: ta.Union[str, bytes]) -> str:
+        return hashlib.sha256(AwsSigner._as_bytes(data)).hexdigest()
+
+    @staticmethod
+    def _sha256_sign(key: bytes, msg: ta.Union[str, bytes]) -> bytes:
+        return hmac.new(key, AwsSigner._as_bytes(msg), hashlib.sha256).digest()
+
+    @staticmethod
+    def _sha256_sign_hex(key: bytes, msg: ta.Union[str, bytes]) -> str:
+        return hmac.new(key, AwsSigner._as_bytes(msg), hashlib.sha256).hexdigest()
+
+    _EMPTY_SHA256: str
+
+    #
+
+    _SIGNED_HEADERS_BLACKLIST = frozenset([
+        'authorization',
+        'expect',
+        'user-agent',
+        'x-amzn-trace-id',
+    ])
+
+    def _validate_request(self, req: Request) -> None:
+        check_non_empty_str(req.method)
+        check_equal(req.method.upper(), req.method)
+        for k, vs in req.headers.items():
+            check_equal(k.strip(), k)
+            for v in vs:
+                check_equal(v.strip(), v)
+
+
+AwsSigner._EMPTY_SHA256 = AwsSigner._sha256(b'')  # noqa
+
+
+##
+
+
+class V4AwsSigner(AwsSigner):
+    def sign(
+            self,
+            req: AwsSigner.Request,
+            *,
+            sign_payload: bool = False,
+            utcnow: ta.Optional[datetime.datetime] = None,
+    ) -> ta.Mapping[str, ta.Sequence[str]]:
+        self._validate_request(req)
+
+        #
+
+        if utcnow is None:
+            utcnow = datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+        req_dt = utcnow.strftime(self.ISO8601)
+
+        #
+
+        parsed_url = urllib.parse.urlsplit(req.url)
+        canon_uri = parsed_url.path
+        canon_qs = parsed_url.query
+
+        #
+
+        headers_to_sign: ta.Dict[str, ta.List[str]] = {
+            k: list(v)
+            for k, v in self._lower_case_http_map(req.headers).items()
+            if k not in self._SIGNED_HEADERS_BLACKLIST
+        }
+
+        if 'host' not in headers_to_sign:
+            headers_to_sign['host'] = [self._host_from_url(req.url)]
+
+        headers_to_sign['x-amz-date'] = [req_dt]
+
+        hashed_payload = self._sha256(req.payload) if req.payload else self._EMPTY_SHA256
+        if sign_payload:
+            headers_to_sign['x-amz-content-sha256'] = [hashed_payload]
+
+        sorted_header_names = sorted(headers_to_sign)
+        canon_headers = ''.join([
+            ':'.join((k, ','.join(headers_to_sign[k]))) + '\n'
+            for k in sorted_header_names
+        ])
+        signed_headers = ';'.join(sorted_header_names)
+
+        #
+
+        canon_req = '\n'.join([
+            req.method,
+            canon_uri,
+            canon_qs,
+            canon_headers,
+            signed_headers,
+            hashed_payload,
+        ])
+
+        #
+
+        algorithm = 'AWS4-HMAC-SHA256'
+        scope_parts = [
+            req_dt[:8],
+            self._region_name,
+            self._service_name,
+            'aws4_request',
+        ]
+        scope = '/'.join(scope_parts)
+        hashed_canon_req = self._sha256(canon_req)
+        string_to_sign = '\n'.join([
+            algorithm,
+            req_dt,
+            scope,
+            hashed_canon_req,
+        ])
+
+        #
+
+        key = self._creds.secret_access_key
+        key_date = self._sha256_sign(f'AWS4{key}'.encode('utf-8'), req_dt[:8])  # noqa
+        key_region = self._sha256_sign(key_date, self._region_name)
+        key_service = self._sha256_sign(key_region, self._service_name)
+        key_signing = self._sha256_sign(key_service, 'aws4_request')
+        sig = self._sha256_sign_hex(key_signing, string_to_sign)
+
+        #
+
+        cred_scope = '/'.join([
+            self._creds.access_key_id,
+            *scope_parts,
+        ])
+        auth = f'{algorithm} ' + ', '.join([
+            f'Credential={cred_scope}',
+            f'SignedHeaders={signed_headers}',
+            f'Signature={sig}',
+        ])
+
+        #
+
+        out = {
+            'Authorization': [auth],
+            'X-Amz-Date': [req_dt],
+        }
+        if sign_payload:
+            out['X-Amz-Content-SHA256'] = [hashed_payload]
+        return out
+
+
+########################################
+# ../../dataclasses.py
+
+
+class AwsDataclass:
+    class Raw(dict):
+        pass
+
+    #
+
+    _aws_meta: ta.ClassVar[ta.Optional['AwsDataclassMeta']] = None
+
+    @classmethod
+    def _get_aws_meta(cls) -> 'AwsDataclassMeta':
+        try:
+            return cls.__dict__['_aws_meta']
+        except KeyError:
+            pass
+        ret = cls._aws_meta = AwsDataclassMeta(cls)
+        return ret
+
+    #
+
+    def to_aws(self) -> ta.Mapping[str, ta.Any]:
+        return self._get_aws_meta().converters().d2a(self)
+
+    @classmethod
+    def from_aws(cls, v: ta.Mapping[str, ta.Any]) -> 'AwsDataclass':
+        return cls._get_aws_meta().converters().a2d(v)
+
+
+@dc.dataclass(frozen=True)
+class AwsDataclassMeta:
+    cls: ta.Type['AwsDataclass']
+
+    #
+
+    class Field(ta.NamedTuple):
+        d_name: str
+        a_name: str
+        is_opt: bool
+        is_seq: bool
+        dc_cls: ta.Optional[ta.Type['AwsDataclass']]
+
+    @cached_nullary
+    def fields(self) -> ta.Sequence[Field]:
+        fs = []
+        for f in dc.fields(self.cls):  # type: ignore  # noqa
+            d_name = f.name
+            a_name = camel_case(d_name, lower=True)
+
+            is_opt = False
+            is_seq = False
+            dc_cls = None
+
+            c = f.type
+            if c is AwsDataclass.Raw:
+                continue
+
+            if is_optional_alias(c):
+                is_opt = True
+                c = get_optional_alias_arg(c)
+
+            if is_generic_alias(c) and ta.get_origin(c) is collections.abc.Sequence:
+                is_seq = True
+                [c] = ta.get_args(c)
+
+            if is_generic_alias(c):
+                raise TypeError(c)
+
+            if isinstance(c, type) and issubclass(c, AwsDataclass):
+                dc_cls = c
+
+            fs.append(AwsDataclassMeta.Field(
+                d_name=d_name,
+                a_name=a_name,
+                is_opt=is_opt,
+                is_seq=is_seq,
+                dc_cls=dc_cls,
+            ))
+
+        return fs
+
+    #
+
+    class Converters(ta.NamedTuple):
+        d2a: ta.Callable
+        a2d: ta.Callable
+
+    @cached_nullary
+    def converters(self) -> Converters:
+        for df in dc.fields(self.cls):  # type: ignore  # noqa
+            c = df.type
+
+            if is_optional_alias(c):
+                c = get_optional_alias_arg(c)
+
+            if c is AwsDataclass.Raw:
+                rf = df.name
+                break
+
+        else:
+            rf = None
+
+        fs = [
+            (f, f.dc_cls._get_aws_meta().converters() if f.dc_cls is not None else None)  # noqa
+            for f in self.fields()
+        ]
+
+        def d2a(o):
+            dct = {}
+            for f, cs in fs:
+                x = getattr(o, f.d_name)
+                if x is None:
+                    continue
+                if cs is not None:
+                    if f.is_seq:
+                        x = list(map(cs.d2a, x))
+                    else:
+                        x = cs.d2a(x)
+                dct[f.a_name] = x
+            return dct
+
+        def a2d(v):
+            dct = {}
+            for f, cs in fs:
+                x = v.get(f.a_name)
+                if x is None:
+                    continue
+                if cs is not None:
+                    if f.is_seq:
+                        x = list(map(cs.a2d, x))
+                    else:
+                        x = cs.a2d(x)
+                dct[f.d_name] = x
+            if rf is not None:
+                dct[rf] = self.cls.Raw(v)
+            return self.cls(**dct)
+
+        return AwsDataclassMeta.Converters(d2a, a2d)
+
+
+########################################
+# ../../../../../omlish/lite/io.py
+
+
+class DelimitingBuffer:
+    """
+    https://github.com/python-trio/trio/issues/796 :|
+    """
+
+    #
+
+    class Error(Exception):
+        def __init__(self, buffer: 'DelimitingBuffer') -> None:
+            super().__init__(buffer)
+            self.buffer = buffer
+
+        def __repr__(self) -> str:
+            return attr_repr(self, 'buffer')
+
+    class ClosedError(Error):
+        pass
+
+    #
+
+    DEFAULT_DELIMITERS: bytes = b'\n'
+
+    def __init__(
+            self,
+            delimiters: ta.Iterable[int] = DEFAULT_DELIMITERS,
+            *,
+            keep_ends: bool = False,
+            max_size: ta.Optional[int] = None,
+    ) -> None:
+        super().__init__()
+
+        self._delimiters = frozenset(check_isinstance(d, int) for d in delimiters)
+        self._keep_ends = keep_ends
+        self._max_size = max_size
+
+        self._buf: ta.Optional[io.BytesIO] = io.BytesIO()
+
+    #
+
+    @property
+    def is_closed(self) -> bool:
+        return self._buf is None
+
+    def tell(self) -> int:
+        if (buf := self._buf) is None:
+            raise self.ClosedError(self)
+        return buf.tell()
+
+    def peek(self) -> bytes:
+        if (buf := self._buf) is None:
+            raise self.ClosedError(self)
+        return buf.getvalue()
+
+    def _find_delim(self, data: ta.Union[bytes, bytearray], i: int) -> ta.Optional[int]:
+        r = None  # type: int | None
+        for d in self._delimiters:
+            if (p := data.find(d, i)) >= 0:
+                if r is None or p < r:
+                    r = p
+        return r
+
+    def _append_and_reset(self, chunk: bytes) -> bytes:
+        buf = check_not_none(self._buf)
+        if not buf.tell():
+            return chunk
+
+        buf.write(chunk)
+        ret = buf.getvalue()
+        buf.seek(0)
+        buf.truncate()
+        return ret
+
+    class Incomplete(ta.NamedTuple):
+        b: bytes
+
+    def feed(self, data: ta.Union[bytes, bytearray]) -> ta.Generator[ta.Union[bytes, Incomplete], None, None]:
+        if (buf := self._buf) is None:
+            raise self.ClosedError(self)
+
+        if not data:
+            self._buf = None
+
+            if buf.tell():
+                yield self.Incomplete(buf.getvalue())
+
+            return
+
+        l = len(data)
+        i = 0
+        while i < l:
+            if (p := self._find_delim(data, i)) is None:
+                break
+
+            n = p + 1
+            if self._keep_ends:
+                p = n
+
+            yield self._append_and_reset(data[i:p])
+
+            i = n
+
+        if i >= l:
+            return
+
+        if self._max_size is None:
+            buf.write(data[i:])
+            return
+
+        while i < l:
+            remaining_data_len = l - i
+            remaining_buf_capacity = self._max_size - buf.tell()
+
+            if remaining_data_len < remaining_buf_capacity:
+                buf.write(data[i:])
+                return
+
+            p = i + remaining_buf_capacity
+            yield self.Incomplete(self._append_and_reset(data[i:p]))
+            i = p
+
+
+########################################
+# ../../../../../omlish/lite/logs.py
+"""
+TODO:
+ - translate json keys
+ - debug
+"""
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class TidLogFilter(logging.Filter):
+
+    def filter(self, record):
+        record.tid = threading.get_native_id()
+        return True
+
+
+##
+
+
+class JsonLogFormatter(logging.Formatter):
+
+    KEYS: ta.Mapping[str, bool] = {
+        'name': False,
+        'msg': False,
+        'args': False,
+        'levelname': False,
+        'levelno': False,
+        'pathname': False,
+        'filename': False,
+        'module': False,
+        'exc_info': True,
+        'exc_text': True,
+        'stack_info': True,
+        'lineno': False,
+        'funcName': False,
+        'created': False,
+        'msecs': False,
+        'relativeCreated': False,
+        'thread': False,
+        'threadName': False,
+        'processName': False,
+        'process': False,
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        dct = {
+            k: v
+            for k, o in self.KEYS.items()
+            for v in [getattr(record, k)]
+            if not (o and v is None)
+        }
+        return json_dumps_compact(dct)
+
+
+##
+
+
+STANDARD_LOG_FORMAT_PARTS = [
+    ('asctime', '%(asctime)-15s'),
+    ('process', 'pid=%(process)-6s'),
+    ('thread', 'tid=%(thread)x'),
+    ('levelname', '%(levelname)s'),
+    ('name', '%(name)s'),
+    ('separator', '::'),
+    ('message', '%(message)s'),
+]
+
+
+class StandardLogFormatter(logging.Formatter):
+
+    @staticmethod
+    def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
+        return ' '.join(v for k, v in parts)
+
+    converter = datetime.datetime.fromtimestamp  # type: ignore
+
+    def formatTime(self, record, datefmt=None):
+        ct = self.converter(record.created)  # type: ignore
+        if datefmt:
+            return ct.strftime(datefmt)  # noqa
+        else:
+            t = ct.strftime("%Y-%m-%d %H:%M:%S")  # noqa
+            return '%s.%03d' % (t, record.msecs)
+
+
+##
+
+
+class ProxyLogFilterer(logging.Filterer):
+    def __init__(self, underlying: logging.Filterer) -> None:  # noqa
+        self._underlying = underlying
+
+    @property
+    def underlying(self) -> logging.Filterer:
+        return self._underlying
+
+    @property
+    def filters(self):
+        return self._underlying.filters
+
+    @filters.setter
+    def filters(self, filters):
+        self._underlying.filters = filters
+
+    def addFilter(self, filter):  # noqa
+        self._underlying.addFilter(filter)
+
+    def removeFilter(self, filter):  # noqa
+        self._underlying.removeFilter(filter)
+
+    def filter(self, record):
+        return self._underlying.filter(record)
+
+
+class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
+    def __init__(self, underlying: logging.Handler) -> None:  # noqa
+        ProxyLogFilterer.__init__(self, underlying)
+
+    _underlying: logging.Handler
+
+    @property
+    def underlying(self) -> logging.Handler:
+        return self._underlying
+
+    def get_name(self):
+        return self._underlying.get_name()
+
+    def set_name(self, name):
+        self._underlying.set_name(name)
+
+    @property
+    def name(self):
+        return self._underlying.name
+
+    @property
+    def level(self):
+        return self._underlying.level
+
+    @level.setter
+    def level(self, level):
+        self._underlying.level = level
+
+    @property
+    def formatter(self):
+        return self._underlying.formatter
+
+    @formatter.setter
+    def formatter(self, formatter):
+        self._underlying.formatter = formatter
+
+    def createLock(self):
+        self._underlying.createLock()
+
+    def acquire(self):
+        self._underlying.acquire()
+
+    def release(self):
+        self._underlying.release()
+
+    def setLevel(self, level):
+        self._underlying.setLevel(level)
+
+    def format(self, record):
+        return self._underlying.format(record)
+
+    def emit(self, record):
+        self._underlying.emit(record)
+
+    def handle(self, record):
+        return self._underlying.handle(record)
+
+    def setFormatter(self, fmt):
+        self._underlying.setFormatter(fmt)
+
+    def flush(self):
+        self._underlying.flush()
+
+    def close(self):
+        self._underlying.close()
+
+    def handleError(self, record):
+        self._underlying.handleError(record)
+
+
+##
+
+
+class StandardLogHandler(ProxyLogHandler):
+    pass
+
+
+##
+
+
+@contextlib.contextmanager
+def _locking_logging_module_lock() -> ta.Iterator[None]:
+    if hasattr(logging, '_acquireLock'):
+        logging._acquireLock()  # noqa
+        try:
+            yield
+        finally:
+            logging._releaseLock()  # type: ignore  # noqa
+
+    elif hasattr(logging, '_lock'):
+        # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
+        with logging._lock:  # noqa
+            yield
+
+    else:
+        raise Exception("Can't find lock in logging module")
+
+
+def configure_standard_logging(
+        level: ta.Union[int, str] = logging.INFO,
+        *,
+        json: bool = False,
+        target: ta.Optional[logging.Logger] = None,
+        force: bool = False,
+) -> ta.Optional[StandardLogHandler]:
+    with _locking_logging_module_lock():
+        if target is None:
+            target = logging.root
+
+        #
+
+        if not force:
+            if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
+                return None
+
+        #
+
+        handler = logging.StreamHandler()
+
+        #
+
+        formatter: logging.Formatter
+        if json:
+            formatter = JsonLogFormatter()
+        else:
+            formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
+        handler.setFormatter(formatter)
+
+        #
+
+        handler.addFilter(TidLogFilter())
+
+        #
+
+        target.addHandler(handler)
+
+        #
+
+        if level is not None:
+            target.setLevel(level)
+
+        #
+
+        return StandardLogHandler(handler)
+
+
+########################################
+# ../../../../../omlish/lite/marshal.py
+"""
+TODO:
+ - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
+ - nonstrict toggle
+"""
+
+
+##
+
+
+class ObjMarshaler(abc.ABC):
+    @abc.abstractmethod
+    def marshal(self, o: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        raise NotImplementedError
+
+
+class NopObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return o
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return o
+
+
+@dc.dataclass()
+class ProxyObjMarshaler(ObjMarshaler):
+    m: ta.Optional[ObjMarshaler] = None
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return check_not_none(self.m).marshal(o)
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return check_not_none(self.m).unmarshal(o)
+
+
+@dc.dataclass(frozen=True)
+class CastObjMarshaler(ObjMarshaler):
+    ty: type
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return o
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty(o)
+
+
+class DynamicObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return marshal_obj(o)
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return o
+
+
+@dc.dataclass(frozen=True)
+class Base64ObjMarshaler(ObjMarshaler):
+    ty: type
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return base64.b64encode(o).decode('ascii')
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty(base64.b64decode(o))
+
+
+@dc.dataclass(frozen=True)
+class EnumObjMarshaler(ObjMarshaler):
+    ty: type
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return o.name
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty.__members__[o]  # type: ignore
+
+
+@dc.dataclass(frozen=True)
+class OptionalObjMarshaler(ObjMarshaler):
+    item: ObjMarshaler
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        if o is None:
+            return None
+        return self.item.marshal(o)
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        if o is None:
+            return None
+        return self.item.unmarshal(o)
+
+
+@dc.dataclass(frozen=True)
+class MappingObjMarshaler(ObjMarshaler):
+    ty: type
+    km: ObjMarshaler
+    vm: ObjMarshaler
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return {self.km.marshal(k): self.vm.marshal(v) for k, v in o.items()}
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty((self.km.unmarshal(k), self.vm.unmarshal(v)) for k, v in o.items())
+
+
+@dc.dataclass(frozen=True)
+class IterableObjMarshaler(ObjMarshaler):
+    ty: type
+    item: ObjMarshaler
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return [self.item.marshal(e) for e in o]
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty(self.item.unmarshal(e) for e in o)
+
+
+@dc.dataclass(frozen=True)
+class DataclassObjMarshaler(ObjMarshaler):
+    ty: type
+    fs: ta.Mapping[str, ObjMarshaler]
+    nonstrict: bool = False
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if self.nonstrict or k in self.fs})
+
+
+@dc.dataclass(frozen=True)
+class PolymorphicObjMarshaler(ObjMarshaler):
+    class Impl(ta.NamedTuple):
+        ty: type
+        tag: str
+        m: ObjMarshaler
+
+    impls_by_ty: ta.Mapping[type, Impl]
+    impls_by_tag: ta.Mapping[str, Impl]
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        impl = self.impls_by_ty[type(o)]
+        return {impl.tag: impl.m.marshal(o)}
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        [(t, v)] = o.items()
+        impl = self.impls_by_tag[t]
+        return impl.m.unmarshal(v)
+
+
+@dc.dataclass(frozen=True)
+class DatetimeObjMarshaler(ObjMarshaler):
+    ty: type
+
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return o.isoformat()
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return self.ty.fromisoformat(o)  # type: ignore
+
+
+class DecimalObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return str(check_isinstance(o, decimal.Decimal))
+
+    def unmarshal(self, v: ta.Any) -> ta.Any:
+        return decimal.Decimal(check_isinstance(v, str))
+
+
+class FractionObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        fr = check_isinstance(o, fractions.Fraction)
+        return [fr.numerator, fr.denominator]
+
+    def unmarshal(self, v: ta.Any) -> ta.Any:
+        num, denom = check_isinstance(v, list)
+        return fractions.Fraction(num, denom)
+
+
+class UuidObjMarshaler(ObjMarshaler):
+    def marshal(self, o: ta.Any) -> ta.Any:
+        return str(o)
+
+    def unmarshal(self, o: ta.Any) -> ta.Any:
+        return uuid.UUID(o)
+
+
+_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
+    **{t: NopObjMarshaler() for t in (type(None),)},
+    **{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
+    **{t: Base64ObjMarshaler(t) for t in (bytes, bytearray)},
+    **{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
+    **{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
+
+    ta.Any: DynamicObjMarshaler(),
+
+    **{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
+    decimal.Decimal: DecimalObjMarshaler(),
+    fractions.Fraction: FractionObjMarshaler(),
+    uuid.UUID: UuidObjMarshaler(),
+}
+
+_OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
+    **{t: t for t in (dict,)},
+    **{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
+}
+
+_OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
+    **{t: t for t in (list, tuple, set, frozenset)},
+    collections.abc.Set: frozenset,
+    collections.abc.MutableSet: set,
+    collections.abc.Sequence: tuple,
+    collections.abc.MutableSequence: list,
+}
+
+
+def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
+    if ty in _OBJ_MARSHALERS:
+        raise KeyError(ty)
+    _OBJ_MARSHALERS[ty] = m
+
+
+def _make_obj_marshaler(ty: ta.Any) -> ObjMarshaler:
+    if isinstance(ty, type):
+        if abc.ABC in ty.__bases__:
+            impls = [  # type: ignore
+                PolymorphicObjMarshaler.Impl(
+                    ity,
+                    ity.__qualname__,
+                    get_obj_marshaler(ity),
+                )
+                for ity in deep_subclasses(ty)
+                if abc.ABC not in ity.__bases__
+            ]
+            return PolymorphicObjMarshaler(
+                {i.ty: i for i in impls},
+                {i.tag: i for i in impls},
+            )
+
+        if issubclass(ty, enum.Enum):
+            return EnumObjMarshaler(ty)
+
+        if dc.is_dataclass(ty):
+            return DataclassObjMarshaler(
+                ty,
+                {f.name: get_obj_marshaler(f.type) for f in dc.fields(ty)},
+            )
+
+    if is_generic_alias(ty):
+        try:
+            mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
+        except KeyError:
+            pass
+        else:
+            k, v = ta.get_args(ty)
+            return MappingObjMarshaler(mt, get_obj_marshaler(k), get_obj_marshaler(v))
+
+        try:
+            st = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES[ta.get_origin(ty)]
+        except KeyError:
+            pass
+        else:
+            [e] = ta.get_args(ty)
+            return IterableObjMarshaler(st, get_obj_marshaler(e))
+
+        if is_union_alias(ty):
+            return OptionalObjMarshaler(get_obj_marshaler(get_optional_alias_arg(ty)))
+
+    raise TypeError(ty)
+
+
+def get_obj_marshaler(ty: ta.Any) -> ObjMarshaler:
+    try:
+        return _OBJ_MARSHALERS[ty]
+    except KeyError:
+        pass
+
+    p = ProxyObjMarshaler()
+    _OBJ_MARSHALERS[ty] = p
+    try:
+        m = _make_obj_marshaler(ty)
+    except Exception:
+        del _OBJ_MARSHALERS[ty]
+        raise
+    else:
+        p.m = m
+        _OBJ_MARSHALERS[ty] = m
+        return m
+
+
+def marshal_obj(o: ta.Any, ty: ta.Any = None) -> ta.Any:
+    return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
+
+
+def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
+    return get_obj_marshaler(ty).unmarshal(o)
+
+
+########################################
+# ../../../../../omlish/lite/runtime.py
+
+
+@cached_nullary
+def is_debugger_attached() -> bool:
+    return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
+
+
+REQUIRED_PYTHON_VERSION = (3, 8)
+
+
+def check_runtime_version() -> None:
+    if sys.version_info < REQUIRED_PYTHON_VERSION:
+        raise OSError(
+            f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}')  # noqa
+
+
+########################################
+# ../journald/messages.py
+
+
+@dc.dataclass(frozen=True)
+class JournalctlMessage:
+    raw: bytes
+    dct: ta.Optional[ta.Mapping[str, ta.Any]] = None
+    cursor: ta.Optional[str] = None
+    ts_us: ta.Optional[int] = None  # microseconds UTC
+
+
+class JournalctlMessageBuilder:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._buf = DelimitingBuffer(b'\n')
+
+    _cursor_field = '__CURSOR'
+    _timestamp_field = '_SOURCE_REALTIME_TIMESTAMP'
+
+    def _make_message(self, raw: bytes) -> JournalctlMessage:
+        dct = None
+        cursor = None
+        ts = None
+
+        try:
+            dct = json.loads(raw.decode('utf-8', 'replace'))
+        except Exception:  # noqa
+            log.exception('Failed to parse raw message: %r', raw)
+
+        else:
+            cursor = dct.get(self._cursor_field)
+
+            if tsv := dct.get(self._timestamp_field):
+                if isinstance(tsv, str):
+                    try:
+                        ts = int(tsv)
+                    except ValueError:
+                        try:
+                            ts = int(float(tsv))
+                        except ValueError:
+                            log.exception('Failed to parse timestamp: %r', tsv)
+                elif isinstance(tsv, (int, float)):
+                    ts = int(tsv)
+                else:
+                    log.exception('Invalid timestamp: %r', tsv)
+
+        return JournalctlMessage(
+            raw=raw,
+            dct=dct,
+            cursor=cursor,
+            ts_us=ts,
+        )
+
+    def feed(self, data: bytes) -> ta.Sequence[JournalctlMessage]:
+        ret: ta.List[JournalctlMessage] = []
+        for line in self._buf.feed(data):
+            ret.append(self._make_message(check_isinstance(line, bytes)))  # type: ignore
+        return ret
+
+
+########################################
+# ../threadworker.py
+
+
+class ThreadWorker(abc.ABC):
+    def __init__(
+            self,
+            *,
+            stop_event: ta.Optional[threading.Event] = None,
+    ) -> None:
+        super().__init__()
+
+        if stop_event is None:
+            stop_event = threading.Event()
+        self._stop_event = stop_event
+
+        self._thread: ta.Optional[threading.Thread] = None
+
+        self._last_heartbeat: ta.Optional[float] = None
+
+    #
+
+    def should_stop(self) -> bool:
+        return self._stop_event.is_set()
+
+    #
+
+    @property
+    def last_heartbeat(self) -> ta.Optional[float]:
+        return self._last_heartbeat
+
+    def _heartbeat(self) -> bool:
+        self._last_heartbeat = time.time()
+
+        if self.should_stop():
+            log.info('Stopping: %s', self)
+            return False
+
+        return True
+
+    #
+
+    def is_alive(self) -> bool:
+        return (thr := self._thread) is not None and thr.is_alive()
+
+    def start(self) -> None:
+        thr = threading.Thread(target=self._run)
+        self._thread = thr
+        thr.start()
+
+    @abc.abstractmethod
+    def _run(self) -> None:
+        raise NotImplementedError
+
+    def stop(self) -> None:
+        raise NotImplementedError
+
+    def cleanup(self) -> None:  # noqa
+        pass
+
+
+########################################
+# ../../logs.py
+"""
+https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html :
+ - The maximum batch size is 1,048,576 bytes. This size is calculated as the sum of all event messages in UTF-8, plus 26
+   bytes for each log event.
+ - None of the log events in the batch can be more than 2 hours in the future.
+ - None of the log events in the batch can be more than 14 days in the past. Also, none of the log events can be from
+   earlier than the retention period of the log group.
+ - The log events in the batch must be in chronological order by their timestamp. The timestamp is the time that the
+   event occurred, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. (In AWS Tools for PowerShell
+   and the AWS SDK for .NET, the timestamp is specified in .NET format: yyyy-mm-ddThh:mm:ss. For example,
+   2017-09-15T13:45:30.)
+ - A batch of log events in a single request cannot span more than 24 hours. Otherwise, the operation fails.
+ - Each log event can be no larger than 256 KB.
+ - The maximum number of log events in a batch is 10,000.
+"""
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class AwsLogEvent(AwsDataclass):
+    message: str
+    timestamp: int  # milliseconds UTC
+
+
+@dc.dataclass(frozen=True)
+class AwsPutLogEventsRequest(AwsDataclass):
+    log_group_name: str
+    log_stream_name: str
+    log_events: ta.Sequence[AwsLogEvent]
+    sequence_token: ta.Optional[str] = None
+
+
+@dc.dataclass(frozen=True)
+class AwsRejectedLogEventsInfo(AwsDataclass):
+    expired_log_event_end_index: ta.Optional[int] = None
+    too_new_log_event_start_index: ta.Optional[int] = None
+    too_old_log_event_end_index: ta.Optional[int] = None
+
+
+@dc.dataclass(frozen=True)
+class AwsPutLogEventsResponse(AwsDataclass):
+    next_sequence_token: ta.Optional[str] = None
+    rejected_log_events_info: ta.Optional[AwsRejectedLogEventsInfo] = None
+
+    raw: ta.Optional[AwsDataclass.Raw] = None
+
+
+##
+
+
+class AwsLogMessagePoster:
+    """
+    TODO:
+     - max_items
+     - max_bytes - manually build body
+     - flush_interval
+     - !! sort by timestamp
+    """
+
+    DEFAULT_URL = 'https://logs.{region_name}.amazonaws.com/'  # noqa
+
+    DEFAULT_SERVICE_NAME = 'logs'
+
+    DEFAULT_TARGET = 'Logs_20140328.PutLogEvents'
+    DEFAULT_CONTENT_TYPE = 'application/x-amz-json-1.1'
+
+    DEFAULT_HEADERS: ta.Mapping[str, str] = {
+        'X-Amz-Target': DEFAULT_TARGET,
+        'Content-Type': DEFAULT_CONTENT_TYPE,
+    }
+
+    def __init__(
+            self,
+            log_group_name: str,
+            log_stream_name: str,
+            region_name: str,
+            credentials: AwsSigner.Credentials,
+
+            url: ta.Optional[str] = None,
+            service_name: str = DEFAULT_SERVICE_NAME,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            extra_headers: ta.Optional[ta.Mapping[str, str]] = None,
+    ) -> None:
+        super().__init__()
+
+        self._log_group_name = check_non_empty_str(log_group_name)
+        self._log_stream_name = check_non_empty_str(log_stream_name)
+
+        if url is None:
+            url = self.DEFAULT_URL.format(region_name=region_name)
+        self._url = url
+
+        if headers is None:
+            headers = self.DEFAULT_HEADERS
+        if extra_headers is not None:
+            headers = {**headers, **extra_headers}
+        self._headers = {k: [v] for k, v in headers.items()}
+
+        self._signer = V4AwsSigner(
+            credentials,
+            region_name,
+            service_name,
+        )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Message:
+        message: str
+        ts_ms: int  # milliseconds UTC
+
+    @dc.dataclass(frozen=True)
+    class Post:
+        url: str
+        headers: ta.Mapping[str, str]
+        data: bytes
+
+    def feed(self, messages: ta.Sequence[Message]) -> ta.Sequence[Post]:
+        if not messages:
+            return []
+
+        payload = AwsPutLogEventsRequest(
+            log_group_name=self._log_group_name,
+            log_stream_name=self._log_stream_name,
+            log_events=[
+                AwsLogEvent(
+                    message=m.message,
+                    timestamp=m.ts_ms,
+                )
+                for m in messages
+            ],
+        )
+
+        body = json.dumps(
+            payload.to_aws(),
+            indent=None,
+            separators=(',', ':'),
+        ).encode('utf-8')
+
+        sig_req = V4AwsSigner.Request(
+            method='POST',
+            url=self._url,
+            headers=self._headers,
+            payload=body,
+        )
+
+        sig_headers = self._signer.sign(
+            sig_req,
+            sign_payload=False,
+        )
+        sig_req = dc.replace(sig_req, headers={**sig_req.headers, **sig_headers})
+
+        post = AwsLogMessagePoster.Post(
+            url=self._url,
+            headers={k: check_single(v) for k, v in sig_req.headers.items()},
+            data=sig_req.payload,
+        )
+
+        return [post]
+
+
+########################################
+# ../../../../../omlish/lite/subprocesses.py
+
+
+##
+
+
+_SUBPROCESS_SHELL_WRAP_EXECS = False
+
+
+def subprocess_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
+    return ('sh', '-c', ' '.join(map(shlex.quote, args)))
+
+
+def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
+    if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
+        return subprocess_shell_wrap_exec(*args)
+    else:
+        return args
+
+
+def _prepare_subprocess_invocation(
+        *args: str,
+        env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+        extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+        quiet: bool = False,
+        shell: bool = False,
+        **kwargs: ta.Any,
+) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
+    log.debug(args)
+    if extra_env:
+        log.debug(extra_env)
+
+    if extra_env:
+        env = {**(env if env is not None else os.environ), **extra_env}
+
+    if quiet and 'stderr' not in kwargs:
+        if not log.isEnabledFor(logging.DEBUG):
+            kwargs['stderr'] = subprocess.DEVNULL
+
+    if not shell:
+        args = subprocess_maybe_shell_wrap_exec(*args)
+
+    return args, dict(
+        env=env,
+        shell=shell,
+        **kwargs,
+    )
+
+
+def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
+    args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
+    return subprocess.check_call(args, **kwargs)  # type: ignore
+
+
+def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
+    args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
+    return subprocess.check_output(args, **kwargs)
+
+
+def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
+    return subprocess_check_output(*args, **kwargs).decode().strip()
+
+
+##
+
+
+DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
+    FileNotFoundError,
+    subprocess.CalledProcessError,
+)
+
+
+def subprocess_try_call(
+        *args: str,
+        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
+        **kwargs: ta.Any,
+) -> bool:
+    try:
+        subprocess_check_call(*args, **kwargs)
+    except try_exceptions as e:  # noqa
+        if log.isEnabledFor(logging.DEBUG):
+            log.exception('command failed')
+        return False
+    else:
+        return True
+
+
+def subprocess_try_output(
+        *args: str,
+        try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
+        **kwargs: ta.Any,
+) -> ta.Optional[bytes]:
+    try:
+        return subprocess_check_output(*args, **kwargs)
+    except try_exceptions as e:  # noqa
+        if log.isEnabledFor(logging.DEBUG):
+            log.exception('command failed')
+        return None
+
+
+def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
+    out = subprocess_try_output(*args, **kwargs)
+    return out.decode().strip() if out is not None else None
+
+
+########################################
+# ../journald/tailer.py
+
+
+class JournalctlTailerWorker(ThreadWorker):
+    DEFAULT_CMD: ta.ClassVar[ta.Sequence[str]] = ['journalctl']
+
+    def __init__(
+            self,
+            output,  # type: queue.Queue[ta.Sequence[JournalctlMessage]]
+            *,
+            since: ta.Optional[str] = None,
+            after_cursor: ta.Optional[str] = None,
+
+            cmd: ta.Optional[ta.Sequence[str]] = None,
+            shell_wrap: bool = False,
+
+            read_size: int = 0x4000,
+            sleep_s: float = 1.,
+
+            **kwargs: ta.Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+        self._output = output
+
+        self._since = since
+        self._after_cursor = after_cursor
+
+        self._cmd = cmd or self.DEFAULT_CMD
+        self._shell_wrap = shell_wrap
+
+        self._read_size = read_size
+        self._sleep_s = sleep_s
+
+        self._mb = JournalctlMessageBuilder()
+
+        self._proc: ta.Optional[subprocess.Popen] = None
+
+    @cached_nullary
+    def _full_cmd(self) -> ta.Sequence[str]:
+        cmd = [
+            *self._cmd,
+            '--output', 'json',
+            '--show-cursor',
+            '--follow',
+        ]
+
+        if self._since is not None:
+            cmd.extend(['--since', self._since])
+
+        if self._after_cursor is not None:
+            cmd.extend(['--after-cursor', self._after_cursor])
+
+        if self._shell_wrap:
+            cmd = list(subprocess_shell_wrap_exec(*cmd))
+
+        return cmd
+
+    def _run(self) -> None:
+        with subprocess.Popen(
+            self._full_cmd(),
+            stdout=subprocess.PIPE,
+        ) as self._proc:
+            stdout = check_not_none(self._proc.stdout)
+
+            fd = stdout.fileno()
+            fl = fcntl.fcntl(fd, fcntl.F_GETFL)
+            fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+
+            while True:
+                if not self._heartbeat():
+                    break
+
+                while stdout.readable():
+                    if not self._heartbeat():
+                        break
+
+                    buf = stdout.read(self._read_size)
+                    if not buf:
+                        log.debug('Journalctl empty read')
+                        break
+
+                    log.debug('Journalctl read buffer: %r', buf)
+                    msgs = self._mb.feed(buf)
+                    if msgs:
+                        self._output.put(msgs)
+
+                if self._proc.poll() is not None:
+                    log.critical('Journalctl process terminated')
+                    break
+
+                log.debug('Journalctl readable')
+                time.sleep(self._sleep_s)
+
+
+########################################
+# main.py
+
+
+@dc.dataclass(frozen=True)
+class JournalctlOpts:
+    after_cursor: ta.Optional[str] = None
+
+    since: ta.Optional[str] = None
+    until: ta.Optional[str] = None
+
+
+class JournalctlToAws:
+    @dc.dataclass(frozen=True)
+    class Config:
+        pid_file: ta.Optional[str] = None
+
+        cursor_file: ta.Optional[str] = None
+
+        #
+
+        aws_log_group_name: str = 'omlish'
+        aws_log_stream_name: ta.Optional[str] = None
+
+        aws_access_key_id: ta.Optional[str] = None
+        aws_secret_access_key: ta.Optional[str] = dc.field(default=None, repr=False)
+
+        aws_region_name: str = 'us-west-1'
+
+        #
+
+        journalctl_cmd: ta.Optional[ta.Sequence[str]] = None
+
+        journalctl_after_cursor: ta.Optional[str] = None
+        journalctl_since: ta.Optional[str] = None
+
+        #
+
+        dry_run: bool = False
+
+    def __init__(self, config: Config) -> None:
+        super().__init__()
+        self._config = config
+
+    #
+
+    _es: contextlib.ExitStack
+
+    def __enter__(self) -> 'JournalctlToAws':
+        self._es = contextlib.ExitStack().__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return self._es.__exit__(exc_type, exc_val, exc_tb)
+
+    #
+
+    @cached_nullary
+    def _pidfile(self) -> ta.Optional[Pidfile]:
+        if self._config.pid_file is None:
+            return None
+
+        pfp = os.path.expanduser(self._config.pid_file)
+
+        log.info('Opening pidfile %s', pfp)
+
+        pf = self._es.enter_context(Pidfile(pfp))
+        pf.write()
+        return pf
+
+    def _ensure_locked(self) -> None:
+        if (pf := self._pidfile()) is not None:
+            pf.ensure_locked()
+
+    #
+
+    def _read_cursor_file(self) -> ta.Optional[str]:
+        self._ensure_locked()
+
+        if not (cf := self._config.cursor_file):
+            return None
+        cf = os.path.expanduser(cf)
+
+        try:
+            with open(cf) as f:
+                return f.read().strip()
+        except FileNotFoundError:
+            return None
+
+    def _write_cursor_file(self, cursor: str) -> None:
+        self._ensure_locked()
+
+        if not (cf := self._config.cursor_file):
+            return
+        cf = os.path.expanduser(cf)
+
+        log.info('Writing cursor file %s : %s', cf, cursor)
+        with open(ncf := cf + '.next', 'w') as f:
+            f.write(cursor)
+
+        os.rename(ncf, cf)
+
+    #
+
+    @cached_nullary
+    def _aws_credentials(self) -> AwsSigner.Credentials:
+        return AwsSigner.Credentials(
+            access_key_id=check_non_empty_str(self._config.aws_access_key_id),
+            secret_access_key=check_non_empty_str(self._config.aws_secret_access_key),
+        )
+
+    @cached_nullary
+    def _aws_log_message_poster(self) -> AwsLogMessagePoster:
+        return AwsLogMessagePoster(
+            log_group_name=self._config.aws_log_group_name,
+            log_stream_name=check_non_empty_str(self._config.aws_log_stream_name),
+            region_name=self._config.aws_region_name,
+            credentials=check_not_none(self._aws_credentials()),
+        )
+
+    #
+
+    @cached_nullary
+    def _journalctl_message_queue(self):  # type: () -> queue.Queue[ta.Sequence[JournalctlMessage]]
+        return queue.Queue()
+
+    @cached_nullary
+    def _journalctl_tailer_worker(self) -> JournalctlTailerWorker:
+        ac: ta.Optional[str] = self._config.journalctl_after_cursor
+        if ac is None:
+            ac = self._read_cursor_file()
+        if ac is not None:
+            log.info('Starting from cursor %s', ac)
+
+        if (since := self._config.journalctl_since):
+            log.info('Starting since %s', since)
+
+        return JournalctlTailerWorker(
+            self._journalctl_message_queue(),
+
+            since=since,
+            after_cursor=ac,
+
+            cmd=self._config.journalctl_cmd,
+            shell_wrap=is_debugger_attached(),
+        )
+
+    #
+
+    def run(self) -> None:
+        self._ensure_locked()
+
+        q = self._journalctl_message_queue()
+        jtw = self._journalctl_tailer_worker()
+        mp = self._aws_log_message_poster()
+
+        jtw.start()
+
+        last_cursor: ta.Optional[str] = None  # noqa
+        while True:
+            if not jtw.is_alive():
+                log.critical('Journalctl tailer worker died')
+                break
+
+            msgs: ta.Sequence[JournalctlMessage] = q.get()
+            log.debug('%r', msgs)
+
+            cur_cursor: ta.Optional[str] = None
+            for m in reversed(msgs):
+                if m.cursor is not None:
+                    cur_cursor = m.cursor
+                    break
+
+            if not msgs:
+                log.warning('Empty queue chunk')
+                continue
+
+            [post] = mp.feed([mp.Message(
+                message=json.dumps(m.dct),
+                ts_ms=int(time.time() * 1000.),
+            ) for m in msgs])
+            log.debug('%r', post)
+
+            if not self._config.dry_run:
+                with urllib.request.urlopen(urllib.request.Request(  # noqa
+                        post.url,
+                        method='POST',
+                        headers=dict(post.headers),
+                        data=post.data,
+                )) as resp:
+                    response = AwsPutLogEventsResponse.from_aws(json.loads(resp.read().decode('utf-8')))
+                log.debug('%r', response)
+
+            if cur_cursor is not None:
+                self._write_cursor_file(cur_cursor)
+                last_cursor = cur_cursor  # noqa
+
+
+def _main() -> None:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--config-file')
+    parser.add_argument('-v', '--verbose', action='store_true')
+
+    parser.add_argument('--after-cursor', nargs='?')
+    parser.add_argument('--since', nargs='?')
+    parser.add_argument('--dry-run', action='store_true')
+
+    parser.add_argument('--message', nargs='?')
+    parser.add_argument('--real', action='store_true')
+
+    args = parser.parse_args()
+
+    #
+
+    configure_standard_logging('DEBUG' if args.verbose else 'INFO')
+
+    #
+
+    config: JournalctlToAws.Config
+    if args.config_file:
+        with open(os.path.expanduser(args.config_file)) as cf:
+            config_dct = json.load(cf)
+        config = unmarshal_obj(config_dct, JournalctlToAws.Config)
+    else:
+        config = JournalctlToAws.Config()
+
+    #
+
+    for k in ['aws_access_key_id', 'aws_secret_access_key']:
+        if not getattr(config, k) and k.upper() in os.environ:
+            config = dc.replace(config, **{k: os.environ.get(k.upper())})  # type: ignore
+
+    #
+
+    if not args.real:
+        config = dc.replace(config, journalctl_cmd=[
+            sys.executable,
+            os.path.join(os.path.dirname(__file__), 'journald', 'genmessages.py'),
+            '--sleep-n', '2',
+            '--sleep-s', '.5',
+            *(['--message', args.message] if args.message else []),
+            '100000',
+        ])
+
+    #
+
+    for a in ['after_cursor', 'since', 'dry_run']:
+        if (pa := getattr(args, a)):
+            config = dc.replace(config, **{a: pa})
+
+    #
+
+    with JournalctlToAws(config) as jta:
+        jta.run()
+
+
+if __name__ == '__main__':
+    _main()