ominfra 0.0.0.dev77__py3-none-any.whl → 0.0.0.dev79__py3-none-any.whl

ominfra/clouds/aws/auth.py

@@ -1,3 +1,5 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
 """
 https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html
 
@@ -6,6 +8,7 @@ TODO:
   - boto / s3transfer upload_fileobj doesn't stream either lol - eagerly calcs Content-MD5
  - sts tokens
  - !! fix canonical_qs - sort params
+ - secrets
 """
 import dataclasses as dc
 import datetime
@@ -14,123 +17,125 @@ import hmac
 import typing as ta
 import urllib.parse
 
-from omlish import check
-from omlish import lang
+from omlish.lite.check import check_equal
+from omlish.lite.check import check_non_empty_str
+from omlish.lite.check import check_not_isinstance
 
 
 ##
 
 
-HttpMap: ta.TypeAlias = ta.Mapping[str, ta.Sequence[str]]
-
-
-def make_http_map(*kvs: tuple[str, str]) -> HttpMap:
-    out: dict[str, list[str]] = {}
-    for k, v in kvs:
-        out.setdefault(k, []).append(v)
-    return out
-
-
-#
-
-@dc.dataclass(frozen=True)
-class Credentials:
-    access_key: str
-    secret_key: str = dc.field(repr=False)
-
+class AwsSigner:
+    def __init__(
+            self,
+            creds: 'AwsSigner.Credentials',
+            region_name: str,
+            service_name: str,
+    ) -> None:
+        super().__init__()
+        self._creds = creds
+        self._region_name = region_name
+        self._service_name = service_name
 
-@dc.dataclass(frozen=True)
-class Request:
-    method: str
-    url: str
-    headers: HttpMap = dc.field(default_factory=dict)
-    payload: bytes = b''
+    #
 
+    @dc.dataclass(frozen=True)
+    class Credentials:
+        access_key: str
+        secret_key: str = dc.field(repr=False)
 
-##
+    @dc.dataclass(frozen=True)
+    class Request:
+        method: str
+        url: str
+        headers: ta.Mapping[str, ta.Sequence[str]] = dc.field(default_factory=dict)
+        payload: bytes = b''
 
+    #
 
-def _host_from_url(url: str) -> str:
-    url_parts = urllib.parse.urlsplit(url)
-    host = check.non_empty_str(url_parts.hostname)
-    default_ports = {
-        'http': 80,
-        'https': 443,
-    }
-    if url_parts.port is not None:
-        if url_parts.port != default_ports.get(url_parts.scheme):
-            host = '%s:%d' % (host, url_parts.port)
-    return host
+    ISO8601 = '%Y%m%dT%H%M%SZ'
 
+    #
 
-def _as_bytes(data: str | bytes) -> bytes:
-    return data if isinstance(data, bytes) else data.encode('utf-8')
+    @staticmethod
+    def _host_from_url(url: str) -> str:
+        url_parts = urllib.parse.urlsplit(url)
+        host = check_non_empty_str(url_parts.hostname)
+        default_ports = {
+            'http': 80,
+            'https': 443,
+        }
+        if url_parts.port is not None:
+            if url_parts.port != default_ports.get(url_parts.scheme):
+                host = '%s:%d' % (host, url_parts.port)
+        return host
 
+    @staticmethod
+    def _lower_case_http_map(d: ta.Mapping[str, ta.Sequence[str]]) -> ta.Mapping[str, ta.Sequence[str]]:
+        o: ta.Dict[str, ta.List[str]] = {}
+        for k, vs in d.items():
+            o.setdefault(k.lower(), []).extend(check_not_isinstance(vs, str))
+        return o
 
-def _sha256(data: str | bytes) -> str:
-    return hashlib.sha256(_as_bytes(data)).hexdigest()
+    #
 
+    @staticmethod
+    def _as_bytes(data: ta.Union[str, bytes]) -> bytes:
+        return data if isinstance(data, bytes) else data.encode('utf-8')
 
-def _sha256_sign(key: bytes, msg: str | bytes) -> bytes:
-    return hmac.new(key, _as_bytes(msg), hashlib.sha256).digest()
+    @staticmethod
+    def _sha256(data: ta.Union[str, bytes]) -> str:
+        return hashlib.sha256(AwsSigner._as_bytes(data)).hexdigest()
 
+    @staticmethod
+    def _sha256_sign(key: bytes, msg: ta.Union[str, bytes]) -> bytes:
+        return hmac.new(key, AwsSigner._as_bytes(msg), hashlib.sha256).digest()
 
-def _sha256_sign_hex(key: bytes, msg: str | bytes) -> str:
-    return hmac.new(key, _as_bytes(msg), hashlib.sha256).hexdigest()
+    @staticmethod
+    def _sha256_sign_hex(key: bytes, msg: ta.Union[str, bytes]) -> str:
+        return hmac.new(key, AwsSigner._as_bytes(msg), hashlib.sha256).hexdigest()
 
+    _EMPTY_SHA256: str
 
-_EMPTY_SHA256 = _sha256(b'')
+    #
 
-_ISO8601 = '%Y%m%dT%H%M%SZ'
+    _SIGNED_HEADERS_BLACKLIST = frozenset([
+        'authorization',
+        'expect',
+        'user-agent',
+        'x-amzn-trace-id',
+    ])
 
-_SIGNED_HEADERS_BLACKLIST = frozenset([
-    'expect',
-    'user-agent',
-    'x-amzn-trace-id',
-])
+    def _validate_request(self, req: Request) -> None:
+        check_non_empty_str(req.method)
+        check_equal(req.method.upper(), req.method)
+        for k, vs in req.headers.items():
+            check_equal(k.strip(), k)
+            for v in vs:
+                check_equal(v.strip(), v)
 
 
-def _lower_case_http_map(d: HttpMap) -> HttpMap:
-    o: dict[str, list[str]] = {}
-    for k, vs in d.items():
-        o.setdefault(k.lower(), []).extend(vs)
-    return o
+AwsSigner._EMPTY_SHA256 = AwsSigner._sha256(b'')  # noqa
 
 
-class V4AwsSigner:
-    def __init__(
-            self,
-            creds: Credentials,
-            region_name: str,
-            service_name: str,
-    ) -> None:
-        super().__init__()
-        self._creds = creds
-        self._region_name = region_name
-        self._service_name = service_name
+##
 
-    def _validate_request(self, req: Request) -> None:
-        check.non_empty_str(req.method)
-        check.equal(req.method.upper(), req.method)
-        for k, vs in req.headers.items():
-            check.equal(k.strip(), k)
-            for v in vs:
-                check.equal(v.strip(), v)
 
+class V4AwsSigner(AwsSigner):
     def sign(
             self,
-            req: Request,
+            req: AwsSigner.Request,
             *,
             sign_payload: bool = False,
-            utcnow: datetime.datetime | None = None,
-    ) -> HttpMap:
+            utcnow: ta.Optional[datetime.datetime] = None,
+    ) -> ta.Mapping[str, ta.Sequence[str]]:
         self._validate_request(req)
 
         #
 
         if utcnow is None:
-            utcnow = lang.utcnow()
-        req_dt = utcnow.strftime(_ISO8601)
+            utcnow = datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+        req_dt = utcnow.strftime(self.ISO8601)
 
         #
 
@@ -140,18 +145,18 @@ class V4AwsSigner:
 
         #
 
-        headers_to_sign = {
-            k: v
-            for k, v in _lower_case_http_map(req.headers).items()
-            if k not in _SIGNED_HEADERS_BLACKLIST
+        headers_to_sign: ta.Dict[str, ta.List[str]] = {
+            k: list(v)
+            for k, v in self._lower_case_http_map(req.headers).items()
+            if k not in self._SIGNED_HEADERS_BLACKLIST
         }
 
         if 'host' not in headers_to_sign:
-            headers_to_sign['host'] = [_host_from_url(req.url)]
+            headers_to_sign['host'] = [self._host_from_url(req.url)]
 
         headers_to_sign['x-amz-date'] = [req_dt]
 
-        hashed_payload = _sha256(req.payload) if req.payload else _EMPTY_SHA256
+        hashed_payload = self._sha256(req.payload) if req.payload else self._EMPTY_SHA256
         if sign_payload:
             headers_to_sign['x-amz-content-sha256'] = [hashed_payload]
 
@@ -183,7 +188,7 @@ class V4AwsSigner:
             'aws4_request',
         ]
         scope = '/'.join(scope_parts)
-        hashed_canon_req = _sha256(canon_req)
+        hashed_canon_req = self._sha256(canon_req)
         string_to_sign = '\n'.join([
             algorithm,
             req_dt,
@@ -194,11 +199,11 @@ class V4AwsSigner:
         #
 
         key = self._creds.secret_key
-        key_date = _sha256_sign(f'AWS4{key}'.encode('utf-8'), req_dt[:8])  # noqa
-        key_region = _sha256_sign(key_date, self._region_name)
-        key_service = _sha256_sign(key_region, self._service_name)
-        key_signing = _sha256_sign(key_service, 'aws4_request')
-        sig = _sha256_sign_hex(key_signing, string_to_sign)
+        key_date = self._sha256_sign(f'AWS4{key}'.encode('utf-8'), req_dt[:8])  # noqa
+        key_region = self._sha256_sign(key_date, self._region_name)
+        key_service = self._sha256_sign(key_region, self._service_name)
+        key_signing = self._sha256_sign(key_service, 'aws4_request')
+        sig = self._sha256_sign_hex(key_signing, string_to_sign)
 
         #
 

ominfra/clouds/aws/dataclasses.py

@@ -0,0 +1,149 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import collections.abc
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.reflect import get_optional_alias_arg
+from omlish.lite.reflect import is_generic_alias
+from omlish.lite.reflect import is_optional_alias
+from omlish.lite.strings import camel_case
+
+
+class AwsDataclass:
+    class Raw(dict):
+        pass
+
+    #
+
+    _aws_meta: ta.ClassVar[ta.Optional['AwsDataclassMeta']] = None
+
+    @classmethod
+    def _get_aws_meta(cls) -> 'AwsDataclassMeta':
+        try:
+            return cls.__dict__['_aws_meta']
+        except KeyError:
+            pass
+        ret = cls._aws_meta = AwsDataclassMeta(cls)
+        return ret
+
+    #
+
+    def to_aws(self) -> ta.Mapping[str, ta.Any]:
+        return self._get_aws_meta().converters().d2a(self)
+
+    @classmethod
+    def from_aws(cls, v: ta.Mapping[str, ta.Any]) -> 'AwsDataclass':
+        return cls._get_aws_meta().converters().a2d(v)
+
+
+@dc.dataclass(frozen=True)
+class AwsDataclassMeta:
+    cls: ta.Type['AwsDataclass']
+
+    #
+
+    class Field(ta.NamedTuple):
+        d_name: str
+        a_name: str
+        is_opt: bool
+        is_seq: bool
+        dc_cls: ta.Optional[ta.Type['AwsDataclass']]
+
+    @cached_nullary
+    def fields(self) -> ta.Sequence[Field]:
+        fs = []
+        for f in dc.fields(self.cls):  # type: ignore  # noqa
+            d_name = f.name
+            a_name = camel_case(d_name, lower=True)
+
+            is_opt = False
+            is_seq = False
+            dc_cls = None
+
+            c = f.type
+            if c is AwsDataclass.Raw:
+                continue
+
+            if is_optional_alias(c):
+                is_opt = True
+                c = get_optional_alias_arg(c)
+
+            if is_generic_alias(c) and ta.get_origin(c) is collections.abc.Sequence:
+                is_seq = True
+                [c] = ta.get_args(c)
+
+            if is_generic_alias(c):
+                raise TypeError(c)
+
+            if isinstance(c, type) and issubclass(c, AwsDataclass):
+                dc_cls = c
+
+            fs.append(AwsDataclassMeta.Field(
+                d_name=d_name,
+                a_name=a_name,
+                is_opt=is_opt,
+                is_seq=is_seq,
+                dc_cls=dc_cls,
+            ))
+
+        return fs
+
+    #
+
+    class Converters(ta.NamedTuple):
+        d2a: ta.Callable
+        a2d: ta.Callable
+
+    @cached_nullary
+    def converters(self) -> Converters:
+        for df in dc.fields(self.cls):  # type: ignore  # noqa
+            c = df.type
+
+            if is_optional_alias(c):
+                c = get_optional_alias_arg(c)
+
+            if c is AwsDataclass.Raw:
+                rf = df.name
+                break
+
+        else:
+            rf = None
+
+        fs = [
+            (f, f.dc_cls._get_aws_meta().converters() if f.dc_cls is not None else None)  # noqa
+            for f in self.fields()
+        ]
+
+        def d2a(o):
+            dct = {}
+            for f, cs in fs:
+                x = getattr(o, f.d_name)
+                if x is None:
+                    continue
+                if cs is not None:
+                    if f.is_seq:
+                        x = list(map(cs.d2a, x))
+                    else:
+                        x = cs.d2a(x)
+                dct[f.a_name] = x
+            return dct
+
+        def a2d(v):
+            dct = {}
+            for f, cs in fs:
+                x = v.get(f.a_name)
+                if x is None:
+                    continue
+                if cs is not None:
+                    if f.is_seq:
+                        x = list(map(cs.a2d, x))
+                    else:
+                        x = cs.a2d(x)
+                dct[f.d_name] = x
+            if rf is not None:
+                dct[rf] = self.cls.Raw(v)
+            return self.cls(**dct)
+
+        return AwsDataclassMeta.Converters(d2a, a2d)

ominfra/clouds/aws/journald2aws/journald.py

@@ -0,0 +1,67 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import typing as ta
+
+from omlish.lite.check import check_isinstance
+from omlish.lite.io import DelimitingBuffer
+from omlish.lite.logs import log
+
+
+@dc.dataclass(frozen=True)
+class JournalctlMessage:
+    raw: bytes
+    dct: ta.Optional[ta.Mapping[str, ta.Any]] = None
+    cursor: ta.Optional[str] = None
+    ts_us: ta.Optional[int] = None  # microseconds UTC
+
+
+class JournalctlMessageBuilder:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._buf = DelimitingBuffer(b'\n')
+
+    _cursor_field = '__CURSOR'
+    _timestamp_field = '_SOURCE_REALTIME_TIMESTAMP'
+
+    def _make_message(self, raw: bytes) -> JournalctlMessage:
+        dct = None
+        cursor = None
+        ts = None
+
+        try:
+            dct = json.loads(raw.decode('utf-8', 'replace'))
+        except Exception:  # noqa
+            log.exception('Failed to parse raw message: %r', raw)
+
+        else:
+            cursor = dct.get(self._cursor_field)
+
+            if tsv := dct.get(self._timestamp_field):
+                if isinstance(tsv, str):
+                    try:
+                        ts = int(tsv)
+                    except ValueError:
+                        try:
+                            ts = int(float(tsv))
+                        except ValueError:
+                            log.exception('Failed to parse timestamp: %r', tsv)
+                elif isinstance(tsv, (int, float)):
+                    ts = int(tsv)
+                else:
+                    log.exception('Invalid timestamp: %r', tsv)
+
+        return JournalctlMessage(
+            raw=raw,
+            dct=dct,
+            cursor=cursor,
+            ts_us=ts,
+        )
+
+    def feed(self, data: bytes) -> ta.Sequence[JournalctlMessage]:
+        ret: ta.List[JournalctlMessage] = []
+        for line in self._buf.feed(data):
+            ret.append(self._make_message(check_isinstance(line, bytes)))  # type: ignore
+        return ret

ominfra/clouds/aws/logs.py

@@ -0,0 +1,173 @@
+# @omlish-lite
+# ruff: noqa: UP007
+"""
+https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html :
+ - The maximum batch size is 1,048,576 bytes. This size is calculated as the sum of all event messages in UTF-8, plus 26
+   bytes for each log event.
+ - None of the log events in the batch can be more than 2 hours in the future.
+ - None of the log events in the batch can be more than 14 days in the past. Also, none of the log events can be from
+   earlier than the retention period of the log group.
+ - The log events in the batch must be in chronological order by their timestamp. The timestamp is the time that the
+   event occurred, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. (In AWS Tools for PowerShell
+   and the AWS SDK for .NET, the timestamp is specified in .NET format: yyyy-mm-ddThh:mm:ss. For example,
+   2017-09-15T13:45:30.)
+ - A batch of log events in a single request cannot span more than 24 hours. Otherwise, the operation fails.
+ - Each log event can be no larger than 256 KB.
+ - The maximum number of log events in a batch is 10,000.
+"""
+import dataclasses as dc
+import json
+import typing as ta
+
+from omlish.lite.check import check_non_empty_str
+from omlish.lite.check import check_single
+
+from .auth import AwsSigner
+from .auth import V4AwsSigner
+from .dataclasses import AwsDataclass
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class AwsLogEvent(AwsDataclass):
+    message: str
+    timestamp: int  # milliseconds UTC
+
+
+@dc.dataclass(frozen=True)
+class AwsPutLogEventsRequest(AwsDataclass):
+    log_group_name: str
+    log_stream_name: str
+    log_events: ta.Sequence[AwsLogEvent]
+    sequence_token: ta.Optional[str] = None
+
+
+@dc.dataclass(frozen=True)
+class AwsRejectedLogEventsInfo(AwsDataclass):
+    expired_log_event_end_index: ta.Optional[int] = None
+    too_new_log_event_start_index: ta.Optional[int] = None
+    too_old_log_event_end_index: ta.Optional[int] = None
+
+
+@dc.dataclass(frozen=True)
+class AwsPutLogEventsResponse(AwsDataclass):
+    next_sequence_token: ta.Optional[str] = None
+    rejected_log_events_info: ta.Optional[AwsRejectedLogEventsInfo] = None
+
+    raw: ta.Optional[AwsDataclass.Raw] = None
+
+
+##
+
+
+class AwsLogMessagePoster:
+    """
+    TODO:
+     - max_items
+     - max_bytes - manually build body
+     - flush_interval
+     - !! sort by timestamp
+    """
+
+    DEFAULT_URL = 'https://logs.{region_name}.amazonaws.com/'  # noqa
+
+    DEFAULT_SERVICE_NAME = 'logs'
+
+    DEFAULT_TARGET = 'Logs_20140328.PutLogEvents'
+    DEFAULT_CONTENT_TYPE = 'application/x-amz-json-1.1'
+
+    DEFAULT_HEADERS: ta.Mapping[str, str] = {
+        'X-Amz-Target': DEFAULT_TARGET,
+        'Content-Type': DEFAULT_CONTENT_TYPE,
+    }
+
+    def __init__(
+            self,
+            log_group_name: str,
+            log_stream_name: str,
+            region_name: str,
+            credentials: AwsSigner.Credentials,
+
+            url: ta.Optional[str] = None,
+            service_name: str = DEFAULT_SERVICE_NAME,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            extra_headers: ta.Optional[ta.Mapping[str, str]] = None,
+    ) -> None:
+        super().__init__()
+
+        self._log_group_name = check_non_empty_str(log_group_name)
+        self._log_stream_name = check_non_empty_str(log_stream_name)
+
+        if url is None:
+            url = self.DEFAULT_URL.format(region_name=region_name)
+        self._url = url
+
+        if headers is None:
+            headers = self.DEFAULT_HEADERS
+        if extra_headers is not None:
+            headers = {**headers, **extra_headers}
+        self._headers = {k: [v] for k, v in headers.items()}
+
+        self._signer = V4AwsSigner(
+            credentials,
+            region_name,
+            service_name,
+        )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Message:
+        message: str
+        ts_ms: int  # milliseconds UTC
+
+    @dc.dataclass(frozen=True)
+    class Post:
+        url: str
+        headers: ta.Mapping[str, str]
+        data: bytes
+
+    def feed(self, messages: ta.Sequence[Message]) -> ta.Sequence[Post]:
+        if not messages:
+            return []
+
+        payload = AwsPutLogEventsRequest(
+            log_group_name=self._log_group_name,
+            log_stream_name=self._log_stream_name,
+            log_events=[
+                AwsLogEvent(
+                    message=m.message,
+                    timestamp=m.ts_ms,
+                )
+                for m in messages
+            ],
+        )
+
+        body = json.dumps(
+            payload.to_aws(),
+            indent=None,
+            separators=(',', ':'),
+        ).encode('utf-8')
+
+        sig_req = V4AwsSigner.Request(
+            method='POST',
+            url=self._url,
+            headers=self._headers,
+            payload=body,
+        )
+
+        sig_headers = self._signer.sign(
+            sig_req,
+            sign_payload=False,
+        )
+        sig_req = dc.replace(sig_req, headers={**sig_req.headers, **sig_headers})
+
+        post = AwsLogMessagePoster.Post(
+            url=self._url,
+            headers={k: check_single(v) for k, v in sig_req.headers.items()},
+            data=sig_req.payload,
+        )
+
+        return [post]

ominfra/deploy/_executor.py

@@ -168,6 +168,23 @@ def check_state(v: bool, msg: str = 'Illegal state') -> None:
         raise ValueError(msg)
 
 
+def check_equal(l: T, r: T) -> T:
+    if l != r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_not_equal(l: T, r: T) -> T:
+    if l == r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_single(vs: ta.Iterable[T]) -> T:
+    [v] = vs
+    return v
+
+
 ########################################
 # ../../../../omlish/lite/json.py
 

ominfra/pyremote/_runcommands.py

@@ -251,6 +251,23 @@ def check_state(v: bool, msg: str = 'Illegal state') -> None:
         raise ValueError(msg)
 
 
+def check_equal(l: T, r: T) -> T:
+    if l != r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_not_equal(l: T, r: T) -> T:
+    if l == r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_single(vs: ta.Iterable[T]) -> T:
+    [v] = vs
+    return v
+
+
 ########################################
 # ../../../omlish/lite/json.py
 

ominfra/{supervisor/_supervisor.py → scripts/supervisor.py}

@@ -2,7 +2,7 @@
 # noinspection DuplicatedCode
 # @omlish-lite
 # @omlish-script
-# @omlish-amalg-output supervisor.py
+# @omlish-amalg-output ../supervisor/supervisor.py
 # ruff: noqa: N802 UP006 UP007 UP036
 import abc
 import contextlib
@@ -717,6 +717,23 @@ def check_state(v: bool, msg: str = 'Illegal state') -> None:
         raise ValueError(msg)
 
 
+def check_equal(l: T, r: T) -> T:
+    if l != r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_not_equal(l: T, r: T) -> T:
+    if l == r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_single(vs: ta.Iterable[T]) -> T:
+    [v] = vs
+    return v
+
+
 ########################################
 # ../../../omlish/lite/json.py
 
@@ -1992,12 +2009,6 @@ class OutputDispatcher(Dispatcher):
     - route the output to the appropriate log handlers as specified in the config.
     """
 
-    child_log = None  # the current logger (normal_log or capture_log)
-    normal_log = None  # the "normal" (non-capture) logger
-    capture_log = None  # the logger used while we're in capture_mode
-    capture_mode = False  # are we capturing process event data
-    output_buffer = b''  # data waiting to be logged
-
     def __init__(self, process: AbstractSubprocess, event_type, fd):
         """
         Initialize the dispatcher.
@@ -2012,7 +2023,10 @@ class OutputDispatcher(Dispatcher):
         self._init_normal_log()
         self._init_capture_log()
 
-        self.child_log = self.normal_log
+        self._child_log = self._normal_log
+
+        self._capture_mode = False  # are we capturing process event data
+        self._output_buffer = b''  # data waiting to be logged
 
         # all code below is purely for minor speedups
         begin_token = self.event_type.BEGIN_TOKEN
@@ -2025,6 +2039,10 @@ class OutputDispatcher(Dispatcher):
         self.stdout_events_enabled = config.stdout.events_enabled
         self.stderr_events_enabled = config.stderr.events_enabled
 
+    _child_log: ta.Optional[logging.Logger]  # the current logger (normal_log or capture_log)
+    _normal_log: ta.Optional[logging.Logger]  # the "normal" (non-capture) logger
+    _capture_log: ta.Optional[logging.Logger]  # the logger used while we're in capture_mode
+
     def _init_normal_log(self) -> None:
         """
         Configure the "normal" (non-capture) log for this channel of this process. Sets self.normal_log if logging is
@@ -2039,7 +2057,7 @@ class OutputDispatcher(Dispatcher):
         to_syslog = self.lc.syslog
 
         if logfile or to_syslog:
-            self.normal_log = logging.getLogger(__name__)
+            self._normal_log = logging.getLogger(__name__)
 
         # if logfile:
         #     loggers.handle_file(
@@ -2057,29 +2075,29 @@ class OutputDispatcher(Dispatcher):
         #         fmt=config.name + ' %(message)s',
         #     )
 
-    def _init_capture_log(self):
+    def _init_capture_log(self) -> None:
         """
         Configure the capture log for this process.  This log is used to temporarily capture output when special output
         is detected. Sets self.capture_log if capturing is enabled.
         """
         capture_maxbytes = self.lc.capture_maxbytes
         if capture_maxbytes:
-            self.capture_log = logging.getLogger(__name__)
+            self._capture_log = logging.getLogger(__name__)
             # loggers.handle_boundIO(
-            #     self.capture_log,
+            #     self._capture_log,
             #     fmt='%(message)s',
             #     maxbytes=capture_maxbytes,
             # )
 
     def remove_logs(self):
-        for log in (self.normal_log, self.capture_log):
+        for log in (self._normal_log, self._capture_log):
             if log is not None:
                 for handler in log.handlers:
                     handler.remove()  # type: ignore
                     handler.reopen()  # type: ignore
 
     def reopen_logs(self):
-        for log in (self.normal_log, self.capture_log):
+        for log in (self._normal_log, self._capture_log):
             if log is not None:
                 for handler in log.handlers:
                     handler.reopen()  # type: ignore
@@ -2088,8 +2106,8 @@ class OutputDispatcher(Dispatcher):
         if data:
             if self._process.context.config.strip_ansi:
                 data = strip_escapes(data)
-            if self.child_log:
-                self.child_log.info(data)  # type: ignore
+            if self._child_log:
+                self._child_log.info(data)
             if self.log_to_main_log:
                 if not isinstance(data, bytes):
                     text = data
@@ -2106,23 +2124,23 @@ class OutputDispatcher(Dispatcher):
                 notify_event(ProcessLogStderrEvent(self._process, self._process.pid, data))
 
     def record_output(self):
-        if self.capture_log is None:
+        if self._capture_log is None:
             # shortcut trying to find capture data
-            data = self.output_buffer
-            self.output_buffer = b''
+            data = self._output_buffer
+            self._output_buffer = b''
             self._log(data)
             return
 
-        if self.capture_mode:
+        if self._capture_mode:
             token, tokenlen = self.end_token_data
         else:
             token, tokenlen = self.begin_token_data
 
-        if len(self.output_buffer) <= tokenlen:
+        if len(self._output_buffer) <= tokenlen:
             return  # not enough data
 
-        data = self.output_buffer
-        self.output_buffer = b''
+        data = self._output_buffer
+        self._output_buffer = b''
 
         try:
             before, after = data.split(token, 1)
@@ -2130,37 +2148,37 @@ class OutputDispatcher(Dispatcher):
             after = None
             index = find_prefix_at_end(data, token)
             if index:
-                self.output_buffer = self.output_buffer + data[-index:]
+                self._output_buffer = self._output_buffer + data[-index:]
                 data = data[:-index]
             self._log(data)
         else:
             self._log(before)
             self.toggle_capture_mode()
-            self.output_buffer = after  # type: ignore
+            self._output_buffer = after  # type: ignore
 
         if after:
             self.record_output()
 
     def toggle_capture_mode(self):
-        self.capture_mode = not self.capture_mode
+        self._capture_mode = not self._capture_mode
 
-        if self.capture_log is not None:
-            if self.capture_mode:
-                self.child_log = self.capture_log
+        if self._capture_log is not None:
+            if self._capture_mode:
+                self._child_log = self._capture_log
             else:
-                for handler in self.capture_log.handlers:
+                for handler in self._capture_log.handlers:
                     handler.flush()
-                data = self.capture_log.getvalue()  # type: ignore
+                data = self._capture_log.getvalue()  # type: ignore
                 channel = self._channel
                 procname = self._process.config.name
                 event = self.event_type(self._process, self._process.pid, data)
                 notify_event(event)
 
                 log.debug('%r %s emitted a comm event', procname, channel)
-                for handler in self.capture_log.handlers:
+                for handler in self._capture_log.handlers:
                     handler.remove()  # type: ignore
                     handler.reopen()  # type: ignore
-                self.child_log = self.normal_log
+                self._child_log = self._normal_log
 
     def writable(self) -> bool:
         return False
@@ -2172,7 +2190,7 @@ class OutputDispatcher(Dispatcher):
 
     def handle_read_event(self) -> None:
         data = readfd(self._fd)
-        self.output_buffer += data
+        self._output_buffer += data
         self.record_output()
         if not data:
             # if we get no data back from the pipe, it means that the child process has ended.  See

ominfra/supervisor/dispatchers.py

@@ -1,7 +1,9 @@
+# ruff: noqa: UP007
 import abc
 import errno
 import logging
 import os
+import typing as ta
 
 from .compat import as_bytes
 from .compat import compact_traceback
@@ -85,12 +87,6 @@ class OutputDispatcher(Dispatcher):
     - route the output to the appropriate log handlers as specified in the config.
     """
 
-    child_log = None  # the current logger (normal_log or capture_log)
-    normal_log = None  # the "normal" (non-capture) logger
-    capture_log = None  # the logger used while we're in capture_mode
-    capture_mode = False  # are we capturing process event data
-    output_buffer = b''  # data waiting to be logged
-
     def __init__(self, process: AbstractSubprocess, event_type, fd):
         """
         Initialize the dispatcher.
@@ -105,7 +101,10 @@ class OutputDispatcher(Dispatcher):
         self._init_normal_log()
         self._init_capture_log()
 
-        self.child_log = self.normal_log
+        self._child_log = self._normal_log
+
+        self._capture_mode = False  # are we capturing process event data
+        self._output_buffer = b''  # data waiting to be logged
 
         # all code below is purely for minor speedups
         begin_token = self.event_type.BEGIN_TOKEN
@@ -118,6 +117,10 @@ class OutputDispatcher(Dispatcher):
         self.stdout_events_enabled = config.stdout.events_enabled
         self.stderr_events_enabled = config.stderr.events_enabled
 
+    _child_log: ta.Optional[logging.Logger]  # the current logger (normal_log or capture_log)
+    _normal_log: ta.Optional[logging.Logger]  # the "normal" (non-capture) logger
+    _capture_log: ta.Optional[logging.Logger]  # the logger used while we're in capture_mode
+
     def _init_normal_log(self) -> None:
         """
         Configure the "normal" (non-capture) log for this channel of this process. Sets self.normal_log if logging is
@@ -132,7 +135,7 @@ class OutputDispatcher(Dispatcher):
         to_syslog = self.lc.syslog
 
         if logfile or to_syslog:
-            self.normal_log = logging.getLogger(__name__)
+            self._normal_log = logging.getLogger(__name__)
 
         # if logfile:
         #     loggers.handle_file(
@@ -150,29 +153,29 @@ class OutputDispatcher(Dispatcher):
         #         fmt=config.name + ' %(message)s',
         #     )
 
-    def _init_capture_log(self):
+    def _init_capture_log(self) -> None:
         """
         Configure the capture log for this process.  This log is used to temporarily capture output when special output
         is detected. Sets self.capture_log if capturing is enabled.
         """
         capture_maxbytes = self.lc.capture_maxbytes
         if capture_maxbytes:
-            self.capture_log = logging.getLogger(__name__)
+            self._capture_log = logging.getLogger(__name__)
             # loggers.handle_boundIO(
-            #     self.capture_log,
+            #     self._capture_log,
             #     fmt='%(message)s',
             #     maxbytes=capture_maxbytes,
             # )
 
     def remove_logs(self):
-        for log in (self.normal_log, self.capture_log):
+        for log in (self._normal_log, self._capture_log):
             if log is not None:
                 for handler in log.handlers:
                     handler.remove()  # type: ignore
                     handler.reopen()  # type: ignore
 
     def reopen_logs(self):
-        for log in (self.normal_log, self.capture_log):
+        for log in (self._normal_log, self._capture_log):
             if log is not None:
                 for handler in log.handlers:
                     handler.reopen()  # type: ignore
@@ -181,8 +184,8 @@ class OutputDispatcher(Dispatcher):
         if data:
             if self._process.context.config.strip_ansi:
                 data = strip_escapes(data)
-            if self.child_log:
-                self.child_log.info(data)  # type: ignore
+            if self._child_log:
+                self._child_log.info(data)
             if self.log_to_main_log:
                 if not isinstance(data, bytes):
                     text = data
@@ -199,23 +202,23 @@ class OutputDispatcher(Dispatcher):
                 notify_event(ProcessLogStderrEvent(self._process, self._process.pid, data))
 
     def record_output(self):
-        if self.capture_log is None:
+        if self._capture_log is None:
             # shortcut trying to find capture data
-            data = self.output_buffer
-            self.output_buffer = b''
+            data = self._output_buffer
+            self._output_buffer = b''
             self._log(data)
             return
 
-        if self.capture_mode:
+        if self._capture_mode:
             token, tokenlen = self.end_token_data
         else:
             token, tokenlen = self.begin_token_data
 
-        if len(self.output_buffer) <= tokenlen:
+        if len(self._output_buffer) <= tokenlen:
             return  # not enough data
 
-        data = self.output_buffer
-        self.output_buffer = b''
+        data = self._output_buffer
+        self._output_buffer = b''
 
         try:
             before, after = data.split(token, 1)
@@ -223,37 +226,37 @@ class OutputDispatcher(Dispatcher):
             after = None
             index = find_prefix_at_end(data, token)
             if index:
-                self.output_buffer = self.output_buffer + data[-index:]
+                self._output_buffer = self._output_buffer + data[-index:]
                 data = data[:-index]
             self._log(data)
         else:
             self._log(before)
             self.toggle_capture_mode()
-            self.output_buffer = after  # type: ignore
+            self._output_buffer = after  # type: ignore
 
         if after:
             self.record_output()
 
     def toggle_capture_mode(self):
-        self.capture_mode = not self.capture_mode
+        self._capture_mode = not self._capture_mode
 
-        if self.capture_log is not None:
-            if self.capture_mode:
-                self.child_log = self.capture_log
+        if self._capture_log is not None:
+            if self._capture_mode:
+                self._child_log = self._capture_log
             else:
-                for handler in self.capture_log.handlers:
+                for handler in self._capture_log.handlers:
                     handler.flush()
-                data = self.capture_log.getvalue()  # type: ignore
+                data = self._capture_log.getvalue()  # type: ignore
                 channel = self._channel
                 procname = self._process.config.name
                 event = self.event_type(self._process, self._process.pid, data)
                 notify_event(event)
 
                 log.debug('%r %s emitted a comm event', procname, channel)
-                for handler in self.capture_log.handlers:
+                for handler in self._capture_log.handlers:
                     handler.remove()  # type: ignore
                     handler.reopen()  # type: ignore
-                self.child_log = self.normal_log
+                self._child_log = self._normal_log
 
     def writable(self) -> bool:
         return False
@@ -265,7 +268,7 @@ class OutputDispatcher(Dispatcher):
 
     def handle_read_event(self) -> None:
         data = readfd(self._fd)
-        self.output_buffer += data
+        self._output_buffer += data
         self.record_output()
         if not data:
             # if we get no data back from the pipe, it means that the child process has ended.  See

ominfra/supervisor/supervisor.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # ruff: noqa: UP006 UP007
-# @omlish-amalg _supervisor.py
+# @omlish-amalg ../scripts/supervisor.py
 import logging
 import signal
 import time

{ominfra-0.0.0.dev77.dist-info → ominfra-0.0.0.dev79.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev77
+Version: 0.0.0.dev79
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omdev ==0.0.0.dev77
-Requires-Dist: omlish ==0.0.0.dev77
+Requires-Dist: omdev ==0.0.0.dev79
+Requires-Dist: omlish ==0.0.0.dev79
 Provides-Extra: all
 Requires-Dist: paramiko ~=3.5 ; extra == 'all'
 Requires-Dist: asyncssh ~=2.17 ; (python_version < "3.13") and extra == 'all'

{ominfra-0.0.0.dev77.dist-info → ominfra-0.0.0.dev79.dist-info}/RECORD

@@ -6,11 +6,15 @@ ominfra/ssh.py,sha256=jQpc4WvkMckIfk4vILda8zFaeharRqc_6wxW50b0OjQ,5431
 ominfra/clouds/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ominfra/clouds/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ominfra/clouds/aws/__main__.py,sha256=HXMoxEl9KHhv6zOOPQxiJAftfR2SjBqeVTYw-og9aFw,163
-ominfra/clouds/aws/auth.py,sha256=EW3lK1U0hnjXkTn1KWJeuv9GG0ibbKdvgLD0P6HJtwo,5502
+ominfra/clouds/aws/auth.py,sha256=2X6S9ARNx3RK3KZSAJe7XunvlN0tBKEcySDvZaJA8fE,6185
 ominfra/clouds/aws/cli.py,sha256=OJVVLIwSy1378drkgP1ke_JltbyzBmnrB_Lom6A83os,510
+ominfra/clouds/aws/dataclasses.py,sha256=rKhtJKJ0JhMssU9n9CABX_JaUiokIboEATJ9TZgZQ6A,3868
+ominfra/clouds/aws/logs.py,sha256=VIT1fQazuaVSOfIlIcA9IxGT0Bqb69dbqcnbaTMEvSE,5185
 ominfra/clouds/aws/metadata.py,sha256=XR1BuMdQheyeFjjA3MN8GCNWVAp5ahoPdbWXEmViutQ,2767
+ominfra/clouds/aws/journald2aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ominfra/clouds/aws/journald2aws/journald.py,sha256=4Yv7GZA-1wloufTNPud5BUYGS5njVokAdb5onFPHza4,1990
 ominfra/deploy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ominfra/deploy/_executor.py,sha256=Xoib_itZzHzAQJ-AuJ2VVmGW5TG20DxUDaeDRbfs5cQ,32569
+ominfra/deploy/_executor.py,sha256=zHn4zAz6Ch1i5R_EdKTfJv_4SE0QPNuQEk7O1ptB_7A,32834
 ominfra/deploy/configs.py,sha256=qi0kwT7G2NH7dXLOQic-u6R3yeadup_QtvrjwWIggbM,435
 ominfra/deploy/remote.py,sha256=6ACmpXU1uBdyGs3Xsp97ktKFq30cJlzN9LRWNUWlGY4,2144
 ominfra/deploy/executor/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
@@ -39,31 +43,32 @@ ominfra/deploy/poly/venv.py,sha256=BoipDEa4NTeodjf3L57KJfq9eGKLagFNKwD8pS4yrzA,1
 ominfra/manage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ominfra/manage/manage.py,sha256=BttL8LFEknHZE_h2Pt5dAqbfUkv6qy43WI0raXBZ1a8,151
 ominfra/pyremote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ominfra/pyremote/_runcommands.py,sha256=nPtvx_oFHIViYb7V0ius23TF_2_SmEfDzdxa1_5LISc,26405
+ominfra/pyremote/_runcommands.py,sha256=2UVHaUJjmWP8jMZE79j2Qk1E5IOyVV1qNSUbdr-zKYM,26670
 ominfra/pyremote/bootstrap.py,sha256=RvMO3YGaN1E4sgUi1JEtiPak8cjvqtc_vRCq1yqbeZg,3370
 ominfra/pyremote/runcommands.py,sha256=bviS0_TDIoZVAe4h-_iavbvJtVSFu8lnk7fQ5iasCWE,1571
+ominfra/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ominfra/scripts/supervisor.py,sha256=eVD0NeSsnv-ckzb3c-EgzDRq7Kyr2dpvpPquW3Y690w,105707
 ominfra/supervisor/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 ominfra/supervisor/__main__.py,sha256=usW9jjq5JPe_2SL8H5PrjDdksO75MX85Ir0HFfb35eM,72
-ominfra/supervisor/_supervisor.py,sha256=ONztv2soL9y73S9Ax8zyG_-0KvEcHzFIbZb73XFkCao,105313
 ominfra/supervisor/compat.py,sha256=sqsvlCNF2iMFdrc0LuTfyCBxXSVACtQx2wCfeHwWvAQ,5044
 ominfra/supervisor/configs.py,sha256=FjgsFijC_ivqJkLua4ZV0UWjDxP3JeDua3aVy4_CnbM,2970
 ominfra/supervisor/context.py,sha256=xh03VN8e4hHj5udjtgUvYnqUticTzCCXdIA0Xp4Ba2c,15335
 ominfra/supervisor/datatypes.py,sha256=cq2p7wnLN0nvKT-jZxaBByqsnCIUz6pX9dPtm69h18Q,4428
-ominfra/supervisor/dispatchers.py,sha256=ye-gPdZ4RnOD2pE0mt2buEwtYdRR1vNa1xvCAMDwsTw,10212
+ominfra/supervisor/dispatchers.py,sha256=4AVtJagOEalRnxwCZjNr8LpkCkykzzg2ncMZiYsC4Bc,10367
 ominfra/supervisor/events.py,sha256=wT-gPfvv2HCAyQXMq3jiek17Jq6kAZb0U2hekzjf3ks,7743
 ominfra/supervisor/exceptions.py,sha256=jq8Md--zmAHri1BB2XeDPFcTur81IRwArOcZoP7-6W0,746
 ominfra/supervisor/poller.py,sha256=oqNEA7i2XXtERBv552sr29a6mlogmosWjeGOZSul5Kg,7273
 ominfra/supervisor/process.py,sha256=phucIv2a-LHXypY3kJ9fCsKEki9G9XdiJtXCVMgGcZI,31291
 ominfra/supervisor/states.py,sha256=JMxXYTZhJkMNQZ2tTV6wId7wrvnWgiZteskACprKskM,1374
-ominfra/supervisor/supervisor.py,sha256=VAClZWVrZzZ6P0i6TIEKcyzI7WwWl5LRbnXngA4HPH4,13801
+ominfra/supervisor/supervisor.py,sha256=GnG8SAhJ5UUay7cM3oBJYb4TmIUaQUdWWqnhANpalPw,13811
 ominfra/supervisor/types.py,sha256=ec62QG0CDJc0XNxCnf3lXxhsxrr4CCScLPI-1SpQjlc,1141
 ominfra/tailscale/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ominfra/tailscale/cli.py,sha256=Ltg6RVFsMLLPjLzoGwM6sxjmwjEVEYHAdrqmCc4N1HM,3174
 ominfra/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ominfra/tools/listresources.py,sha256=L4t5rszm9ulcdWyr7n48_R9d5Etg4S2a4WQhlbHDtnQ,6106
-ominfra-0.0.0.dev77.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-ominfra-0.0.0.dev77.dist-info/METADATA,sha256=F3x-psxct-590MisOe_YAKVshGjbDwiVnPHTs8CVYpw,799
-ominfra-0.0.0.dev77.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-ominfra-0.0.0.dev77.dist-info/entry_points.txt,sha256=kgecQ2MgGrM9qK744BoKS3tMesaC3yjLnl9pa5CRczg,37
-ominfra-0.0.0.dev77.dist-info/top_level.txt,sha256=E-b2OHkk_AOBLXHYZQ2EOFKl-_6uOGd8EjeG-Zy6h_w,8
-ominfra-0.0.0.dev77.dist-info/RECORD,,
+ominfra-0.0.0.dev79.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+ominfra-0.0.0.dev79.dist-info/METADATA,sha256=qHE4z-IH2J8uJL_TOTh54TNP6YYBUHZKzbci30YOnb8,799
+ominfra-0.0.0.dev79.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+ominfra-0.0.0.dev79.dist-info/entry_points.txt,sha256=kgecQ2MgGrM9qK744BoKS3tMesaC3yjLnl9pa5CRczg,37
+ominfra-0.0.0.dev79.dist-info/top_level.txt,sha256=E-b2OHkk_AOBLXHYZQ2EOFKl-_6uOGd8EjeG-Zy6h_w,8
+ominfra-0.0.0.dev79.dist-info/RECORD,,