ominfra 0.0.0.dev76__py3-none-any.whl → 0.0.0.dev78__py3-none-any.whl

ominfra/clouds/aws/auth.py

@@ -1,3 +1,5 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
 """
 https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html
 
@@ -6,6 +8,7 @@ TODO:
   - boto / s3transfer upload_fileobj doesn't stream either lol - eagerly calcs Content-MD5
  - sts tokens
  - !! fix canonical_qs - sort params
+ - secrets
 """
 import dataclasses as dc
 import datetime
@@ -14,123 +17,125 @@ import hmac
 import typing as ta
 import urllib.parse
 
-from omlish import check
-from omlish import lang
+from omlish.lite.check import check_equal
+from omlish.lite.check import check_non_empty_str
+from omlish.lite.check import check_not_isinstance
 
 
 ##
 
 
-HttpMap: ta.TypeAlias = ta.Mapping[str, ta.Sequence[str]]
-
-
-def make_http_map(*kvs: tuple[str, str]) -> HttpMap:
-    out: dict[str, list[str]] = {}
-    for k, v in kvs:
-        out.setdefault(k, []).append(v)
-    return out
-
-
-#
-
-@dc.dataclass(frozen=True)
-class Credentials:
-    access_key: str
-    secret_key: str = dc.field(repr=False)
-
+class AwsSigner:
+    def __init__(
+            self,
+            creds: 'AwsSigner.Credentials',
+            region_name: str,
+            service_name: str,
+    ) -> None:
+        super().__init__()
+        self._creds = creds
+        self._region_name = region_name
+        self._service_name = service_name
 
-@dc.dataclass(frozen=True)
-class Request:
-    method: str
-    url: str
-    headers: HttpMap = dc.field(default_factory=dict)
-    payload: bytes = b''
+    #
 
+    @dc.dataclass(frozen=True)
+    class Credentials:
+        access_key: str
+        secret_key: str = dc.field(repr=False)
 
-##
+    @dc.dataclass(frozen=True)
+    class Request:
+        method: str
+        url: str
+        headers: ta.Mapping[str, ta.Sequence[str]] = dc.field(default_factory=dict)
+        payload: bytes = b''
 
+    #
 
-def _host_from_url(url: str) -> str:
-    url_parts = urllib.parse.urlsplit(url)
-    host = check.non_empty_str(url_parts.hostname)
-    default_ports = {
-        'http': 80,
-        'https': 443,
-    }
-    if url_parts.port is not None:
-        if url_parts.port != default_ports.get(url_parts.scheme):
-            host = '%s:%d' % (host, url_parts.port)
-    return host
+    ISO8601 = '%Y%m%dT%H%M%SZ'
 
+    #
 
-def _as_bytes(data: str | bytes) -> bytes:
-    return data if isinstance(data, bytes) else data.encode('utf-8')
+    @staticmethod
+    def _host_from_url(url: str) -> str:
+        url_parts = urllib.parse.urlsplit(url)
+        host = check_non_empty_str(url_parts.hostname)
+        default_ports = {
+            'http': 80,
+            'https': 443,
+        }
+        if url_parts.port is not None:
+            if url_parts.port != default_ports.get(url_parts.scheme):
+                host = '%s:%d' % (host, url_parts.port)
+        return host
 
+    @staticmethod
+    def _lower_case_http_map(d: ta.Mapping[str, ta.Sequence[str]]) -> ta.Mapping[str, ta.Sequence[str]]:
+        o: ta.Dict[str, ta.List[str]] = {}
+        for k, vs in d.items():
+            o.setdefault(k.lower(), []).extend(check_not_isinstance(vs, str))
+        return o
 
-def _sha256(data: str | bytes) -> str:
-    return hashlib.sha256(_as_bytes(data)).hexdigest()
+    #
 
+    @staticmethod
+    def _as_bytes(data: ta.Union[str, bytes]) -> bytes:
+        return data if isinstance(data, bytes) else data.encode('utf-8')
 
-def _sha256_sign(key: bytes, msg: str | bytes) -> bytes:
-    return hmac.new(key, _as_bytes(msg), hashlib.sha256).digest()
+    @staticmethod
+    def _sha256(data: ta.Union[str, bytes]) -> str:
+        return hashlib.sha256(AwsSigner._as_bytes(data)).hexdigest()
 
+    @staticmethod
+    def _sha256_sign(key: bytes, msg: ta.Union[str, bytes]) -> bytes:
+        return hmac.new(key, AwsSigner._as_bytes(msg), hashlib.sha256).digest()
 
-def _sha256_sign_hex(key: bytes, msg: str | bytes) -> str:
-    return hmac.new(key, _as_bytes(msg), hashlib.sha256).hexdigest()
+    @staticmethod
+    def _sha256_sign_hex(key: bytes, msg: ta.Union[str, bytes]) -> str:
+        return hmac.new(key, AwsSigner._as_bytes(msg), hashlib.sha256).hexdigest()
 
+    _EMPTY_SHA256: str
 
-_EMPTY_SHA256 = _sha256(b'')
+    #
 
-_ISO8601 = '%Y%m%dT%H%M%SZ'
+    _SIGNED_HEADERS_BLACKLIST = frozenset([
+        'authorization',
+        'expect',
+        'user-agent',
+        'x-amzn-trace-id',
+    ])
 
-_SIGNED_HEADERS_BLACKLIST = frozenset([
-    'expect',
-    'user-agent',
-    'x-amzn-trace-id',
-])
+    def _validate_request(self, req: Request) -> None:
+        check_non_empty_str(req.method)
+        check_equal(req.method.upper(), req.method)
+        for k, vs in req.headers.items():
+            check_equal(k.strip(), k)
+            for v in vs:
+                check_equal(v.strip(), v)
 
 
-def _lower_case_http_map(d: HttpMap) -> HttpMap:
-    o: dict[str, list[str]] = {}
-    for k, vs in d.items():
-        o.setdefault(k.lower(), []).extend(vs)
-    return o
+AwsSigner._EMPTY_SHA256 = AwsSigner._sha256(b'')  # noqa
 
 
-class V4AwsSigner:
-    def __init__(
-            self,
-            creds: Credentials,
-            region_name: str,
-            service_name: str,
-    ) -> None:
-        super().__init__()
-        self._creds = creds
-        self._region_name = region_name
-        self._service_name = service_name
+##
 
-    def _validate_request(self, req: Request) -> None:
-        check.non_empty_str(req.method)
-        check.equal(req.method.upper(), req.method)
-        for k, vs in req.headers.items():
-            check.equal(k.strip(), k)
-            for v in vs:
-                check.equal(v.strip(), v)
 
+class V4AwsSigner(AwsSigner):
     def sign(
             self,
-            req: Request,
+            req: AwsSigner.Request,
             *,
             sign_payload: bool = False,
-            utcnow: datetime.datetime | None = None,
-    ) -> HttpMap:
+            utcnow: ta.Optional[datetime.datetime] = None,
+    ) -> ta.Mapping[str, ta.Sequence[str]]:
         self._validate_request(req)
 
         #
 
         if utcnow is None:
-            utcnow = lang.utcnow()
-        req_dt = utcnow.strftime(_ISO8601)
+            utcnow = datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+        req_dt = utcnow.strftime(self.ISO8601)
 
         #
 
@@ -140,18 +145,18 @@ class V4AwsSigner:
 
         #
 
-        headers_to_sign = {
-            k: v
-            for k, v in _lower_case_http_map(req.headers).items()
-            if k not in _SIGNED_HEADERS_BLACKLIST
+        headers_to_sign: ta.Dict[str, ta.List[str]] = {
+            k: list(v)
+            for k, v in self._lower_case_http_map(req.headers).items()
+            if k not in self._SIGNED_HEADERS_BLACKLIST
         }
 
         if 'host' not in headers_to_sign:
-            headers_to_sign['host'] = [_host_from_url(req.url)]
+            headers_to_sign['host'] = [self._host_from_url(req.url)]
 
         headers_to_sign['x-amz-date'] = [req_dt]
 
-        hashed_payload = _sha256(req.payload) if req.payload else _EMPTY_SHA256
+        hashed_payload = self._sha256(req.payload) if req.payload else self._EMPTY_SHA256
         if sign_payload:
             headers_to_sign['x-amz-content-sha256'] = [hashed_payload]
 
@@ -183,7 +188,7 @@ class V4AwsSigner:
             'aws4_request',
         ]
         scope = '/'.join(scope_parts)
-        hashed_canon_req = _sha256(canon_req)
+        hashed_canon_req = self._sha256(canon_req)
         string_to_sign = '\n'.join([
             algorithm,
             req_dt,
@@ -194,11 +199,11 @@ class V4AwsSigner:
         #
 
         key = self._creds.secret_key
-        key_date = _sha256_sign(f'AWS4{key}'.encode('utf-8'), req_dt[:8])  # noqa
-        key_region = _sha256_sign(key_date, self._region_name)
-        key_service = _sha256_sign(key_region, self._service_name)
-        key_signing = _sha256_sign(key_service, 'aws4_request')
-        sig = _sha256_sign_hex(key_signing, string_to_sign)
+        key_date = self._sha256_sign(f'AWS4{key}'.encode('utf-8'), req_dt[:8])  # noqa
+        key_region = self._sha256_sign(key_date, self._region_name)
+        key_service = self._sha256_sign(key_region, self._service_name)
+        key_signing = self._sha256_sign(key_service, 'aws4_request')
+        sig = self._sha256_sign_hex(key_signing, string_to_sign)
 
         #
 

ominfra/clouds/aws/dataclasses.py

@@ -0,0 +1,149 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import collections.abc
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.reflect import get_optional_alias_arg
+from omlish.lite.reflect import is_generic_alias
+from omlish.lite.reflect import is_optional_alias
+from omlish.lite.strings import camel_case
+
+
+class AwsDataclass:
+    class Raw(dict):
+        pass
+
+    #
+
+    _aws_meta: ta.ClassVar[ta.Optional['AwsDataclassMeta']] = None
+
+    @classmethod
+    def _get_aws_meta(cls) -> 'AwsDataclassMeta':
+        try:
+            return cls.__dict__['_aws_meta']
+        except KeyError:
+            pass
+        ret = cls._aws_meta = AwsDataclassMeta(cls)
+        return ret
+
+    #
+
+    def to_aws(self) -> ta.Mapping[str, ta.Any]:
+        return self._get_aws_meta().converters().d2a(self)
+
+    @classmethod
+    def from_aws(cls, v: ta.Mapping[str, ta.Any]) -> 'AwsDataclass':
+        return cls._get_aws_meta().converters().a2d(v)
+
+
+@dc.dataclass(frozen=True)
+class AwsDataclassMeta:
+    cls: ta.Type['AwsDataclass']
+
+    #
+
+    class Field(ta.NamedTuple):
+        d_name: str
+        a_name: str
+        is_opt: bool
+        is_seq: bool
+        dc_cls: ta.Optional[ta.Type['AwsDataclass']]
+
+    @cached_nullary
+    def fields(self) -> ta.Sequence[Field]:
+        fs = []
+        for f in dc.fields(self.cls):  # type: ignore  # noqa
+            d_name = f.name
+            a_name = camel_case(d_name, lower=True)
+
+            is_opt = False
+            is_seq = False
+            dc_cls = None
+
+            c = f.type
+            if c is AwsDataclass.Raw:
+                continue
+
+            if is_optional_alias(c):
+                is_opt = True
+                c = get_optional_alias_arg(c)
+
+            if is_generic_alias(c) and ta.get_origin(c) is collections.abc.Sequence:
+                is_seq = True
+                [c] = ta.get_args(c)
+
+            if is_generic_alias(c):
+                raise TypeError(c)
+
+            if isinstance(c, type) and issubclass(c, AwsDataclass):
+                dc_cls = c
+
+            fs.append(AwsDataclassMeta.Field(
+                d_name=d_name,
+                a_name=a_name,
+                is_opt=is_opt,
+                is_seq=is_seq,
+                dc_cls=dc_cls,
+            ))
+
+        return fs
+
+    #
+
+    class Converters(ta.NamedTuple):
+        d2a: ta.Callable
+        a2d: ta.Callable
+
+    @cached_nullary
+    def converters(self) -> Converters:
+        for df in dc.fields(self.cls):  # type: ignore  # noqa
+            c = df.type
+
+            if is_optional_alias(c):
+                c = get_optional_alias_arg(c)
+
+            if c is AwsDataclass.Raw:
+                rf = df.name
+                break
+
+        else:
+            rf = None
+
+        fs = [
+            (f, f.dc_cls._get_aws_meta().converters() if f.dc_cls is not None else None)  # noqa
+            for f in self.fields()
+        ]
+
+        def d2a(o):
+            dct = {}
+            for f, cs in fs:
+                x = getattr(o, f.d_name)
+                if x is None:
+                    continue
+                if cs is not None:
+                    if f.is_seq:
+                        x = list(map(cs.d2a, x))
+                    else:
+                        x = cs.d2a(x)
+                dct[f.a_name] = x
+            return dct
+
+        def a2d(v):
+            dct = {}
+            for f, cs in fs:
+                x = v.get(f.a_name)
+                if x is None:
+                    continue
+                if cs is not None:
+                    if f.is_seq:
+                        x = list(map(cs.a2d, x))
+                    else:
+                        x = cs.a2d(x)
+                dct[f.d_name] = x
+            if rf is not None:
+                dct[rf] = self.cls.Raw(v)
+            return self.cls(**dct)
+
+        return AwsDataclassMeta.Converters(d2a, a2d)

ominfra/clouds/aws/journald2aws/journald.py

@@ -0,0 +1,67 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import typing as ta
+
+from omlish.lite.check import check_isinstance
+from omlish.lite.io import DelimitingBuffer
+from omlish.lite.logs import log
+
+
+@dc.dataclass(frozen=True)
+class JournalctlMessage:
+    raw: bytes
+    dct: ta.Optional[ta.Mapping[str, ta.Any]] = None
+    cursor: ta.Optional[str] = None
+    ts_us: ta.Optional[int] = None  # microseconds UTC
+
+
+class JournalctlMessageBuilder:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._buf = DelimitingBuffer(b'\n')
+
+    _cursor_field = '__CURSOR'
+    _timestamp_field = '_SOURCE_REALTIME_TIMESTAMP'
+
+    def _make_message(self, raw: bytes) -> JournalctlMessage:
+        dct = None
+        cursor = None
+        ts = None
+
+        try:
+            dct = json.loads(raw.decode('utf-8', 'replace'))
+        except Exception:  # noqa
+            log.exception('Failed to parse raw message: %r', raw)
+
+        else:
+            cursor = dct.get(self._cursor_field)
+
+            if tsv := dct.get(self._timestamp_field):
+                if isinstance(tsv, str):
+                    try:
+                        ts = int(tsv)
+                    except ValueError:
+                        try:
+                            ts = int(float(tsv))
+                        except ValueError:
+                            log.exception('Failed to parse timestamp: %r', tsv)
+                elif isinstance(tsv, (int, float)):
+                    ts = int(tsv)
+                else:
+                    log.exception('Invalid timestamp: %r', tsv)
+
+        return JournalctlMessage(
+            raw=raw,
+            dct=dct,
+            cursor=cursor,
+            ts_us=ts,
+        )
+
+    def feed(self, data: bytes) -> ta.Sequence[JournalctlMessage]:
+        ret: ta.List[JournalctlMessage] = []
+        for line in self._buf.feed(data):
+            ret.append(self._make_message(check_isinstance(line, bytes)))  # type: ignore
+        return ret

ominfra/clouds/aws/logs.py

@@ -0,0 +1,173 @@
+# @omlish-lite
+# ruff: noqa: UP007
+"""
+https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html :
+ - The maximum batch size is 1,048,576 bytes. This size is calculated as the sum of all event messages in UTF-8, plus 26
+   bytes for each log event.
+ - None of the log events in the batch can be more than 2 hours in the future.
+ - None of the log events in the batch can be more than 14 days in the past. Also, none of the log events can be from
+   earlier than the retention period of the log group.
+ - The log events in the batch must be in chronological order by their timestamp. The timestamp is the time that the
+   event occurred, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. (In AWS Tools for PowerShell
+   and the AWS SDK for .NET, the timestamp is specified in .NET format: yyyy-mm-ddThh:mm:ss. For example,
+   2017-09-15T13:45:30.)
+ - A batch of log events in a single request cannot span more than 24 hours. Otherwise, the operation fails.
+ - Each log event can be no larger than 256 KB.
+ - The maximum number of log events in a batch is 10,000.
+"""
+import dataclasses as dc
+import json
+import typing as ta
+
+from omlish.lite.check import check_non_empty_str
+from omlish.lite.check import check_single
+
+from .auth import AwsSigner
+from .auth import V4AwsSigner
+from .dataclasses import AwsDataclass
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class AwsLogEvent(AwsDataclass):
+    message: str
+    timestamp: int  # milliseconds UTC
+
+
+@dc.dataclass(frozen=True)
+class AwsPutLogEventsRequest(AwsDataclass):
+    log_group_name: str
+    log_stream_name: str
+    log_events: ta.Sequence[AwsLogEvent]
+    sequence_token: ta.Optional[str] = None
+
+
+@dc.dataclass(frozen=True)
+class AwsRejectedLogEventsInfo(AwsDataclass):
+    expired_log_event_end_index: ta.Optional[int] = None
+    too_new_log_event_start_index: ta.Optional[int] = None
+    too_old_log_event_end_index: ta.Optional[int] = None
+
+
+@dc.dataclass(frozen=True)
+class AwsPutLogEventsResponse(AwsDataclass):
+    next_sequence_token: ta.Optional[str] = None
+    rejected_log_events_info: ta.Optional[AwsRejectedLogEventsInfo] = None
+
+    raw: ta.Optional[AwsDataclass.Raw] = None
+
+
+##
+
+
+class AwsLogMessagePoster:
+    """
+    TODO:
+     - max_items
+     - max_bytes - manually build body
+     - flush_interval
+     - !! sort by timestamp
+    """
+
+    DEFAULT_URL = 'https://logs.{region_name}.amazonaws.com/'  # noqa
+
+    DEFAULT_SERVICE_NAME = 'logs'
+
+    DEFAULT_TARGET = 'Logs_20140328.PutLogEvents'
+    DEFAULT_CONTENT_TYPE = 'application/x-amz-json-1.1'
+
+    DEFAULT_HEADERS: ta.Mapping[str, str] = {
+        'X-Amz-Target': DEFAULT_TARGET,
+        'Content-Type': DEFAULT_CONTENT_TYPE,
+    }
+
+    def __init__(
+            self,
+            log_group_name: str,
+            log_stream_name: str,
+            region_name: str,
+            credentials: AwsSigner.Credentials,
+
+            url: ta.Optional[str] = None,
+            service_name: str = DEFAULT_SERVICE_NAME,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            extra_headers: ta.Optional[ta.Mapping[str, str]] = None,
+    ) -> None:
+        super().__init__()
+
+        self._log_group_name = check_non_empty_str(log_group_name)
+        self._log_stream_name = check_non_empty_str(log_stream_name)
+
+        if url is None:
+            url = self.DEFAULT_URL.format(region_name=region_name)
+        self._url = url
+
+        if headers is None:
+            headers = self.DEFAULT_HEADERS
+        if extra_headers is not None:
+            headers = {**headers, **extra_headers}
+        self._headers = {k: [v] for k, v in headers.items()}
+
+        self._signer = V4AwsSigner(
+            credentials,
+            region_name,
+            service_name,
+        )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Message:
+        message: str
+        ts_ms: int  # milliseconds UTC
+
+    @dc.dataclass(frozen=True)
+    class Post:
+        url: str
+        headers: ta.Mapping[str, str]
+        data: bytes
+
+    def feed(self, messages: ta.Sequence[Message]) -> ta.Sequence[Post]:
+        if not messages:
+            return []
+
+        payload = AwsPutLogEventsRequest(
+            log_group_name=self._log_group_name,
+            log_stream_name=self._log_stream_name,
+            log_events=[
+                AwsLogEvent(
+                    message=m.message,
+                    timestamp=m.ts_ms,
+                )
+                for m in messages
+            ],
+        )
+
+        body = json.dumps(
+            payload.to_aws(),
+            indent=None,
+            separators=(',', ':'),
+        ).encode('utf-8')
+
+        sig_req = V4AwsSigner.Request(
+            method='POST',
+            url=self._url,
+            headers=self._headers,
+            payload=body,
+        )
+
+        sig_headers = self._signer.sign(
+            sig_req,
+            sign_payload=False,
+        )
+        sig_req = dc.replace(sig_req, headers={**sig_req.headers, **sig_headers})
+
+        post = AwsLogMessagePoster.Post(
+            url=self._url,
+            headers={k: check_single(v) for k, v in sig_req.headers.items()},
+            data=sig_req.payload,
+        )
+
+        return [post]

ominfra/deploy/_executor.py

@@ -168,6 +168,23 @@ def check_state(v: bool, msg: str = 'Illegal state') -> None:
         raise ValueError(msg)
 
 
+def check_equal(l: T, r: T) -> T:
+    if l != r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_not_equal(l: T, r: T) -> T:
+    if l == r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_single(vs: ta.Iterable[T]) -> T:
+    [v] = vs
+    return v
+
+
 ########################################
 # ../../../../omlish/lite/json.py