ominfra 0.0.0.dev77__tar.gz → 0.0.0.dev79__tar.gz

{ominfra-0.0.0.dev77/ominfra.egg-info → ominfra-0.0.0.dev79}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ominfra
-Version: 0.0.0.dev77
+Version: 0.0.0.dev79
 Summary: ominfra
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omdev==0.0.0.dev77
-Requires-Dist: omlish==0.0.0.dev77
+Requires-Dist: omdev==0.0.0.dev79
+Requires-Dist: omlish==0.0.0.dev79
 Provides-Extra: all
 Requires-Dist: paramiko~=3.5; extra == "all"
 Requires-Dist: asyncssh~=2.17; python_version < "3.13" and extra == "all"

ominfra-0.0.0.dev79/ominfra/clouds/aws/auth.py

@@ -0,0 +1,228 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+"""
+https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html
+
+TODO:
+ - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
+  - boto / s3transfer upload_fileobj doesn't stream either lol - eagerly calcs Content-MD5
+ - sts tokens
+ - !! fix canonical_qs - sort params
+ - secrets
+"""
+import dataclasses as dc
+import datetime
+import hashlib
+import hmac
+import typing as ta
+import urllib.parse
+
+from omlish.lite.check import check_equal
+from omlish.lite.check import check_non_empty_str
+from omlish.lite.check import check_not_isinstance
+
+
+##
+
+
+class AwsSigner:
+    def __init__(
+            self,
+            creds: 'AwsSigner.Credentials',
+            region_name: str,
+            service_name: str,
+    ) -> None:
+        super().__init__()
+        self._creds = creds
+        self._region_name = region_name
+        self._service_name = service_name
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Credentials:
+        access_key: str
+        secret_key: str = dc.field(repr=False)
+
+    @dc.dataclass(frozen=True)
+    class Request:
+        method: str
+        url: str
+        headers: ta.Mapping[str, ta.Sequence[str]] = dc.field(default_factory=dict)
+        payload: bytes = b''
+
+    #
+
+    ISO8601 = '%Y%m%dT%H%M%SZ'
+
+    #
+
+    @staticmethod
+    def _host_from_url(url: str) -> str:
+        url_parts = urllib.parse.urlsplit(url)
+        host = check_non_empty_str(url_parts.hostname)
+        default_ports = {
+            'http': 80,
+            'https': 443,
+        }
+        if url_parts.port is not None:
+            if url_parts.port != default_ports.get(url_parts.scheme):
+                host = '%s:%d' % (host, url_parts.port)
+        return host
+
+    @staticmethod
+    def _lower_case_http_map(d: ta.Mapping[str, ta.Sequence[str]]) -> ta.Mapping[str, ta.Sequence[str]]:
+        o: ta.Dict[str, ta.List[str]] = {}
+        for k, vs in d.items():
+            o.setdefault(k.lower(), []).extend(check_not_isinstance(vs, str))
+        return o
+
+    #
+
+    @staticmethod
+    def _as_bytes(data: ta.Union[str, bytes]) -> bytes:
+        return data if isinstance(data, bytes) else data.encode('utf-8')
+
+    @staticmethod
+    def _sha256(data: ta.Union[str, bytes]) -> str:
+        return hashlib.sha256(AwsSigner._as_bytes(data)).hexdigest()
+
+    @staticmethod
+    def _sha256_sign(key: bytes, msg: ta.Union[str, bytes]) -> bytes:
+        return hmac.new(key, AwsSigner._as_bytes(msg), hashlib.sha256).digest()
+
+    @staticmethod
+    def _sha256_sign_hex(key: bytes, msg: ta.Union[str, bytes]) -> str:
+        return hmac.new(key, AwsSigner._as_bytes(msg), hashlib.sha256).hexdigest()
+
+    _EMPTY_SHA256: str
+
+    #
+
+    _SIGNED_HEADERS_BLACKLIST = frozenset([
+        'authorization',
+        'expect',
+        'user-agent',
+        'x-amzn-trace-id',
+    ])
+
+    def _validate_request(self, req: Request) -> None:
+        check_non_empty_str(req.method)
+        check_equal(req.method.upper(), req.method)
+        for k, vs in req.headers.items():
+            check_equal(k.strip(), k)
+            for v in vs:
+                check_equal(v.strip(), v)
+
+
+AwsSigner._EMPTY_SHA256 = AwsSigner._sha256(b'')  # noqa
+
+
+##
+
+
+class V4AwsSigner(AwsSigner):
+    def sign(
+            self,
+            req: AwsSigner.Request,
+            *,
+            sign_payload: bool = False,
+            utcnow: ta.Optional[datetime.datetime] = None,
+    ) -> ta.Mapping[str, ta.Sequence[str]]:
+        self._validate_request(req)
+
+        #
+
+        if utcnow is None:
+            utcnow = datetime.datetime.now(tz=datetime.timezone.utc)  # noqa
+        req_dt = utcnow.strftime(self.ISO8601)
+
+        #
+
+        parsed_url = urllib.parse.urlsplit(req.url)
+        canon_uri = parsed_url.path
+        canon_qs = parsed_url.query
+
+        #
+
+        headers_to_sign: ta.Dict[str, ta.List[str]] = {
+            k: list(v)
+            for k, v in self._lower_case_http_map(req.headers).items()
+            if k not in self._SIGNED_HEADERS_BLACKLIST
+        }
+
+        if 'host' not in headers_to_sign:
+            headers_to_sign['host'] = [self._host_from_url(req.url)]
+
+        headers_to_sign['x-amz-date'] = [req_dt]
+
+        hashed_payload = self._sha256(req.payload) if req.payload else self._EMPTY_SHA256
+        if sign_payload:
+            headers_to_sign['x-amz-content-sha256'] = [hashed_payload]
+
+        sorted_header_names = sorted(headers_to_sign)
+        canon_headers = ''.join([
+            ':'.join((k, ','.join(headers_to_sign[k]))) + '\n'
+            for k in sorted_header_names
+        ])
+        signed_headers = ';'.join(sorted_header_names)
+
+        #
+
+        canon_req = '\n'.join([
+            req.method,
+            canon_uri,
+            canon_qs,
+            canon_headers,
+            signed_headers,
+            hashed_payload,
+        ])
+
+        #
+
+        algorithm = 'AWS4-HMAC-SHA256'
+        scope_parts = [
+            req_dt[:8],
+            self._region_name,
+            self._service_name,
+            'aws4_request',
+        ]
+        scope = '/'.join(scope_parts)
+        hashed_canon_req = self._sha256(canon_req)
+        string_to_sign = '\n'.join([
+            algorithm,
+            req_dt,
+            scope,
+            hashed_canon_req,
+        ])
+
+        #
+
+        key = self._creds.secret_key
+        key_date = self._sha256_sign(f'AWS4{key}'.encode('utf-8'), req_dt[:8])  # noqa
+        key_region = self._sha256_sign(key_date, self._region_name)
+        key_service = self._sha256_sign(key_region, self._service_name)
+        key_signing = self._sha256_sign(key_service, 'aws4_request')
+        sig = self._sha256_sign_hex(key_signing, string_to_sign)
+
+        #
+
+        cred_scope = '/'.join([
+            self._creds.access_key,
+            *scope_parts,
+        ])
+        auth = f'{algorithm} ' + ', '.join([
+            f'Credential={cred_scope}',
+            f'SignedHeaders={signed_headers}',
+            f'Signature={sig}',
+        ])
+
+        #
+
+        out = {
+            'Authorization': [auth],
+            'X-Amz-Date': [req_dt],
+        }
+        if sign_payload:
+            out['X-Amz-Content-SHA256'] = [hashed_payload]
+        return out

ominfra-0.0.0.dev79/ominfra/clouds/aws/dataclasses.py

@@ -0,0 +1,149 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import collections.abc
+import dataclasses as dc
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.reflect import get_optional_alias_arg
+from omlish.lite.reflect import is_generic_alias
+from omlish.lite.reflect import is_optional_alias
+from omlish.lite.strings import camel_case
+
+
+class AwsDataclass:
+    class Raw(dict):
+        pass
+
+    #
+
+    _aws_meta: ta.ClassVar[ta.Optional['AwsDataclassMeta']] = None
+
+    @classmethod
+    def _get_aws_meta(cls) -> 'AwsDataclassMeta':
+        try:
+            return cls.__dict__['_aws_meta']
+        except KeyError:
+            pass
+        ret = cls._aws_meta = AwsDataclassMeta(cls)
+        return ret
+
+    #
+
+    def to_aws(self) -> ta.Mapping[str, ta.Any]:
+        return self._get_aws_meta().converters().d2a(self)
+
+    @classmethod
+    def from_aws(cls, v: ta.Mapping[str, ta.Any]) -> 'AwsDataclass':
+        return cls._get_aws_meta().converters().a2d(v)
+
+
+@dc.dataclass(frozen=True)
+class AwsDataclassMeta:
+    cls: ta.Type['AwsDataclass']
+
+    #
+
+    class Field(ta.NamedTuple):
+        d_name: str
+        a_name: str
+        is_opt: bool
+        is_seq: bool
+        dc_cls: ta.Optional[ta.Type['AwsDataclass']]
+
+    @cached_nullary
+    def fields(self) -> ta.Sequence[Field]:
+        fs = []
+        for f in dc.fields(self.cls):  # type: ignore  # noqa
+            d_name = f.name
+            a_name = camel_case(d_name, lower=True)
+
+            is_opt = False
+            is_seq = False
+            dc_cls = None
+
+            c = f.type
+            if c is AwsDataclass.Raw:
+                continue
+
+            if is_optional_alias(c):
+                is_opt = True
+                c = get_optional_alias_arg(c)
+
+            if is_generic_alias(c) and ta.get_origin(c) is collections.abc.Sequence:
+                is_seq = True
+                [c] = ta.get_args(c)
+
+            if is_generic_alias(c):
+                raise TypeError(c)
+
+            if isinstance(c, type) and issubclass(c, AwsDataclass):
+                dc_cls = c
+
+            fs.append(AwsDataclassMeta.Field(
+                d_name=d_name,
+                a_name=a_name,
+                is_opt=is_opt,
+                is_seq=is_seq,
+                dc_cls=dc_cls,
+            ))
+
+        return fs
+
+    #
+
+    class Converters(ta.NamedTuple):
+        d2a: ta.Callable
+        a2d: ta.Callable
+
+    @cached_nullary
+    def converters(self) -> Converters:
+        for df in dc.fields(self.cls):  # type: ignore  # noqa
+            c = df.type
+
+            if is_optional_alias(c):
+                c = get_optional_alias_arg(c)
+
+            if c is AwsDataclass.Raw:
+                rf = df.name
+                break
+
+        else:
+            rf = None
+
+        fs = [
+            (f, f.dc_cls._get_aws_meta().converters() if f.dc_cls is not None else None)  # noqa
+            for f in self.fields()
+        ]
+
+        def d2a(o):
+            dct = {}
+            for f, cs in fs:
+                x = getattr(o, f.d_name)
+                if x is None:
+                    continue
+                if cs is not None:
+                    if f.is_seq:
+                        x = list(map(cs.d2a, x))
+                    else:
+                        x = cs.d2a(x)
+                dct[f.a_name] = x
+            return dct
+
+        def a2d(v):
+            dct = {}
+            for f, cs in fs:
+                x = v.get(f.a_name)
+                if x is None:
+                    continue
+                if cs is not None:
+                    if f.is_seq:
+                        x = list(map(cs.a2d, x))
+                    else:
+                        x = cs.a2d(x)
+                dct[f.d_name] = x
+            if rf is not None:
+                dct[rf] = self.cls.Raw(v)
+            return self.cls(**dct)
+
+        return AwsDataclassMeta.Converters(d2a, a2d)

ominfra-0.0.0.dev79/ominfra/clouds/aws/journald2aws/journald.py

@@ -0,0 +1,67 @@
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import dataclasses as dc
+import json
+import typing as ta
+
+from omlish.lite.check import check_isinstance
+from omlish.lite.io import DelimitingBuffer
+from omlish.lite.logs import log
+
+
+@dc.dataclass(frozen=True)
+class JournalctlMessage:
+    raw: bytes
+    dct: ta.Optional[ta.Mapping[str, ta.Any]] = None
+    cursor: ta.Optional[str] = None
+    ts_us: ta.Optional[int] = None  # microseconds UTC
+
+
+class JournalctlMessageBuilder:
+    def __init__(self) -> None:
+        super().__init__()
+
+        self._buf = DelimitingBuffer(b'\n')
+
+    _cursor_field = '__CURSOR'
+    _timestamp_field = '_SOURCE_REALTIME_TIMESTAMP'
+
+    def _make_message(self, raw: bytes) -> JournalctlMessage:
+        dct = None
+        cursor = None
+        ts = None
+
+        try:
+            dct = json.loads(raw.decode('utf-8', 'replace'))
+        except Exception:  # noqa
+            log.exception('Failed to parse raw message: %r', raw)
+
+        else:
+            cursor = dct.get(self._cursor_field)
+
+            if tsv := dct.get(self._timestamp_field):
+                if isinstance(tsv, str):
+                    try:
+                        ts = int(tsv)
+                    except ValueError:
+                        try:
+                            ts = int(float(tsv))
+                        except ValueError:
+                            log.exception('Failed to parse timestamp: %r', tsv)
+                elif isinstance(tsv, (int, float)):
+                    ts = int(tsv)
+                else:
+                    log.exception('Invalid timestamp: %r', tsv)
+
+        return JournalctlMessage(
+            raw=raw,
+            dct=dct,
+            cursor=cursor,
+            ts_us=ts,
+        )
+
+    def feed(self, data: bytes) -> ta.Sequence[JournalctlMessage]:
+        ret: ta.List[JournalctlMessage] = []
+        for line in self._buf.feed(data):
+            ret.append(self._make_message(check_isinstance(line, bytes)))  # type: ignore
+        return ret

ominfra-0.0.0.dev79/ominfra/clouds/aws/logs.py

@@ -0,0 +1,173 @@
+# @omlish-lite
+# ruff: noqa: UP007
+"""
+https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html :
+ - The maximum batch size is 1,048,576 bytes. This size is calculated as the sum of all event messages in UTF-8, plus 26
+   bytes for each log event.
+ - None of the log events in the batch can be more than 2 hours in the future.
+ - None of the log events in the batch can be more than 14 days in the past. Also, none of the log events can be from
+   earlier than the retention period of the log group.
+ - The log events in the batch must be in chronological order by their timestamp. The timestamp is the time that the
+   event occurred, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. (In AWS Tools for PowerShell
+   and the AWS SDK for .NET, the timestamp is specified in .NET format: yyyy-mm-ddThh:mm:ss. For example,
+   2017-09-15T13:45:30.)
+ - A batch of log events in a single request cannot span more than 24 hours. Otherwise, the operation fails.
+ - Each log event can be no larger than 256 KB.
+ - The maximum number of log events in a batch is 10,000.
+"""
+import dataclasses as dc
+import json
+import typing as ta
+
+from omlish.lite.check import check_non_empty_str
+from omlish.lite.check import check_single
+
+from .auth import AwsSigner
+from .auth import V4AwsSigner
+from .dataclasses import AwsDataclass
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class AwsLogEvent(AwsDataclass):
+    message: str
+    timestamp: int  # milliseconds UTC
+
+
+@dc.dataclass(frozen=True)
+class AwsPutLogEventsRequest(AwsDataclass):
+    log_group_name: str
+    log_stream_name: str
+    log_events: ta.Sequence[AwsLogEvent]
+    sequence_token: ta.Optional[str] = None
+
+
+@dc.dataclass(frozen=True)
+class AwsRejectedLogEventsInfo(AwsDataclass):
+    expired_log_event_end_index: ta.Optional[int] = None
+    too_new_log_event_start_index: ta.Optional[int] = None
+    too_old_log_event_end_index: ta.Optional[int] = None
+
+
+@dc.dataclass(frozen=True)
+class AwsPutLogEventsResponse(AwsDataclass):
+    next_sequence_token: ta.Optional[str] = None
+    rejected_log_events_info: ta.Optional[AwsRejectedLogEventsInfo] = None
+
+    raw: ta.Optional[AwsDataclass.Raw] = None
+
+
+##
+
+
+class AwsLogMessagePoster:
+    """
+    TODO:
+     - max_items
+     - max_bytes - manually build body
+     - flush_interval
+     - !! sort by timestamp
+    """
+
+    DEFAULT_URL = 'https://logs.{region_name}.amazonaws.com/'  # noqa
+
+    DEFAULT_SERVICE_NAME = 'logs'
+
+    DEFAULT_TARGET = 'Logs_20140328.PutLogEvents'
+    DEFAULT_CONTENT_TYPE = 'application/x-amz-json-1.1'
+
+    DEFAULT_HEADERS: ta.Mapping[str, str] = {
+        'X-Amz-Target': DEFAULT_TARGET,
+        'Content-Type': DEFAULT_CONTENT_TYPE,
+    }
+
+    def __init__(
+            self,
+            log_group_name: str,
+            log_stream_name: str,
+            region_name: str,
+            credentials: AwsSigner.Credentials,
+
+            url: ta.Optional[str] = None,
+            service_name: str = DEFAULT_SERVICE_NAME,
+            headers: ta.Optional[ta.Mapping[str, str]] = None,
+            extra_headers: ta.Optional[ta.Mapping[str, str]] = None,
+    ) -> None:
+        super().__init__()
+
+        self._log_group_name = check_non_empty_str(log_group_name)
+        self._log_stream_name = check_non_empty_str(log_stream_name)
+
+        if url is None:
+            url = self.DEFAULT_URL.format(region_name=region_name)
+        self._url = url
+
+        if headers is None:
+            headers = self.DEFAULT_HEADERS
+        if extra_headers is not None:
+            headers = {**headers, **extra_headers}
+        self._headers = {k: [v] for k, v in headers.items()}
+
+        self._signer = V4AwsSigner(
+            credentials,
+            region_name,
+            service_name,
+        )
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Message:
+        message: str
+        ts_ms: int  # milliseconds UTC
+
+    @dc.dataclass(frozen=True)
+    class Post:
+        url: str
+        headers: ta.Mapping[str, str]
+        data: bytes
+
+    def feed(self, messages: ta.Sequence[Message]) -> ta.Sequence[Post]:
+        if not messages:
+            return []
+
+        payload = AwsPutLogEventsRequest(
+            log_group_name=self._log_group_name,
+            log_stream_name=self._log_stream_name,
+            log_events=[
+                AwsLogEvent(
+                    message=m.message,
+                    timestamp=m.ts_ms,
+                )
+                for m in messages
+            ],
+        )
+
+        body = json.dumps(
+            payload.to_aws(),
+            indent=None,
+            separators=(',', ':'),
+        ).encode('utf-8')
+
+        sig_req = V4AwsSigner.Request(
+            method='POST',
+            url=self._url,
+            headers=self._headers,
+            payload=body,
+        )
+
+        sig_headers = self._signer.sign(
+            sig_req,
+            sign_payload=False,
+        )
+        sig_req = dc.replace(sig_req, headers={**sig_req.headers, **sig_headers})
+
+        post = AwsLogMessagePoster.Post(
+            url=self._url,
+            headers={k: check_single(v) for k, v in sig_req.headers.items()},
+            data=sig_req.payload,
+        )
+
+        return [post]

{ominfra-0.0.0.dev77 → ominfra-0.0.0.dev79}/ominfra/deploy/_executor.py

@@ -168,6 +168,23 @@ def check_state(v: bool, msg: str = 'Illegal state') -> None:
         raise ValueError(msg)
 
 
+def check_equal(l: T, r: T) -> T:
+    if l != r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_not_equal(l: T, r: T) -> T:
+    if l == r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_single(vs: ta.Iterable[T]) -> T:
+    [v] = vs
+    return v
+
+
 ########################################
 # ../../../../omlish/lite/json.py
 

{ominfra-0.0.0.dev77 → ominfra-0.0.0.dev79}/ominfra/pyremote/_runcommands.py

@@ -251,6 +251,23 @@ def check_state(v: bool, msg: str = 'Illegal state') -> None:
         raise ValueError(msg)
 
 
+def check_equal(l: T, r: T) -> T:
+    if l != r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_not_equal(l: T, r: T) -> T:
+    if l == r:
+        raise ValueError(l, r)
+    return l
+
+
+def check_single(vs: ta.Iterable[T]) -> T:
+    [v] = vs
+    return v
+
+
 ########################################
 # ../../../omlish/lite/json.py