aws-durable-execution-sdk-python-telemetry-exporter 0.0.1__py3-none-any.whl

aws_durable_execution_sdk_python_telemetry_exporter/exporters/opensearch_exporter.py

@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+"""OpenSearch (Index API) Durable Execution Telemetry exporter."""
+
+from __future__ import annotations
+
+import base64
+from enum import StrEnum
+from typing import Any, Literal
+from urllib.parse import quote
+
+from aws_durable_execution_sdk_python_telemetry_exporter.exporters._common import (
+    compact_dumps,
+    http_send,
+)
+
+
+class OpenSearchAuth(StrEnum):
+    """Authentication method for the index request."""
+
+    # IAM credentials, SigV4-signed (Amazon OpenSearch Service)
+    SIGV4 = "sigv4"
+    # username / password (self-managed, or a domain with basic auth)
+    BASIC = "basic"
+
+
+# Accepted string inputs, kept in lockstep with the enum values above.
+OpenSearchAuthInput = Literal["sigv4", "basic"]
+
+# Same character set that browsers leave unescaped in URI components.
+_DOC_ID_SAFE = "-_.!~*'()"
+
+
+class OpenSearchExporter:
+    """Indexes each record as one document, keyed by ``executionArn``.
+
+    A later export for the same execution overwrites the document. Supports
+    SigV4 (IAM) and basic authentication; no OpenSearch client library needed.
+    """
+
+    def __init__(
+        self,
+        endpoint: str,
+        region: str,
+        index_name: str = "durable-execution-telemetry",
+        auth: OpenSearchAuth | OpenSearchAuthInput = OpenSearchAuth.SIGV4,
+        username: str | None = None,
+        password: str | None = None,
+        max_record_size_bytes: int | None = None,
+    ) -> None:
+        self.endpoint = endpoint[:-1] if endpoint.endswith("/") else endpoint
+        self.index_name = index_name
+        self.region = region
+        self.auth = OpenSearchAuth(auth)
+        if self.auth == OpenSearchAuth.BASIC and (username is None or password is None):
+            msg = "OpenSearchExporter: auth='basic' requires username and password."
+            raise ValueError(msg)
+        self.username = username
+        self.password = password
+        self.max_record_size_bytes: int | None = (
+            10_000_000 if max_record_size_bytes is None else max_record_size_bytes
+        )
+        self._credentials: Any = None
+
+    def render(self, record: dict[str, Any]) -> dict[str, Any]:
+        return record
+
+    def export(self, record: dict[str, Any]) -> None:
+        doc_id = quote(record["executionArn"], safe=_DOC_ID_SAFE)
+        url = f"{self.endpoint}/{self.index_name}/_doc/{doc_id}"
+        body = compact_dumps(record).encode("utf-8")
+        headers: dict[str, str] = {"Content-Type": "application/json"}
+
+        if self.auth == OpenSearchAuth.BASIC:
+            token = base64.b64encode(
+                f"{self.username}:{self.password}".encode()
+            ).decode("ascii")
+            headers["Authorization"] = f"Basic {token}"
+        else:
+            # The signed header set is sent as-is: it already carries
+            # content-type, host, date, token and authorization.
+            headers = self._sign(url, body, headers)
+
+        status, reason, detail = http_send("PUT", url, headers, body)
+        if not 200 <= status < 300:
+            msg = f"OpenSearch index failed: {status} {reason}"
+            if detail:
+                msg += f" — {detail[:500]}"
+            raise RuntimeError(msg)
+
+    def flush(self) -> None:
+        return None
+
+    def _sign(self, url: str, body: bytes, headers: dict[str, str]) -> dict[str, str]:
+        # deferred: botocore is provided by the Lambda runtime
+        from botocore.auth import SigV4Auth
+        from botocore.awsrequest import AWSRequest
+
+        if self._credentials is None:
+            import botocore.session
+
+            self._credentials = botocore.session.get_session().get_credentials()
+            if self._credentials is None:
+                msg = "OpenSearchExporter: no AWS credentials found for SigV4 signing."
+                raise RuntimeError(msg)
+        request = AWSRequest(method="PUT", url=url, data=body, headers=headers)
+        SigV4Auth(self._credentials, "es", self.region).add_auth(request)
+        return dict(request.headers.items())

aws_durable_execution_sdk_python_telemetry_exporter/exporters/otel_exporter.py

@@ -0,0 +1,151 @@
+# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+"""OpenTelemetry (OTLP/HTTP logs) Durable Execution Telemetry exporter."""
+
+from __future__ import annotations
+
+import datetime
+from enum import StrEnum
+from typing import Any, Literal
+
+from aws_durable_execution_sdk_python_telemetry_exporter.exporters._common import (
+    compact_dumps,
+    http_send,
+    parse_iso_datetime,
+)
+from aws_durable_execution_sdk_python_telemetry_exporter.operations_index import (
+    OperationsFormat,
+    OperationsFormatInput,
+    apply_operations_format,
+)
+
+
+class OTelProtocol(StrEnum):
+    """OTLP transport encoding."""
+
+    HTTP_JSON = "http/json"
+    HTTP_PROTOBUF = "http/protobuf"
+
+
+# Accepted string inputs, kept in lockstep with the enum values above.
+OTelProtocolInput = Literal["http/json", "http/protobuf"]
+
+_SCOPE_NAME = "aws-durable-execution-sdk-python-telemetry-exporter"
+_SEVERITY = {"FAILED": 17, "RUNNING": 9, "SUCCEEDED": 9}  # ERROR / INFO / INFO
+_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.UTC)
+
+
+def _kv(key: str, value: str | int) -> dict[str, Any]:
+    if isinstance(value, int):
+        return {"key": key, "value": {"intValue": str(value)}}
+    return {"key": key, "value": {"stringValue": value}}
+
+
+def _to_nano(iso: str) -> str:
+    # Keep the record's full microsecond precision (integer math, no float).
+    delta = parse_iso_datetime(iso) - _EPOCH
+    seconds = delta.days * 86_400 + delta.seconds
+    return str(seconds * 1_000_000_000 + delta.microseconds * 1_000)
+
+
+class OTelExporter:
+    """Posts each record as one OTLP log record to an OTLP/HTTP endpoint.
+
+    Record identity fields become resource and log attributes; the record
+    itself, rendered per ``operations_format``, is the log body. Only
+    ``http/json`` is supported; ``http/protobuf`` raises at construction.
+    """
+
+    def __init__(
+        self,
+        endpoint: str,
+        headers: dict[str, str] | None = None,
+        protocol: OTelProtocol | OTelProtocolInput = OTelProtocol.HTTP_JSON,
+        operations_format: OperationsFormat | OperationsFormatInput = (
+            OperationsFormat.ARRAY
+        ),
+        max_record_size_bytes: int | None = None,
+    ) -> None:
+        if OTelProtocol(protocol) == OTelProtocol.HTTP_PROTOBUF:
+            msg = "OTelExporter: http/protobuf is not yet supported. Use http/json."
+            raise ValueError(msg)
+        self.endpoint = endpoint
+        self.headers = dict(headers or {})
+        self.operations_format = OperationsFormat(operations_format)
+        self.max_record_size_bytes: int | None = (
+            1_000_000 if max_record_size_bytes is None else max_record_size_bytes
+        )
+
+    def render(self, record: dict[str, Any]) -> dict[str, Any]:
+        # The whole request is measured: the record sits inside body.stringValue,
+        # so the size limit covers both the record and the OTLP envelope.
+        return self._build_payload(record)
+
+    def export(self, record: dict[str, Any]) -> None:
+        body = compact_dumps(self._build_payload(record)).encode("utf-8")
+        headers = {"Content-Type": "application/json", **self.headers}
+        status, reason, _ = http_send("POST", self.endpoint, headers, body)
+        if not 200 <= status < 300:
+            msg = f"OTelExporter: OTLP endpoint returned {status} {reason}"
+            raise RuntimeError(msg)
+
+    def flush(self) -> None:
+        return None
+
+    def _build_payload(self, record: dict[str, Any]) -> dict[str, Any]:
+        """Build an OTLP ``ExportLogsServiceRequest``."""
+        function_name = record.get("functionName", "")
+        status = record.get("status", "")
+        return {
+            "resourceLogs": [
+                {
+                    "resource": {
+                        "attributes": [
+                            _kv("service.name", function_name),
+                            _kv("cloud.region", record.get("region", "")),
+                            _kv("cloud.account.id", record.get("accountId", "")),
+                            _kv("faas.name", function_name),
+                            _kv("faas.version", record.get("functionQualifier", "")),
+                        ]
+                    },
+                    "scopeLogs": [
+                        {
+                            "scope": {
+                                "name": _SCOPE_NAME,
+                                "version": record.get("schemaVersion", ""),
+                            },
+                            "logRecords": [
+                                {
+                                    "timeUnixNano": _to_nano(record["emittedAt"]),
+                                    "severityNumber": _SEVERITY.get(status, 0),
+                                    "severityText": status,
+                                    "body": {
+                                        "stringValue": compact_dumps(
+                                            apply_operations_format(
+                                                record, self.operations_format
+                                            )
+                                        )
+                                    },
+                                    "attributes": [
+                                        _kv(
+                                            "workflow.execution_arn",
+                                            record["executionArn"],
+                                        ),
+                                        _kv(
+                                            "workflow.execution_name",
+                                            record.get("executionName") or "",
+                                        ),
+                                        _kv("workflow.status", status),
+                                        _kv(
+                                            "workflow.duration_ms",
+                                            record.get("durationMs") or 0,
+                                        ),
+                                    ],
+                                }
+                            ],
+                        }
+                    ],
+                }
+            ]
+        }

aws_durable_execution_sdk_python_telemetry_exporter/exporters/redshift_exporter.py

@@ -0,0 +1,155 @@
+# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+"""Redshift (Data API) Durable Execution Telemetry exporter."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from aws_durable_execution_sdk_python_telemetry_exporter.exporters._common import (
+    compact_dumps,
+    sql_identifier,
+)
+
+
+_COLUMNS = (
+    "execution_arn, execution_name, function_name, status, start_time, "
+    "end_time, duration_ms, record_json, emitted_at"
+)
+
+
+class RedshiftExporter:
+    """Upserts one row per execution through the Redshift Data API.
+
+    Works with Redshift Serverless (``workgroup_name``) and provisioned
+    clusters (``cluster_identifier``). Rows are merged on ``execution_arn`` and
+    the full record lands in the ``record_json`` SUPER column. The statement is
+    submitted and not awaited.
+    """
+
+    def __init__(
+        self,
+        database: str,
+        workgroup_name: str | None = None,
+        cluster_identifier: str | None = None,
+        db_user: str | None = None,
+        secret_arn: str | None = None,
+        table: str = "durable_execution_telemetry",
+        schema: str = "public",
+        region: str | None = None,
+        max_record_size_bytes: int | None = None,
+        client: Any = None,
+    ) -> None:
+        if bool(workgroup_name) == bool(cluster_identifier):
+            msg = (
+                "RedshiftExporter: provide exactly one of workgroup_name or "
+                "cluster_identifier."
+            )
+            raise ValueError(msg)
+        self.database = database
+        self.fq_table = f"{sql_identifier(schema)}.{sql_identifier(table)}"
+        self.workgroup_name = workgroup_name
+        self.cluster_identifier = cluster_identifier
+        self.db_user = db_user
+        self.secret_arn = secret_arn
+        self.max_record_size_bytes: int | None = (
+            1_000_000 if max_record_size_bytes is None else max_record_size_bytes
+        )
+        if client is not None:
+            self._client = client
+        else:
+            import boto3  # deferred: boto3 is provided by the Lambda runtime
+
+            self._client = (
+                boto3.client("redshift-data", region_name=region)
+                if region
+                else boto3.client("redshift-data")
+            )
+
+    def render(self, record: dict[str, Any]) -> dict[str, Any]:
+        return record
+
+    def export(self, record: dict[str, Any]) -> None:
+        parameters: list[dict[str, str]] = [
+            {"name": "execution_arn", "value": record["executionArn"]},
+            {"name": "function_name", "value": record["functionName"]},
+            {"name": "status", "value": record["status"]},
+            {"name": "start_time", "value": record["startTime"]},
+            {"name": "record_json", "value": compact_dumps(record)},
+            {"name": "emitted_at", "value": record["emittedAt"]},
+        ]
+
+        # The Data API takes neither NULL nor empty-string parameter values, so
+        # an absent nullable field becomes a typed NULL literal in the source
+        # projection instead of a bound parameter.
+        if record.get("endTime"):
+            parameters.append({"name": "end_time", "value": record["endTime"]})
+            end_time_sel = ":end_time::timestamptz"
+        else:
+            end_time_sel = "NULL::timestamptz"
+        if record.get("durationMs") is not None:
+            parameters.append(
+                {"name": "duration_ms", "value": str(record["durationMs"])}
+            )
+            duration_sel = ":duration_ms::bigint"
+        else:
+            duration_sel = "NULL::bigint"
+        if record.get("executionName"):
+            parameters.append(
+                {"name": "execution_name", "value": record["executionName"]}
+            )
+            exec_name_sel = ":execution_name::varchar"
+        else:
+            exec_name_sel = "NULL::varchar"
+
+        # The source row is a subquery and MERGE joins on a source column:
+        # Redshift rejects a MERGE whose join is on a parameter or constant
+        # (NestedLoop). Time columns are cast to timestamptz and record_json is
+        # JSON_PARSEd into the SUPER column.
+        sql = (
+            f"MERGE INTO {self.fq_table} USING (\n"
+            "      SELECT\n"
+            "        :execution_arn::varchar AS execution_arn,\n"
+            f"        {exec_name_sel} AS execution_name,\n"
+            "        :function_name::varchar AS function_name,\n"
+            "        :status::varchar AS status,\n"
+            "        :start_time::timestamptz AS start_time,\n"
+            f"        {end_time_sel} AS end_time,\n"
+            f"        {duration_sel} AS duration_ms,\n"
+            "        JSON_PARSE(:record_json) AS record_json,\n"
+            "        :emitted_at::timestamptz AS emitted_at\n"
+            "    ) AS src\n"
+            f"    ON {self.fq_table}.execution_arn = src.execution_arn\n"
+            "    WHEN MATCHED THEN UPDATE SET\n"
+            "      status = src.status,\n"
+            "      end_time = src.end_time,\n"
+            "      duration_ms = src.duration_ms,\n"
+            "      record_json = src.record_json,\n"
+            "      emitted_at = src.emitted_at\n"
+            "    WHEN NOT MATCHED THEN INSERT\n"
+            f"      ({_COLUMNS})\n"
+            "    VALUES\n"
+            "      (src.execution_arn, src.execution_name, src.function_name, "
+            "src.status, src.start_time, src.end_time, src.duration_ms, "
+            "src.record_json, src.emitted_at)"
+        )
+
+        kwargs: dict[str, Any] = {
+            "Database": self.database,
+            "Sql": sql,
+            "Parameters": parameters,
+        }
+        # Unset optional targets are omitted rather than passed as None.
+        if self.workgroup_name:
+            kwargs["WorkgroupName"] = self.workgroup_name
+        if self.cluster_identifier:
+            kwargs["ClusterIdentifier"] = self.cluster_identifier
+        if self.db_user:
+            kwargs["DbUser"] = self.db_user
+        if self.secret_arn:
+            kwargs["SecretArn"] = self.secret_arn
+        self._client.execute_statement(**kwargs)
+
+    def flush(self) -> None:
+        return None

aws_durable_execution_sdk_python_telemetry_exporter/exporters/s3_exporter.py

@@ -0,0 +1,105 @@
+# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+"""S3 Durable Execution Telemetry exporter."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Any, Literal
+
+from aws_durable_execution_sdk_python_telemetry_exporter.exporters._common import (
+    compact_dumps,
+    sanitize,
+)
+
+
+class S3Partitioning(StrEnum):
+    """S3 key partitioning scheme. The values match the JS S3 exporter's options."""
+
+    # year=YYYY/month=MM/day=DD/ derived from the record startTime (default)
+    DATE = "date"
+    # function=<functionName>/
+    FUNCTION_NAME = "function-name"
+    # no partition prefix
+    NONE = "none"
+
+
+# Accepted string inputs, kept in lockstep with the enum values above. The
+# constructor is typed as this ``Literal`` union (never bare ``str``) so a typoed
+# scheme fails a static type check, while ``S3Partitioning(partitioning)`` in
+# ``__init__`` normalizes any accepted value to the matching enum member and
+# raises ``ValueError`` for an invalid dynamic string.
+S3PartitioningInput = Literal["date", "function-name", "none"]
+
+
+class S3Exporter:
+    """Writes canonical ``operations``-array records to S3.
+
+    Port of the JS ``S3Exporter``. Each record is a JSON object keyed by
+    execution name, so updates to the same execution overwrite the same object.
+    Emits the lossless ``operations`` array (``OPERATIONS_ARRAY``).
+    """
+
+    def __init__(
+        self,
+        bucket: str,
+        prefix: str = "durable-execution-telemetry/",
+        partitioning: S3Partitioning | S3PartitioningInput = S3Partitioning.DATE,
+        region: str | None = None,
+        max_record_size_bytes: int | None = None,
+        client: Any = None,
+    ) -> None:
+        self.bucket = bucket
+        self.prefix = prefix
+        # ``S3Partitioning(x)`` is idempotent for members, accepts the exact
+        # JS-style strings, and raises ``ValueError`` for an unrecognized dynamic
+        # string so an invalid scheme fails at construction rather than silently
+        # falling through to no partitioning.
+        self.partitioning = S3Partitioning(partitioning)
+        self.max_record_size_bytes: int | None = (
+            5_000_000 if max_record_size_bytes is None else max_record_size_bytes
+        )
+        if client is not None:
+            self._client = client
+        else:
+            import boto3  # deferred: boto3 is provided by the Lambda runtime
+
+            self._client = (
+                boto3.client("s3", region_name=region) if region else boto3.client("s3")
+            )
+
+    def render(self, record: dict[str, Any]) -> dict[str, Any]:
+        return record
+
+    def export(self, record: dict[str, Any]) -> None:
+        key = self._build_key(record)
+        self._client.put_object(
+            Bucket=self.bucket,
+            Key=key,
+            Body=compact_dumps(record).encode("utf-8"),
+            ContentType="application/json",
+        )
+
+    def flush(self) -> None:
+        return None
+
+    def _build_key(self, record: dict[str, Any]) -> str:
+        file_name = (
+            sanitize(
+                record.get("executionName") or record.get("executionArn") or "record"
+            )
+            + ".json"
+        )
+        return f"{self.prefix}{self._partition(record)}{file_name}"
+
+    def _partition(self, record: dict[str, Any]) -> str:
+        if self.partitioning == S3Partitioning.FUNCTION_NAME:
+            return f"function={sanitize(record.get('functionName', ''))}/"
+        if self.partitioning == S3Partitioning.DATE:
+            start = str(record.get("startTime", ""))
+            # YYYY-MM-DD... -> year=YYYY/month=MM/day=DD/
+            if len(start) >= 10 and start[4] == "-" and start[7] == "-":
+                return f"year={start[0:4]}/month={start[5:7]}/day={start[8:10]}/"
+            return ""
+        return ""

aws_durable_execution_sdk_python_telemetry_exporter/exporters/sqs_exporter.py

@@ -0,0 +1,96 @@
+# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+"""SQS Durable Execution Telemetry exporter."""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+from aws_durable_execution_sdk_python_telemetry_exporter.exporters._common import (
+    compact_dumps,
+)
+from aws_durable_execution_sdk_python_telemetry_exporter.operations_index import (
+    OperationsFormat,
+    OperationsFormatInput,
+    apply_operations_format,
+)
+
+
+# SQS caps MessageGroupId and MessageDeduplicationId at 128 characters.
+_MAX_ID_LENGTH = 128
+
+
+class SQSExporter:
+    """Sends each record as one SQS message with SendMessage.
+
+    The rendered record is the message body; ``status`` and ``functionName``
+    are message attributes. On a FIFO queue (URL ending in ``.fifo``) the group
+    id defaults to ``executionArn`` and the deduplication id is
+    ``executionArn:emittedAt``; an id longer than 128 characters is replaced by
+    its SHA-256 hex digest so the message is never rejected.
+    """
+
+    def __init__(
+        self,
+        queue_url: str,
+        message_group_id: str | None = None,
+        region: str | None = None,
+        operations_format: OperationsFormat | OperationsFormatInput = (
+            OperationsFormat.ARRAY
+        ),
+        max_record_size_bytes: int | None = None,
+        client: Any = None,
+    ) -> None:
+        self.queue_url = queue_url
+        self.message_group_id = message_group_id
+        self.is_fifo = queue_url.endswith(".fifo")
+        self.operations_format = OperationsFormat(operations_format)
+        self.max_record_size_bytes: int | None = (
+            256_000 if max_record_size_bytes is None else max_record_size_bytes
+        )
+        if client is not None:
+            self._client = client
+        else:
+            import boto3  # deferred: boto3 is provided by the Lambda runtime
+
+            self._client = (
+                boto3.client("sqs", region_name=region)
+                if region
+                else boto3.client("sqs")
+            )
+
+    def render(self, record: dict[str, Any]) -> dict[str, Any]:
+        return apply_operations_format(record, self.operations_format)
+
+    def export(self, record: dict[str, Any]) -> None:
+        kwargs: dict[str, Any] = {
+            "QueueUrl": self.queue_url,
+            "MessageBody": compact_dumps(self.render(record)),
+            "MessageAttributes": {
+                "status": {"DataType": "String", "StringValue": record["status"]},
+                "functionName": {
+                    "DataType": "String",
+                    "StringValue": record["functionName"],
+                },
+            },
+        }
+        if self.is_fifo:
+            kwargs["MessageGroupId"] = _bounded_id(
+                self.message_group_id or record["executionArn"]
+            )
+            kwargs["MessageDeduplicationId"] = _bounded_id(
+                f"{record['executionArn']}:{record['emittedAt']}"
+            )
+        self._client.send_message(**kwargs)
+
+    def flush(self) -> None:
+        return None
+
+
+def _bounded_id(value: str) -> str:
+    """Return ``value`` if it fits the SQS 128-character id limit, else its SHA-256."""
+    if len(value) <= _MAX_ID_LENGTH:
+        return value
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()