aws-lambda-powertools 3.14.1a5__tar.gz → 3.15.0__tar.gz

{aws_lambda_powertools-3.14.1a5 → aws_lambda_powertools-3.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: aws_lambda_powertools
-Version: 3.14.1a5
+Version: 3.15.0
 Summary: Powertools for AWS Lambda (Python) is a developer toolkit to implement Serverless best practices and increase developer velocity.
 License: MIT
 Keywords: aws_lambda_powertools,aws,tracing,logging,lambda,powertools,feature_flags,idempotency,middleware
@@ -21,11 +21,14 @@ Provides-Extra: all
 Provides-Extra: aws-sdk
 Provides-Extra: datadog
 Provides-Extra: datamasking
+Provides-Extra: kafka-consumer-avro
+Provides-Extra: kafka-consumer-protobuf
 Provides-Extra: parser
 Provides-Extra: redis
 Provides-Extra: tracer
 Provides-Extra: validation
 Provides-Extra: valkey
+Requires-Dist: avro (>=1.12.0,<2.0.0) ; extra == "kafka-consumer-avro"
 Requires-Dist: aws-encryption-sdk (>=3.1.1,<5.0.0) ; extra == "all" or extra == "datamasking"
 Requires-Dist: aws-xray-sdk (>=2.8.0,<3.0.0) ; extra == "tracer" or extra == "all"
 Requires-Dist: boto3 (>=1.34.32,<2.0.0) ; extra == "aws-sdk"
@@ -33,6 +36,7 @@ Requires-Dist: datadog-lambda (>=6.106.0,<7.0.0) ; extra == "datadog"
 Requires-Dist: fastjsonschema (>=2.14.5,<3.0.0) ; extra == "validation" or extra == "all"
 Requires-Dist: jmespath (>=1.0.1,<2.0.0)
 Requires-Dist: jsonpath-ng (>=1.6.0,<2.0.0) ; extra == "all" or extra == "datamasking"
+Requires-Dist: protobuf (>=6.30.2,<7.0.0) ; extra == "kafka-consumer-protobuf"
 Requires-Dist: pydantic (>=2.4.0,<3.0.0) ; extra == "parser" or extra == "all"
 Requires-Dist: pydantic-settings (>=2.6.1,<3.0.0) ; extra == "all"
 Requires-Dist: redis (>=4.4,<7.0) ; extra == "redis"

{aws_lambda_powertools-3.14.1a5 → aws_lambda_powertools-3.15.0}/aws_lambda_powertools/event_handler/api_gateway.py

@@ -407,7 +407,7 @@ class Route:
 
         # OpenAPI spec only understands paths with { }. So we'll have to convert Powertools' < >.
         # https://swagger.io/specification/#path-templating
-        self.openapi_path = re.sub(r"<(.*?)>", lambda m: f"{{{''.join(m.group(1))}}}", self.path)
+        self.openapi_path = re.sub(r"<(.*?)>", lambda m: f"{{{''.join(m.group(1))}}}", self.path)  # type: ignore[arg-type]
 
         self.rule = rule
         self.func = func

{aws_lambda_powertools-3.14.1a5 → aws_lambda_powertools-3.15.0}/aws_lambda_powertools/shared/version.py

@@ -1,3 +1,3 @@
 """Exposes version constant to avoid circular dependencies."""
 
-VERSION = "3.14.1a5"
+VERSION = "3.15.0"

{aws_lambda_powertools-3.14.1a5 → aws_lambda_powertools-3.15.0}/aws_lambda_powertools/utilities/data_classes/kafka_event.py

@@ -10,7 +10,19 @@ if TYPE_CHECKING:
     from collections.abc import Iterator
 
 
-class KafkaEventRecord(DictWrapper):
+class KafkaEventRecordSchemaMetadata(DictWrapper):
+    @property
+    def data_format(self) -> str | None:
+        """The data format of the Kafka record."""
+        return self.get("dataFormat", None)
+
+    @property
+    def schema_id(self) -> str | None:
+        """The schema id of the Kafka record."""
+        return self.get("schemaId", None)
+
+
+class KafkaEventRecordBase(DictWrapper):
     @property
     def topic(self) -> str:
         """The Kafka topic."""
@@ -36,6 +48,24 @@ class KafkaEventRecord(DictWrapper):
         """The Kafka record timestamp type."""
         return self["timestampType"]
 
+    @property
+    def key_schema_metadata(self) -> KafkaEventRecordSchemaMetadata | None:
+        """The metadata of the Key Kafka record."""
+        return (
+            None if self.get("keySchemaMetadata") is None else KafkaEventRecordSchemaMetadata(self["keySchemaMetadata"])
+        )
+
+    @property
+    def value_schema_metadata(self) -> KafkaEventRecordSchemaMetadata | None:
+        """The metadata of the Value Kafka record."""
+        return (
+            None
+            if self.get("valueSchemaMetadata") is None
+            else KafkaEventRecordSchemaMetadata(self["valueSchemaMetadata"])
+        )
+
+
+class KafkaEventRecord(KafkaEventRecordBase):
     @property
     def key(self) -> str | None:
         """
@@ -83,18 +113,7 @@ class KafkaEventRecord(DictWrapper):
         return CaseInsensitiveDict((k, bytes(v)) for chunk in self.headers for k, v in chunk.items())
 
 
-class KafkaEvent(DictWrapper):
-    """Self-managed or MSK Apache Kafka event trigger
-    Documentation:
-    --------------
-    - https://docs.aws.amazon.com/lambda/latest/dg/with-kafka.html
-    - https://docs.aws.amazon.com/lambda/latest/dg/with-msk.html
-    """
-
-    def __init__(self, data: dict[str, Any]):
-        super().__init__(data)
-        self._records: Iterator[KafkaEventRecord] | None = None
-
+class KafkaEventBase(DictWrapper):
     @property
     def event_source(self) -> str:
         """The AWS service from which the Kafka event record originated."""
@@ -115,6 +134,19 @@ class KafkaEvent(DictWrapper):
         """The decoded Kafka bootstrap URL."""
         return self.bootstrap_servers.split(",")
 
+
+class KafkaEvent(KafkaEventBase):
+    """Self-managed or MSK Apache Kafka event trigger
+    Documentation:
+    --------------
+    - https://docs.aws.amazon.com/lambda/latest/dg/with-kafka.html
+    - https://docs.aws.amazon.com/lambda/latest/dg/with-msk.html
+    """
+
+    def __init__(self, data: dict[str, Any]):
+        super().__init__(data)
+        self._records: Iterator[KafkaEventRecord] | None = None
+
     @property
     def records(self) -> Iterator[KafkaEventRecord]:
         """The Kafka records."""

aws_lambda_powertools-3.15.0/aws_lambda_powertools/utilities/kafka/__init__.py

@@ -0,0 +1,9 @@
+from aws_lambda_powertools.utilities.kafka.consumer_records import ConsumerRecords
+from aws_lambda_powertools.utilities.kafka.kafka_consumer import kafka_consumer
+from aws_lambda_powertools.utilities.kafka.schema_config import SchemaConfig
+
+__all__ = [
+    "kafka_consumer",
+    "ConsumerRecords",
+    "SchemaConfig",
+]

aws_lambda_powertools-3.15.0/aws_lambda_powertools/utilities/kafka/consumer_records.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from functools import cached_property
+from typing import TYPE_CHECKING, Any
+
+from aws_lambda_powertools.utilities.data_classes.common import CaseInsensitiveDict
+from aws_lambda_powertools.utilities.data_classes.kafka_event import KafkaEventBase, KafkaEventRecordBase
+from aws_lambda_powertools.utilities.kafka.deserializer.deserializer import get_deserializer
+from aws_lambda_powertools.utilities.kafka.serialization.serialization import serialize_to_output_type
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from aws_lambda_powertools.utilities.kafka.schema_config import SchemaConfig
+
+
+class ConsumerRecordRecords(KafkaEventRecordBase):
+    """
+    A Kafka Consumer Record
+    """
+
+    def __init__(self, data: dict[str, Any], schema_config: SchemaConfig | None = None):
+        super().__init__(data)
+        self.schema_config = schema_config
+
+    @cached_property
+    def key(self) -> Any:
+        key = self.get("key")
+
+        # Return None if key doesn't exist
+        if not key:
+            return None
+
+        # Determine schema type and schema string
+        schema_type = None
+        schema_str = None
+        output_serializer = None
+
+        if self.schema_config and self.schema_config.key_schema_type:
+            schema_type = self.schema_config.key_schema_type
+            schema_str = self.schema_config.key_schema
+            output_serializer = self.schema_config.key_output_serializer
+
+        # Always use get_deserializer if None it will default to DEFAULT
+        deserializer = get_deserializer(schema_type, schema_str)
+        deserialized_value = deserializer.deserialize(key)
+
+        # Apply output serializer if specified
+        if output_serializer:
+            return serialize_to_output_type(deserialized_value, output_serializer)
+
+        return deserialized_value
+
+    @cached_property
+    def value(self) -> Any:
+        value = self["value"]
+
+        # Determine schema type and schema string
+        schema_type = None
+        schema_str = None
+        output_serializer = None
+
+        if self.schema_config and self.schema_config.value_schema_type:
+            schema_type = self.schema_config.value_schema_type
+            schema_str = self.schema_config.value_schema
+            output_serializer = self.schema_config.value_output_serializer
+
+        # Always use get_deserializer if None it will default to DEFAULT
+        deserializer = get_deserializer(schema_type, schema_str)
+        deserialized_value = deserializer.deserialize(value)
+
+        # Apply output serializer if specified
+        if output_serializer:
+            return serialize_to_output_type(deserialized_value, output_serializer)
+
+        return deserialized_value
+
+    @property
+    def original_value(self) -> str:
+        """The original (base64 encoded) Kafka record value."""
+        return self["value"]
+
+    @property
+    def original_key(self) -> str | None:
+        """
+        The original (base64 encoded) Kafka record key.
+
+        This key is optional; if not provided,
+        a round-robin algorithm will be used to determine
+        the partition for the message.
+        """
+
+        return self.get("key")
+
+    @property
+    def original_headers(self) -> list[dict[str, list[int]]]:
+        """The raw Kafka record headers."""
+        return self["headers"]
+
+    @cached_property
+    def headers(self) -> dict[str, bytes]:
+        """Decodes the headers as a single dictionary."""
+        return CaseInsensitiveDict((k, bytes(v)) for chunk in self.original_headers for k, v in chunk.items())
+
+
+class ConsumerRecords(KafkaEventBase):
+    """Self-managed or MSK Apache Kafka event trigger
+    Documentation:
+    --------------
+    - https://docs.aws.amazon.com/lambda/latest/dg/with-kafka.html
+    - https://docs.aws.amazon.com/lambda/latest/dg/with-msk.html
+    """
+
+    def __init__(self, data: dict[str, Any], schema_config: SchemaConfig | None = None):
+        super().__init__(data)
+        self._records: Iterator[ConsumerRecordRecords] | None = None
+        self.schema_config = schema_config
+
+    @property
+    def records(self) -> Iterator[ConsumerRecordRecords]:
+        """The Kafka records."""
+        for chunk in self["records"].values():
+            for record in chunk:
+                yield ConsumerRecordRecords(data=record, schema_config=self.schema_config)
+
+    @property
+    def record(self) -> ConsumerRecordRecords:
+        """
+        Returns the next Kafka record using an iterator.
+
+        Returns
+        -------
+        ConsumerRecordRecords
+            The next Kafka record.
+
+        Raises
+        ------
+        StopIteration
+            If there are no more records available.
+
+        """
+        if self._records is None:
+            self._records = self.records
+        return next(self._records)

aws_lambda_powertools-3.15.0/aws_lambda_powertools/utilities/kafka/deserializer/avro.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import io
+
+from avro.io import BinaryDecoder, DatumReader
+from avro.schema import parse as parse_schema
+
+from aws_lambda_powertools.utilities.kafka.deserializer.base import DeserializerBase
+from aws_lambda_powertools.utilities.kafka.exceptions import (
+    KafkaConsumerAvroSchemaParserError,
+    KafkaConsumerDeserializationError,
+)
+
+
+class AvroDeserializer(DeserializerBase):
+    """
+    Deserializer for Apache Avro formatted data.
+
+    This class provides functionality to deserialize Avro binary data using
+    a provided Avro schema definition.
+    """
+
+    def __init__(self, schema_str: str):
+        try:
+            self.parsed_schema = parse_schema(schema_str)
+            self.reader = DatumReader(self.parsed_schema)
+        except Exception as e:
+            raise KafkaConsumerAvroSchemaParserError(
+                f"Invalid Avro schema. Please ensure the provided avro schema is valid: {type(e).__name__}: {str(e)}",
+            ) from e
+
+    def deserialize(self, data: bytes | str) -> object:
+        """
+        Deserialize Avro binary data to a Python dictionary.
+
+        Parameters
+        ----------
+        data : bytes or str
+            The Avro binary data to deserialize. If provided as a string,
+            it will be decoded to bytes first.
+
+        Returns
+        -------
+        dict[str, Any]
+            Deserialized data as a dictionary.
+
+        Raises
+        ------
+        KafkaConsumerDeserializationError
+            When the data cannot be deserialized according to the schema,
+            typically due to data format incompatibility.
+
+        Examples
+        --------
+        >>> deserializer = AvroDeserializer(schema_str)
+        >>> avro_data = b'...'  # binary Avro data
+        >>> try:
+        ...     result = deserializer.deserialize(avro_data)
+        ...     # Process the deserialized data
+        ... except KafkaConsumerDeserializationError as e:
+        ...     print(f"Failed to deserialize: {e}")
+        """
+        try:
+            value = self._decode_input(data)
+            bytes_reader = io.BytesIO(value)
+            decoder = BinaryDecoder(bytes_reader)
+            return self.reader.read(decoder)
+        except Exception as e:
+            raise KafkaConsumerDeserializationError(
+                f"Error trying to deserialize avro data - {type(e).__name__}: {str(e)}",
+            ) from e

aws_lambda_powertools-3.15.0/aws_lambda_powertools/utilities/kafka/deserializer/base.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import base64
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class DeserializerBase(ABC):
+    """
+    Abstract base class for deserializers.
+
+    This class defines the interface for all deserializers in the Kafka consumer utility
+    and provides a common method for decoding input data.
+
+    Methods
+    -------
+    deserialize(data)
+        Abstract method that must be implemented by subclasses to deserialize data.
+    _decode_input(data)
+        Helper method to decode input data to bytes.
+
+    Examples
+    --------
+    >>> class MyDeserializer(DeserializerBase):
+    ...     def deserialize(self, data: bytes | str) -> dict[str, Any]:
+    ...         value = self._decode_input(data)
+    ...         # Custom deserialization logic here
+    ...         return {"key": "value"}
+    """
+
+    @abstractmethod
+    def deserialize(self, data: str) -> dict[str, Any] | str | object:
+        """
+        Deserialize input data to a Python dictionary.
+
+        This abstract method must be implemented by subclasses to provide
+        specific deserialization logic.
+
+        Parameters
+        ----------
+        data : str
+            The data to deserialize, it's always a base64 encoded string
+
+        Returns
+        -------
+        dict[str, Any]
+            The deserialized data as a dictionary.
+        """
+        raise NotImplementedError("Subclasses must implement the deserialize method")
+
+    def _decode_input(self, data: bytes | str) -> bytes:
+        return base64.b64decode(data)

aws_lambda_powertools-3.15.0/aws_lambda_powertools/utilities/kafka/deserializer/default.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import base64
+
+from aws_lambda_powertools.utilities.kafka.deserializer.base import DeserializerBase
+
+
+class DefaultDeserializer(DeserializerBase):
+    """
+    A default deserializer that performs base64 decode + binary decode on the input data.
+
+    This deserializer simply returns the input data with base64 decode, which is useful when
+    no customized deserialization is needed or when handling raw data formats.
+    """
+
+    def deserialize(self, data: bytes | str) -> str:
+        """
+        Return the input data base64 decoded.
+
+        This method implements the deserialize interface and performs base64 decode.
+
+        Parameters
+        ----------
+        data : bytes or str
+            The input data to "deserialize".
+
+        Returns
+        -------
+        dict[str, Any]
+            The input data base64 decoded.
+
+        Example
+        --------
+        >>> deserializer = NoOpDeserializer()
+        >>>
+        >>> # With string input
+        >>> string_data = "Hello, world!"
+        >>> result = deserializer.deserialize(string_data)
+        >>> print(result == string_data)  # Output: True
+        >>>
+        >>> # With bytes input
+        >>> bytes_data = b"Binary data"
+        >>> result = deserializer.deserialize(bytes_data)
+        >>> print(result == bytes_data)  # Output: True
+        """
+        return base64.b64decode(data).decode("utf-8")

aws_lambda_powertools-3.15.0/aws_lambda_powertools/utilities/kafka/deserializer/deserializer.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import hashlib
+from typing import TYPE_CHECKING, Any
+
+from aws_lambda_powertools.utilities.kafka.deserializer.default import DefaultDeserializer
+from aws_lambda_powertools.utilities.kafka.deserializer.json import JsonDeserializer
+
+if TYPE_CHECKING:
+    from aws_lambda_powertools.utilities.kafka.deserializer.base import DeserializerBase
+
+# Cache for deserializers
+_deserializer_cache: dict[str, DeserializerBase] = {}
+
+
+def _get_cache_key(schema_type: str | object, schema_value: Any) -> str:
+    if schema_value is None:
+        return str(schema_type)
+
+    if isinstance(schema_value, str):
+        # For string schemas like Avro, hash the content
+        schema_hash = hashlib.md5(schema_value.encode("utf-8"), usedforsecurity=False).hexdigest()
+    else:
+        # For objects like Protobuf, use the object id
+        schema_hash = str(id(schema_value))
+
+    return f"{schema_type}_{schema_hash}"
+
+
+def get_deserializer(schema_type: str | object, schema_value: Any) -> DeserializerBase:
+    """
+    Factory function to get the appropriate deserializer based on schema type.
+
+    This function creates and returns a deserializer instance that corresponds to the
+    specified schema type. It handles lazy imports for optional dependencies.
+
+    Parameters
+    ----------
+    schema_type : str
+        The type of schema to use for deserialization.
+        Supported values are: "AVRO", "PROTOBUF", "JSON", or any other value for no-op.
+    schema_value : Any
+        The schema definition to use for deserialization. The format depends on the
+        schema_type:
+        - For "AVRO": A string containing the Avro schema definition
+        - For "PROTOBUF": A object containing the Protobuf schema definition
+        - For "JSON": Not used (can be None)
+        - For other types: Not used (can be None)
+
+    Returns
+    -------
+    DeserializerBase
+        An instance of a deserializer that implements the DeserializerBase interface.
+
+    Examples
+    --------
+    >>> # Get an Avro deserializer
+    >>> avro_schema = '''
+    ...     {
+    ...       "type": "record",
+    ...       "name": "User",
+    ...       "fields": [
+    ...         {"name": "name", "type": "string"},
+    ...         {"name": "age", "type": "int"}
+    ...       ]
+    ...     }
+    ... '''
+    >>> deserializer = get_deserializer("AVRO", avro_schema)
+    >>>
+    >>> # Get a JSON deserializer
+    >>> json_deserializer = get_deserializer("JSON", None)
+    >>>
+    >>> # Get a no-op deserializer for raw data
+    >>> no_op_deserializer = get_deserializer("RAW", None)
+    """
+
+    # Generate a cache key based on schema type and value
+    cache_key = _get_cache_key(schema_type, schema_value)
+
+    # Check if we already have this deserializer in cache
+    if cache_key in _deserializer_cache:
+        return _deserializer_cache[cache_key]
+
+    deserializer: DeserializerBase
+
+    if schema_type == "AVRO":
+        # Import here to avoid dependency if not used
+        from aws_lambda_powertools.utilities.kafka.deserializer.avro import AvroDeserializer
+
+        deserializer = AvroDeserializer(schema_value)
+    elif schema_type == "PROTOBUF":
+        # Import here to avoid dependency if not used
+        from aws_lambda_powertools.utilities.kafka.deserializer.protobuf import ProtobufDeserializer
+
+        deserializer = ProtobufDeserializer(schema_value)
+    elif schema_type == "JSON":
+        deserializer = JsonDeserializer()
+
+    else:
+        # Default to no-op deserializer
+        deserializer = DefaultDeserializer()
+
+    # Store in cache for future use
+    _deserializer_cache[cache_key] = deserializer
+
+    # Default to default deserializer that is base64 decode + bytes decoded
+    return deserializer

aws_lambda_powertools-3.15.0/aws_lambda_powertools/utilities/kafka/deserializer/json.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import base64
+import json
+
+from aws_lambda_powertools.utilities.kafka.deserializer.base import DeserializerBase
+from aws_lambda_powertools.utilities.kafka.exceptions import KafkaConsumerDeserializationError
+
+
+class JsonDeserializer(DeserializerBase):
+    """
+    Deserializer for JSON formatted data.
+
+    This class provides functionality to deserialize JSON data from bytes or string
+    into Python dictionaries.
+    """
+
+    def deserialize(self, data: bytes | str) -> dict:
+        """
+        Deserialize JSON data to a Python dictionary.
+
+        Parameters
+        ----------
+        data : bytes or str
+            The JSON data to deserialize. If provided as bytes, it will be decoded as UTF-8.
+            If provided as a string, it's assumed to be base64-encoded and will be decoded first.
+
+        Returns
+        -------
+        dict
+            Deserialized data as a dictionary.
+
+        Raises
+        ------
+        KafkaConsumerDeserializationError
+            When the data cannot be deserialized as valid JSON.
+
+        Examples
+        --------
+        >>> deserializer = JsonDeserializer()
+        >>> json_data = '{"key": "value", "number": 123}'
+        >>> try:
+        ...     result = deserializer.deserialize(json_data)
+        ...     print(result["key"])  # Output: value
+        ... except KafkaConsumerDeserializationError as e:
+        ...     print(f"Failed to deserialize: {e}")
+        """
+        try:
+            return json.loads(base64.b64decode(data).decode("utf-8"))
+        except Exception as e:
+            raise KafkaConsumerDeserializationError(
+                f"Error trying to deserialize json data -  {type(e).__name__}: {str(e)}",
+            ) from e