modmex-lambda 0.5.2__tar.gz → 0.5.4__tar.gz

{modmex_lambda-0.5.2 → modmex_lambda-0.5.4}/.gitignore

@@ -218,4 +218,6 @@ __marimo__/
 .streamlit/secrets.toml
 .benchmark/
 .vscode/
-.doc/
+.doc/
+
+.DS_Store

{modmex_lambda-0.5.2 → modmex_lambda-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modmex-lambda
-Version: 0.5.2
+Version: 0.5.4
 Summary: Ultra-lightweight AWS Lambda utilities for API Gateway routing and event handling.
 Author: Modmex
 License: MIT
@@ -36,6 +36,8 @@ API Gateway proxy events, fast routing, request binding, response serialization,
 middleware, dependency injection, stream sources, parsing, and structured
 logging.
 
+The fuller documentation site lives in [`docs/`](docs/).
+
 ## Install
 
 ```bash
@@ -979,37 +981,66 @@ materialize so queries stay local and fast, and each service owns the model it
 needs instead of querying another service synchronously.
 
 ```python
+from modmex_lambda.persistence.dynamodb import MaterializedViewMixin
 from modmex_lambda.stream.flavors.materialize import Materialize
 from modmex_lambda.stream.rules_registry import RulesRegistry
 from modmex_lambda.stream.sources import KinesisSource
-from modmex_lambda.stream.utils.dynamodb import update_expression
 
 
-def to_thing_view_update(uow):
-    thing = uow["event"]["thing"]
-    return {
-        "Key": {
-            "pk": thing["id"],
-            "sk": "THING",
-        },
-        **update_expression({
-            "name": thing["name"],
-            "timestamp": uow["event"]["timestamp"],
-        }),
-    }
+class ThingViewRepository(MaterializedViewMixin):
+    discriminator = "thing"
+
+
+to_update_request = ThingViewRepository.materialized_update_request_mapper()
 
 
 registry = RulesRegistry().registry(
     Materialize({
         "id": "materialize-thing",
         "event_type": "thing-created",
-        "to_update_request": to_thing_view_update,
+        "to_update_request": to_update_request,
     })
 )
 
 handler = KinesisSource(registry, concurrency=False).handle
 ```
 
+`MaterializedViewMixin` is a persistence helper for the common DynamoDB case:
+copy the entity from `uow["event"][discriminator]`, build a `Key` with
+`pk=entity["id"]` and `sk=discriminator`, add stream-compatible fields, and
+protect the write with `timestamp_condition()` so older events do not overwrite
+newer records. It adds stream-compatible fields such as `discriminator`,
+`timestamp`, `deleted`, `latched`, `ttl`, and `awsregion` to the update
+expression.
+
+Override the hooks when the event source, key, projection, or enrichment is
+different from the default:
+
+```python
+from modmex_lambda.persistence.dynamodb import MaterializedViewMixin
+
+
+class ThingSearchViewRepository(MaterializedViewMixin):
+    discriminator = "thing-search"
+    materialized_source_name = "thing"
+
+    def materialized_key(self, uow, thing):
+        return {
+            "pk": thing["tenant_id"],
+            "sk": f"thing-search#{thing['id']}",
+        }
+
+    def materialized_fields(self, uow, thing):
+        return {
+            **super().materialized_fields(uow, thing),
+            "search_text": f"{thing['name']} {thing['tenant_id']}",
+        }
+```
+
+Use `DynamoDBUpdateRequestMixin.build_update_request()` directly when you already
+have the key and fields but still want the standard `UpdateItem` expression and
+timestamp guard.
+
 Use `split_on` and `split_target_field` when one event updates several records,
 for example one `order-created` event materializing each order item.
 

{modmex_lambda-0.5.2 → modmex_lambda-0.5.4}/README.md

@@ -13,6 +13,8 @@ API Gateway proxy events, fast routing, request binding, response serialization,
 middleware, dependency injection, stream sources, parsing, and structured
 logging.
 
+The fuller documentation site lives in [`docs/`](docs/).
+
 ## Install
 
 ```bash
@@ -956,37 +958,66 @@ materialize so queries stay local and fast, and each service owns the model it
 needs instead of querying another service synchronously.
 
 ```python
+from modmex_lambda.persistence.dynamodb import MaterializedViewMixin
 from modmex_lambda.stream.flavors.materialize import Materialize
 from modmex_lambda.stream.rules_registry import RulesRegistry
 from modmex_lambda.stream.sources import KinesisSource
-from modmex_lambda.stream.utils.dynamodb import update_expression
 
 
-def to_thing_view_update(uow):
-    thing = uow["event"]["thing"]
-    return {
-        "Key": {
-            "pk": thing["id"],
-            "sk": "THING",
-        },
-        **update_expression({
-            "name": thing["name"],
-            "timestamp": uow["event"]["timestamp"],
-        }),
-    }
+class ThingViewRepository(MaterializedViewMixin):
+    discriminator = "thing"
+
+
+to_update_request = ThingViewRepository.materialized_update_request_mapper()
 
 
 registry = RulesRegistry().registry(
     Materialize({
         "id": "materialize-thing",
         "event_type": "thing-created",
-        "to_update_request": to_thing_view_update,
+        "to_update_request": to_update_request,
     })
 )
 
 handler = KinesisSource(registry, concurrency=False).handle
 ```
 
+`MaterializedViewMixin` is a persistence helper for the common DynamoDB case:
+copy the entity from `uow["event"][discriminator]`, build a `Key` with
+`pk=entity["id"]` and `sk=discriminator`, add stream-compatible fields, and
+protect the write with `timestamp_condition()` so older events do not overwrite
+newer records. It adds stream-compatible fields such as `discriminator`,
+`timestamp`, `deleted`, `latched`, `ttl`, and `awsregion` to the update
+expression.
+
+Override the hooks when the event source, key, projection, or enrichment is
+different from the default:
+
+```python
+from modmex_lambda.persistence.dynamodb import MaterializedViewMixin
+
+
+class ThingSearchViewRepository(MaterializedViewMixin):
+    discriminator = "thing-search"
+    materialized_source_name = "thing"
+
+    def materialized_key(self, uow, thing):
+        return {
+            "pk": thing["tenant_id"],
+            "sk": f"thing-search#{thing['id']}",
+        }
+
+    def materialized_fields(self, uow, thing):
+        return {
+            **super().materialized_fields(uow, thing),
+            "search_text": f"{thing['name']} {thing['tenant_id']}",
+        }
+```
+
+Use `DynamoDBUpdateRequestMixin.build_update_request()` directly when you already
+have the key and fields but still want the standard `UpdateItem` expression and
+timestamp guard.
+
 Use `split_on` and `split_target_field` when one event updates several records,
 for example one `order-created` event materializing each order item.
 

modmex_lambda-0.5.4/modmex_lambda/connectors/dynamodb.py

@@ -0,0 +1,282 @@
+from functools import reduce
+from typing import Iterable
+
+from modmex_lambda.connectors.idynamodb import IDynamodbConnector
+from modmex_lambda.stream.utils.retry import (
+    assert_max_retries,
+    DEFAULT_RETRY_CONFIG,
+    wait,
+    get_delay
+)
+
+
+class Connector(IDynamodbConnector):
+
+    def __init__(self,# pylint: disable=W0102
+                 table_name = 'undefined',
+                 retry_config = DEFAULT_RETRY_CONFIG,
+                 client = None) -> None:
+        self.table_name = table_name
+        self._client = client
+        self.retry_config = retry_config
+        self._serializer = None
+        self._deserializer = None
+
+    @property
+    def client(self):
+        if not self._client:
+            import boto3
+            self._client = boto3.client('dynamodb')
+        return self._client
+
+    def get(self, input_params):
+        response = self.client.get_item(
+            TableName=self.table_name,
+            **self._marshall_request(input_params),
+        )
+        return self._unmarshall_response(response)
+
+    def update(self, input_params):
+        response = self.client.update_item(
+            TableName=self.table_name,
+            **self._marshall_request(input_params),
+        )
+        return self._unmarshall_response(response)
+
+    def put(self, input_params):
+        response = self.client.put_item(
+            TableName=self.table_name,
+            **self._marshall_request(input_params),
+        )
+        return self._unmarshall_response(response)
+
+    def query(self, input_params):
+        response = self.client.query(
+            TableName=self.table_name,
+            **self._marshall_request(input_params),
+        )
+        return self._unmarshall_response(response)
+
+    def query_page(self, input_params):
+        return self.query(input_params)
+
+    def scan(self, input_params):
+        response = self.client.scan(
+            TableName=self.table_name,
+            **self._marshall_request(input_params),
+        )
+        return self._unmarshall_response(response)
+
+    def query_all(self, input_params):
+        items = []
+        while True:
+            result = self.query(input_params)
+            items.extend(result.get('Items', []))
+            if result.get('LastEvaluatedKey'):
+                input_params['ExclusiveStartKey'] = result['LastEvaluatedKey']
+            else:
+                break
+        return items
+
+    def batch_get(self, input_params):
+        return self._batch_get(input_params, [])
+
+    def _batch_get(self, params, attempts):
+        assert_max_retries(attempts, self.retry_config['max_retries'])
+        wait(get_delay(self.retry_config['retry_wait'], len(attempts)))
+        response = self.client.batch_get_item(**self._marshall_batch_get_request(params))
+        response = self._unmarshall_response(response)
+        if response.get('UnprocessedKeys'):
+            return self._batch_get(
+                unprocessed(params, response),
+                [*attempts, response]
+            )
+        return accumulate(attempts, response)
+
+    def bulk_insert(self, items: Iterable):
+        self._batch_write([
+            {
+                'PutRequest': {
+                    'Item': item,
+                },
+            }
+            for item in items
+        ])
+
+    def bulk_delete(self, items: Iterable):
+        self._batch_write([
+            {
+                'DeleteRequest': {
+                    'Key': key,
+                },
+            }
+            for key in items
+        ])
+
+    def _marshall_request(self, params):
+        return {
+            key: self._marshall_value(key, value)
+            for key, value in params.items()
+        }
+
+    def _marshall_batch_get_request(self, params):
+        return {
+            **params,
+            'RequestItems': {
+                table_name: {
+                    **request,
+                    'Keys': [
+                        self._marshall_item(key)
+                        for key in request.get('Keys', [])
+                    ],
+                }
+                for table_name, request in params.get('RequestItems', {}).items()
+            },
+        }
+
+    def _marshall_value(self, key, value):
+        if key in {'Key', 'Item', 'LastEvaluatedKey', 'ExclusiveStartKey'}:
+            return self._marshall_item(value)
+        if key == 'ExpressionAttributeValues':
+            return {
+                attr: self.serializer.serialize(attr_value)
+                for attr, attr_value in value.items()
+            }
+        return value
+
+    def _marshall_item(self, item):
+        return {
+            key: self.serializer.serialize(value)
+            for key, value in item.items()
+        }
+
+    def _unmarshall_response(self, response):
+        return {
+            key: self._unmarshall_value(key, value)
+            for key, value in response.items()
+        }
+
+    def _unmarshall_value(self, key, value):
+        if key in {'Item', 'LastEvaluatedKey'}:
+            return self._unmarshall_item(value)
+        if key == 'Items':
+            return [self._unmarshall_item(item) for item in value]
+        if key == 'Attributes':
+            return self._unmarshall_item(value)
+        if key == 'Responses':
+            return {
+                table_name: [self._unmarshall_item(item) for item in items]
+                for table_name, items in value.items()
+            }
+        if key == 'UnprocessedKeys':
+            return {
+                table_name: {
+                    **request,
+                    'Keys': [self._unmarshall_item(item) for item in request.get('Keys', [])],
+                }
+                for table_name, request in value.items()
+            }
+        return value
+
+    def _unmarshall_item(self, item):
+        return {
+            key: self.deserializer.deserialize(value)
+            for key, value in item.items()
+        }
+
+    @property
+    def serializer(self):
+        if not self._serializer:
+            self._load_dynamodb_types()
+        return self._serializer
+
+    @property
+    def deserializer(self):
+        if not self._deserializer:
+            self._load_dynamodb_types()
+        return self._deserializer
+
+    def _load_dynamodb_types(self):
+        from boto3.dynamodb.types import TypeDeserializer, TypeSerializer
+        self._serializer = TypeSerializer()
+        self._deserializer = TypeDeserializer()
+
+    def _batch_write(self, requests):
+        for index in range(0, len(requests), 25):
+            self._batch_write_chunk(requests[index:index + 25], [])
+
+    def _batch_write_chunk(self, requests, attempts):
+        if not requests:
+            return
+        assert_max_retries(attempts, self.retry_config['max_retries'])
+        wait(get_delay(self.retry_config['retry_wait'], len(attempts)))
+        response = self.client.batch_write_item(
+            RequestItems={
+                self.table_name: [
+                    self._marshall_write_request(request)
+                    for request in requests
+                ],
+            },
+        )
+        unprocessed_items = response.get('UnprocessedItems', {}).get(self.table_name, [])
+        if unprocessed_items:
+            self._batch_write_chunk(
+                [
+                    self._unmarshall_write_request(request)
+                    for request in unprocessed_items
+                ],
+                [*attempts, response],
+            )
+
+    def _marshall_write_request(self, request):
+        if 'PutRequest' in request:
+            return {
+                'PutRequest': {
+                    'Item': self._marshall_item(request['PutRequest']['Item']),
+                },
+            }
+        return {
+            'DeleteRequest': {
+                'Key': self._marshall_item(request['DeleteRequest']['Key']),
+            },
+        }
+
+    def _unmarshall_write_request(self, request):
+        if 'PutRequest' in request:
+            return {
+                'PutRequest': {
+                    'Item': self._unmarshall_item(request['PutRequest']['Item']),
+                },
+            }
+        return {
+            'DeleteRequest': {
+                'Key': self._unmarshall_item(request['DeleteRequest']['Key']),
+            },
+        }
+
+
+def unprocessed(params, resp):
+    return {
+        **params,
+        'RequestItems': resp['UnprocessedKeys']
+    }
+
+def accumulate(attempts, resp):
+    def reducer(a, c):
+        return {
+            **a,
+            'Responses': reduce(
+                lambda a2, c2: {
+                    **a2,
+                    c2: [
+                        *a2.get(c2, []),
+                        *a['Responses'].get(c2, [])
+                    ]
+                },
+                list(a['Responses']),
+                {**c['Responses']}
+            ),
+            'attempts': [*attempts, resp]
+        }
+
+    return reduce(reducer, reversed(attempts), resp)

{modmex_lambda-0.5.2 → modmex_lambda-0.5.4}/modmex_lambda/connectors/module.py

@@ -17,9 +17,12 @@ class AwsConnectorsModule(Module):
     @singleton
     @provider
     def provide_dynamodb(self) -> IDynamodbConnector:
+        import os
         from modmex_lambda.connectors.dynamodb import Connector
 
-        return Connector()
+        return Connector(
+            table_name=os.getenv('ENTITY_TABLE_NAME') or os.getenv('EVENT_TABLE_NAME'),
+        )
 
     @singleton
     @provider

modmex_lambda-0.5.4/modmex_lambda/persistence/__init__.py

@@ -0,0 +1,2 @@
+"""Persistence helpers for common modmex-lambda application patterns."""
+

modmex_lambda-0.5.4/modmex_lambda/persistence/dynamodb/__init__.py

@@ -0,0 +1,39 @@
+"""DynamoDB persistence helpers."""
+
+from modmex_lambda.persistence.dynamodb.expressions import (
+    pk_condition,
+    timestamp_condition,
+    update_expression,
+)
+from modmex_lambda.persistence.dynamodb.keys import (
+    AggregateKeyStrategy,
+    KeyStrategy,
+    SingleEntityKeyStrategy,
+    TenantPartitionKeyStrategy,
+    TenantScopedSortKeyStrategy,
+)
+from modmex_lambda.persistence.dynamodb.materialized_views import (
+    DynamoDBUpdateRequestMixin,
+    MaterializedViewMixin,
+)
+from modmex_lambda.persistence.dynamodb.stream_fields import (
+    DefaultStreamFieldsStrategy,
+    StreamFieldsStrategy,
+    stream_entity_fields,
+)
+
+__all__ = [
+    "AggregateKeyStrategy",
+    "DynamoDBUpdateRequestMixin",
+    "KeyStrategy",
+    "MaterializedViewMixin",
+    "SingleEntityKeyStrategy",
+    "DefaultStreamFieldsStrategy",
+    "StreamFieldsStrategy",
+    "TenantPartitionKeyStrategy",
+    "TenantScopedSortKeyStrategy",
+    "pk_condition",
+    "stream_entity_fields",
+    "timestamp_condition",
+    "update_expression",
+]

modmex_lambda-0.5.4/modmex_lambda/persistence/dynamodb/expressions.py

@@ -0,0 +1,62 @@
+from functools import reduce
+
+
+def update_expression(item: dict):
+    keys = item.keys()
+    result = {}
+    result['ExpressionAttributeNames'] = reduce(
+        lambda accumulator, el: {**accumulator, **el},
+        map(
+            lambda attrName: {f"#{attrName}": attrName },
+            keys
+        ),
+        {}
+    )
+    result['ExpressionAttributeValues'] = reduce(
+        lambda accumulator, el: {**accumulator, **el},
+        map(
+            lambda attrName: {f":{attrName}": item[attrName] },
+            filter(
+                lambda attrName: item[attrName] is not None,
+                keys
+            )
+        ),
+        {}
+    )
+    result['UpdateExpression'] = "SET "+", ".join(map(
+            lambda attrName: f"#{attrName} = :{attrName}",
+            filter(
+                lambda attrName: item[attrName] is not None,
+                keys
+            )
+        ))
+    update_expression_remove = ", ".join(map(
+            lambda attrName: f"#{attrName}",
+            filter(
+                lambda attrName: item[attrName] is None,
+                keys
+            )
+        ))
+    if update_expression_remove:
+        result['UpdateExpression'] = "{} REMOVE {}".format(
+            result['UpdateExpression'],
+            update_expression_remove
+        )
+    result['ReturnValues'] = 'ALL_NEW'
+    return result
+
+
+def timestamp_condition(field_name = 'timestamp'):
+    return {
+        'ConditionExpression': "attribute_not_exists(#{fn}) OR #{fn} < :{fn}".format(
+            fn=field_name
+        )
+    }
+
+
+def pk_condition(field_name = 'pk'):
+    return {
+        'ConditionExpression': "attribute_not_exists({})".format(
+            field_name
+        )
+    }