daplug-ddb 1.0.0b1__py3-none-any.whl

daplug_ddb/__init__.py

@@ -0,0 +1,17 @@
+"""Public interface for the daplug_ddb package."""
+
+from typing import Any
+
+from .adapter import BatchItemException, DynamodbAdapter
+
+
+def adapter(**kwargs: Any) -> DynamodbAdapter:
+    """Factory helper mirroring legacy entry point for DynamoDB adapters."""
+
+    engine = kwargs.pop("engine", "dynamodb")
+    if engine and engine != "dynamodb":  # preserve deterministic erroring for unsupported engines
+        raise ValueError(f"engine {engine} not supported; only 'dynamodb' is available")
+    return DynamodbAdapter(**kwargs)
+
+
+__all__ = ["adapter", "DynamodbAdapter", "BatchItemException"]

daplug_ddb/adapter.py

@@ -0,0 +1,135 @@
+"""DynamoDB adapter exposing normalized CRUD operations."""
+
+from functools import lru_cache
+from typing import Any, Dict, Iterable, Optional, Union, cast
+
+import boto3
+from boto3.dynamodb.conditions import Attr
+
+from daplug_ddb.types import DynamoItem, DynamoItems
+
+from .common import map_to_schema, merge
+from .common.base_adapter import BaseAdapter
+from .exception import BatchItemException
+
+
+class DynamodbAdapter(BaseAdapter):
+    """Implements DynamoDB CRUD operations with schema normalization."""
+
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self.table = self._get_dynamo_table(kwargs["table"], kwargs.get("endpoint"))
+        self.schema_file: str = kwargs["schema_file"]
+        self.schema: str = kwargs["schema"]
+        self.identifier: str = kwargs["identifier"]
+        self.idempotence_key: Optional[str] = kwargs.get("idempotence_key")
+
+    @lru_cache(maxsize=128)
+    def _get_dynamo_table(self, table: str, endpoint: Optional[str] = None) -> Any:
+        return boto3.resource("dynamodb", endpoint_url=endpoint).Table(table)
+
+    def create(self, **kwargs: Any) -> DynamoItem:
+        if kwargs.get("operation") == "overwrite":
+            return self.overwrite(**kwargs)
+        return self.insert(**kwargs)
+
+    def read(self, **kwargs: Any) -> Union[DynamoItem, DynamoItems, Dict[str, Any]]:
+        if kwargs.get("operation") == "query":
+            return self.query(**kwargs)
+        if kwargs.get("operation") == "scan":
+            return self.scan(**kwargs)
+        return self.get(**kwargs)
+
+    def scan(self, **kwargs: Any) -> Union[DynamoItems, Dict[str, Any]]:
+        if kwargs.get("raw_scan"):
+            return self.table.scan(**kwargs.get("query", {}))
+        return self.table.scan(**kwargs.get("query", {})).get("Items", [])
+
+    def get(self, **kwargs: Any) -> DynamoItem:
+        result: Dict[str, Any] = self.table.get_item(**kwargs.get("query", {}))
+        return result.get("Item", {})
+
+    def query(self, **kwargs: Any) -> Union[DynamoItems, Dict[str, Any]]:
+        if kwargs.get("raw_query"):
+            return self.table.query(**kwargs.get("query", {}))
+        return self.table.query(**kwargs.get("query", {})).get("Items", [])
+
+    def overwrite(self, **kwargs: Any) -> DynamoItem:
+        overwrite_item = map_to_schema(kwargs["data"], self.schema_file, self.schema)
+        self.table.put_item(Item=overwrite_item)
+        super().publish("create", overwrite_item, **kwargs)
+        return overwrite_item
+
+    def insert(self, **kwargs: Any) -> DynamoItem:
+        new_item = map_to_schema(kwargs["data"], self.schema_file, self.schema)
+        self.table.put_item(
+            Item=new_item, ConditionExpression=Attr(self.identifier).not_exists()
+        )
+        super().publish("create", new_item, **kwargs)
+        return new_item
+
+    def batch_insert(self, **kwargs: Any) -> None:
+        data = kwargs["data"]
+        batch_size: int = kwargs.get("batch_size", 25)
+
+        if not isinstance(data, list):
+            raise BatchItemException("Batched data must be contained within a list")
+
+        batched_data: Iterable[DynamoItems] = (
+            data[pos : pos + batch_size] for pos in range(0, len(data), batch_size)
+        )
+        with self.table.batch_writer() as writer:
+            for batch in batched_data:
+                for item in batch:
+                    writer.put_item(Item=item)
+
+    def delete(self, **kwargs: Any) -> DynamoItem:
+        kwargs["query"]["ReturnValues"] = "ALL_OLD"
+        result = self.table.delete_item(**kwargs["query"]).get("Attributes", {})
+        super().publish("delete", result, **kwargs)
+        return result
+
+    def batch_delete(self, **kwargs: Any) -> None:
+        batch_size: int = kwargs.get("batch_size", 25)
+        if not isinstance(kwargs["data"], list):
+            raise BatchItemException("Batched data must be contained within a list")
+        batched_data: Iterable[DynamoItems] = (
+            kwargs["data"][pos : pos + batch_size]
+            for pos in range(0, len(kwargs["data"]), batch_size)
+        )
+        with self.table.batch_writer() as writer:
+            for batch in batched_data:
+                for item in batch:
+                    writer.delete_item(Key=item)
+
+    def update(self, **kwargs: Any) -> DynamoItem:
+        original_data = self._get_original_data(**kwargs)
+        merged_data = merge(original_data, kwargs["data"], **kwargs)
+        updated_data = map_to_schema(merged_data, self.schema_file, self.schema)
+        put_kwargs = {"Item": updated_data}
+        if self.idempotence_key:
+            original_value = original_data.get(self.idempotence_key)
+            if original_value is None:
+                raise ValueError(
+                    f"idempotence key '{self.idempotence_key}' not found in original item"
+                )
+            put_kwargs["ConditionExpression"] = cast(Any, Attr(self.idempotence_key).eq(original_value))
+        self.table.put_item(**put_kwargs)
+        super().publish("update", updated_data, **kwargs)
+        return updated_data
+
+    def _get_original_data(self, **kwargs: Any) -> DynamoItem:
+        if kwargs["operation"] == "get":
+            original_data = self.get(**kwargs)
+        else:
+            query_result = self.query(**kwargs)
+            if isinstance(query_result, list):
+                items = query_result
+            else:
+                items = cast(DynamoItems, query_result.get("Items", []))
+            if not items:
+                raise ValueError("update: no data found to update")
+            original_data = items[0]
+        if not original_data:
+            raise ValueError("update: no data found to update")
+        return original_data

daplug_ddb/common/__init__.py

@@ -0,0 +1,11 @@
+"""Shared utilities used by the DynamoDB adapter."""
+
+from .base_adapter import BaseAdapter
+from .dict_merger import merge
+from .schema_mapper import map_to_schema
+
+__all__ = [
+    "BaseAdapter",
+    "map_to_schema",
+    "merge",
+]

daplug_ddb/common/base_adapter.py

@@ -0,0 +1,57 @@
+"""Shared base adapter that wraps SNS publishing."""
+
+from typing import Any, Dict, Optional
+
+from daplug_ddb.types import MessageAttributes
+
+from . import publisher
+
+
+class BaseAdapter:
+    """Provides shared publish helper logic for adapters."""
+
+    def __init__(self, **kwargs: Any) -> None:
+        self.sns_arn: Optional[str] = kwargs.get("sns_arn")
+        self.sns_custom: Dict[str, Any] = kwargs.get("sns_attributes", {})
+        self.sns_defaults: bool = kwargs.get("sns_default_attributes", True)
+        self.sns_endpoint: Optional[str] = kwargs.get("sns_endpoint")
+        self.publisher = publisher
+        self.default_attributes: Dict[str, Any] = {
+            "schema": kwargs.get("schema"),
+            "identifier": kwargs.get("identifier"),
+            "idempotence_key": kwargs.get("idempotence_key"),
+            "author_identifier": kwargs.get("author_identifier"),
+        }
+
+    def publish(self, db_operation: str, db_data: Dict[str, Any], **kwargs: Any) -> None:
+        attributes = self.create_format_attibutes(db_operation)
+        self.publisher.publish(
+            endpoint=self.sns_endpoint,
+            arn=self.sns_arn,
+            attributes=attributes,
+            data=db_data,
+            fifo_group_id=kwargs.get("fifo_group_id"),
+            fifo_duplication_id=kwargs.get("fifo_duplication_id"),
+        )
+
+    def create_format_attibutes(self, operation: str) -> MessageAttributes:
+        self.default_attributes["operation"] = operation
+        custom_attributes = self.get_attributes()
+        formatted_attributes: MessageAttributes = {}
+        for key, value in custom_attributes.items():
+            if value is not None:
+                data_type = "String" if isinstance(value, str) else "Number"
+                formatted_attributes[key] = {
+                    "DataType": data_type,
+                    "StringValue": value,
+                }
+        return formatted_attributes
+
+    def get_attributes(self) -> Dict[str, Any]:
+        if self.sns_defaults and self.sns_custom:
+            return {**self.default_attributes, **self.sns_custom}
+        if not self.sns_defaults and self.sns_custom:
+            return self.sns_custom
+        if self.sns_defaults and not self.sns_custom:
+            return self.default_attributes
+        return {}

daplug_ddb/common/dict_merger.py

@@ -0,0 +1,66 @@
+"""Utilities to merge dictionaries and lists with optional strategies."""
+
+from __future__ import annotations
+
+import copy
+from typing import Any, List
+
+import simplejson as json
+
+from daplug_ddb.types import DynamoItem
+
+
+def merge(original_data: DynamoItem, new_data: DynamoItem, **kwargs: Any) -> DynamoItem:
+    updated_data = copy.deepcopy(original_data)
+    _walk_dict(updated_data, new_data, **kwargs)
+    return updated_data
+
+
+def _walk_dict(old_data: DynamoItem, new_data: DynamoItem, **kwargs: Any) -> None:
+    for new_key in new_data.keys():
+        if old_data.get(new_key) and isinstance(new_data[new_key], dict):
+            _walk_dict(old_data[new_key], new_data[new_key], **kwargs)
+        elif isinstance(old_data.get(new_key), list) and isinstance(new_data[new_key], list):
+            old_data[new_key] = _merge_lists(
+                old_data[new_key], new_data[new_key], kwargs.get("update_list_operation", "add")
+            )
+        else:
+            _merge_dicts(new_key, old_data, new_data, kwargs.get("update_dict_operation", "upsert"))
+
+
+def _merge_dicts(
+    dict_key: str,
+    old_dict: DynamoItem,
+    new_dict: DynamoItem,
+    update_dict_operation: str = "upsert",
+) -> None:
+    if update_dict_operation == "remove":
+        old_dict.pop(dict_key, None)
+    else:
+        old_dict[dict_key] = new_dict[dict_key]
+
+
+def _merge_lists(
+    old_list: List[Any], new_list: List[Any], update_list_operation: str = "add"
+) -> List[Any]:
+    if update_list_operation == "remove":
+        _remove_item_in_list(old_list, new_list)
+    elif update_list_operation == "add":
+        _add_unique_item_in_list(old_list, new_list)
+    elif update_list_operation == "replace":
+        old_list = new_list
+    return old_list
+
+
+def _remove_item_in_list(old_list: List[Any], new_list: List[Any]) -> None:
+    for old_item in old_list:
+        old = sorted(old_item.items()) if isinstance(old_item, dict) else old_item
+        new = sorted(new_list[0].items()) if isinstance(new_list[0], dict) else new_list[0]
+        if json.dumps(old) == json.dumps(new):
+            old_list.remove(old_item)
+
+
+def _add_unique_item_in_list(old_list: List[Any], new_list: List[Any]) -> None:
+    for item in new_list:
+        if item not in old_list:
+            old_list.append(item)

daplug_ddb/common/json_helper.py

@@ -0,0 +1,19 @@
+"""Safe JSON encoding/decoding helpers."""
+
+from typing import Any
+
+import json
+
+
+def try_decode_json(possible_json: Any) -> Any:
+    try:
+        return json.loads(possible_json)
+    except Exception:  # pylint: disable=broad-except
+        return possible_json
+
+
+def try_encode_json(possible_json: Any) -> Any:
+    try:
+        return json.dumps(possible_json)
+    except Exception:  # pylint: disable=broad-except
+        return possible_json

daplug_ddb/common/logger.py

@@ -0,0 +1,16 @@
+"""Lightweight stdout logger honoring RUN_MODE."""
+
+from typing import Any, Dict
+
+import os
+
+from . import json_helper
+
+
+def log(**kwargs: Any) -> None:
+    if os.getenv("RUN_MODE") != "unittest":
+        payload: Dict[str, Any] = {
+            "level": kwargs.get("level", "INFO"),
+            "log": kwargs.get("log", {}),
+        }
+        print(json_helper.try_encode_json(payload))

daplug_ddb/common/publisher.py

@@ -0,0 +1,29 @@
+"""SNS publisher helper used by adapters for event fan-out."""
+
+from typing import Any, Dict
+
+import boto3
+import simplejson as json
+
+from . import logger
+
+
+def publish(**kwargs: Any) -> None:
+    if not kwargs.get("arn") or not kwargs.get("data"):
+        return
+    try:
+        sns_client = boto3.client(
+            "sns", region_name=kwargs.get("region"), endpoint_url=kwargs.get("endpoint")
+        )
+        publish_kwargs: Dict[str, Any] = {
+            "TopicArn": kwargs["arn"],
+            "Message": json.dumps(kwargs["data"]),
+            "MessageAttributes": kwargs.get("attributes", {}),
+        }
+        if kwargs.get("fifo_group_id"):
+            publish_kwargs["MessageGroupId"] = kwargs["fifo_group_id"]
+        if kwargs.get("fifo_duplication_id"):
+            publish_kwargs["MessageDeduplicationId"] = kwargs["fifo_duplication_id"]
+        sns_client.publish(**publish_kwargs)
+    except Exception as exc:  # pylint: disable=broad-except
+        logger.log(level="WARN", log={"error": f"publish_sns_error: {exc}"})

daplug_ddb/common/schema_loader.py

@@ -0,0 +1,13 @@
+"""Loads schema definitions with JSON reference resolution."""
+
+from typing import Any, Dict
+
+import jsonref
+import simplejson as json
+import yaml
+
+
+def load_schema(schema_file: str, schema_key: str) -> Dict[str, Any]:
+    with open(schema_file, encoding="UTF-8") as openapi:
+        api_doc = yaml.load(openapi, Loader=yaml.FullLoader)
+    return jsonref.loads(json.dumps(api_doc))["components"]["schemas"][schema_key]

daplug_ddb/common/schema_mapper.py

@@ -0,0 +1,51 @@
+"""Helpers that map arbitrary data to JSON schema-defined structures."""
+
+from typing import Any, Dict, List
+
+from daplug_ddb.types import DynamoItem
+
+from . import schema_loader
+
+
+def map_to_schema(data: DynamoItem, schema_file: str, schema_key: str) -> DynamoItem:
+    model_data: DynamoItem = {}
+    model_schema = schema_loader.load_schema(schema_file, schema_key)
+    schemas = model_schema["allOf"] if model_schema.get("allOf") else [model_schema]
+    for model in schemas:
+        if model.get("type") == "object":
+            _populate_model_data(model.get("properties", {}), data, model_data)
+    return model_data
+
+
+def _populate_model_data(
+    properties: Dict[str, Any], data: Any, model_data: DynamoItem
+) -> DynamoItem:
+    if data and isinstance(data, dict):
+        _populate_model_dict(properties, data, model_data)
+    return model_data
+
+
+def _populate_model_dict(properties: Dict[str, Any], data: Dict[str, Any], model_data: DynamoItem) -> None:
+    for property_key, property_value in properties.items():
+        model_data[property_key] = {}
+        if property_value.get("properties"):
+            _populate_model_data(
+                property_value["properties"], data.get(property_key), model_data[property_key]
+            )
+        elif property_value.get("items", {}).get("properties"):
+            _populate_model_list(model_data, property_key, property_value, data)
+        else:
+            model_data[property_key] = data.get(property_key)
+
+
+def _populate_model_list(
+    model_data: DynamoItem, property_key: str, property_value: Dict[str, Any], data: Dict[str, Any]
+) -> None:
+    model_data[property_key] = []
+    items: List[Any] = data.get(property_key, [])
+    for index in range(len(items)):  # pylint: disable=consider-using-enumerate
+        if data.get(property_key) and isinstance(items, list) and index < len(items):
+            pop = _populate_model_data(
+                property_value["items"]["properties"], items[index], {}
+            )
+            model_data[property_key].append(pop)

daplug_ddb/exception.py

@@ -0,0 +1,2 @@
+class BatchItemException(Exception):
+    """Raised when batched operations receive invalid input."""

daplug_ddb/types/__init__.py

@@ -0,0 +1,7 @@
+"""Type exports for daplug_ddb."""
+
+from .dynamo_item import DynamoItem
+from .dynamo_items import DynamoItems
+from .message_attributes import MessageAttributes
+
+__all__ = ["DynamoItem", "DynamoItems", "MessageAttributes"]

daplug_ddb/types/dynamo_item.py

@@ -0,0 +1,7 @@
+"""Type alias for a DynamoDB item."""
+
+from typing import Any, Dict
+
+DynamoItem = Dict[str, Any]
+
+__all__ = ["DynamoItem"]

daplug_ddb/types/dynamo_items.py

@@ -0,0 +1,9 @@
+"""Type alias for a list of DynamoDB items."""
+
+from typing import List
+
+from .dynamo_item import DynamoItem
+
+DynamoItems = List[DynamoItem]
+
+__all__ = ["DynamoItems"]

daplug_ddb/types/message_attributes.py

@@ -0,0 +1,7 @@
+"""Type alias for SNS message attributes."""
+
+from typing import Any, Dict
+
+MessageAttributes = Dict[str, Dict[str, Any]]
+
+__all__ = ["MessageAttributes"]