daplug-ddb 1.0.0b8__py3-none-any.whl

daplug_ddb/__init__.py

@@ -0,0 +1,15 @@
+"""Public interface for the daplug_ddb package."""
+
+from typing import Any
+
+from .adapter import BatchItemException, DynamodbAdapter
+
+
+def adapter(**kwargs: Any) -> DynamodbAdapter:
+    """Factory helper for creating a DynamoDB adapter."""
+
+    kwargs.pop("engine", None)  # allow legacy callers to pass engine without effect
+    return DynamodbAdapter(**kwargs)
+
+
+__all__ = ["adapter", "DynamodbAdapter", "BatchItemException"]

daplug_ddb/adapter.py

@@ -0,0 +1,303 @@
+"""DynamoDB adapter exposing normalized CRUD operations."""
+
+from copy import deepcopy
+from datetime import datetime
+from functools import lru_cache
+from typing import Any, Dict, Iterable, Optional, Union
+
+import boto3
+from boto3.dynamodb.conditions import Attr
+
+from daplug_ddb.prefixer import DynamodbPrefixer
+from daplug_ddb.types import DynamoItem, DynamoItems
+
+from .common import map_to_schema, merge
+from .common.base_adapter import BaseAdapter
+from .exception import BatchItemException
+
+
+class DynamodbAdapter(BaseAdapter):
+    """Implements DynamoDB CRUD operations with schema normalization."""
+
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self.table = self.__get_dynamo_table(kwargs["table"], kwargs.get("endpoint"))
+        self.schema_file: Optional[str] = kwargs.get("schema_file")
+        self.identifier: str = kwargs.get("identifier", "")
+        self.idempotence_key: Optional[str] = kwargs.get("idempotence_key")
+        self.raise_idempotence_error: bool = kwargs.get("raise_idempotence_error", False)
+        self.idempotence_use_latest: bool = kwargs.get("idempotence_use_latest", False)
+        self._prefix_keys = ("hash_key", "hash_prefix", "range_key", "range_prefix")
+        self._default_prefix_config = self.__extract_prefix_config(kwargs)
+
+    @property
+    def prefixing_enabled(self) -> bool:
+        """Returns True when adapter-level prefix config has been supplied."""
+
+        return bool(self._default_prefix_config)
+
+    def create(self, **kwargs: Any) -> DynamoItem:
+        if kwargs.get("operation") == "overwrite":
+            return self.overwrite(**kwargs)
+        return self.insert(**kwargs)
+
+    def read(self, **kwargs: Any) -> Union[DynamoItem, DynamoItems, Dict[str, Any]]:
+        if kwargs.get("operation") == "query":
+            return self.query(**kwargs)
+        if kwargs.get("operation") == "scan":
+            return self.scan(**kwargs)
+        return self.get(**kwargs)
+
+    def scan(self, **kwargs: Any) -> Union[DynamoItems, Dict[str, Any]]:
+        prefixer = self.__build_prefixer(kwargs)
+        request_args = self.__prepare_request_arguments(prefixer, kwargs)
+        response = self.table.scan(**request_args)
+        if not prefixer.enabled:
+            return response if kwargs.get("raw_scan") else response.get("Items", [])
+        cleaned_response = prefixer.apply_response(response, add=False)
+        if kwargs.get("raw_scan"):
+            return cleaned_response if cleaned_response is not None else response
+        cleaned = cleaned_response or {}
+        return cleaned.get("Items", [])
+
+    def get(self, **kwargs: Any) -> DynamoItem:
+        prefixer = self.__build_prefixer(kwargs)
+        request_args = self.__prepare_request_arguments(prefixer, kwargs)
+        result: Dict[str, Any] = self.table.get_item(**request_args)
+        item = result.get("Item", {})
+        if not prefixer.enabled:
+            return item if isinstance(item, dict) else {}
+        cleaned = prefixer.apply_item(item, add=False)
+        return cleaned if isinstance(cleaned, dict) else item
+
+    def query(self, **kwargs: Any) -> Union[DynamoItems, Dict[str, Any]]:
+        prefixer = self.__build_prefixer(kwargs)
+        request_args = self.__prepare_request_arguments(prefixer, kwargs)
+        response = self.table.query(**request_args)
+        if not prefixer.enabled:
+            return response if kwargs.get("raw_query") else response.get("Items", [])
+        cleaned_response = prefixer.apply_response(response, add=False)
+        if kwargs.get("raw_query"):
+            return cleaned_response if cleaned_response is not None else response
+        cleaned = cleaned_response or {}
+        return cleaned.get("Items", [])
+
+    def overwrite(self, **kwargs: Any) -> DynamoItem:
+        payload = self.__map_with_schema(kwargs["data"], kwargs)
+        prefixer = self.__build_prefixer(kwargs)
+        item_to_store = (
+            prefixer.apply_item(payload, add=True)
+            if prefixer.enabled
+            else payload
+        )
+        self.table.put_item(Item=item_to_store)
+        response_item = (
+            prefixer.apply_item(item_to_store, add=False)
+            if prefixer.enabled
+            else payload
+        )
+        result_item = response_item if isinstance(response_item, dict) else payload
+        super().publish("create", result_item, **kwargs)
+        return result_item
+
+    def insert(self, **kwargs: Any) -> DynamoItem:
+        payload = self.__map_with_schema(kwargs["data"], kwargs)
+        prefixer = self.__build_prefixer(kwargs)
+        item_to_store = (
+            prefixer.apply_item(payload, add=True)
+            if prefixer.enabled
+            else payload
+        )
+        self.table.put_item(
+            Item=item_to_store,
+            ConditionExpression=Attr(self.identifier).not_exists(),
+        )
+        response_item = (
+            prefixer.apply_item(item_to_store, add=False)
+            if prefixer.enabled
+            else payload
+        )
+        result_item = response_item if isinstance(response_item, dict) else payload
+        super().publish("create", result_item, **kwargs)
+        return result_item
+
+    def batch_insert(self, **kwargs: Any) -> None:
+        data = kwargs["data"]
+        batch_size: int = kwargs.get("batch_size", 25)
+
+        if not isinstance(data, list):
+            raise BatchItemException("Batched data must be contained within a list")
+        mapped_items = [self.__map_with_schema(item, kwargs) for item in data]
+        prefixer = self.__build_prefixer(kwargs)
+        batched_data: Iterable[DynamoItems] = (
+            mapped_items[pos: pos + batch_size] for pos in range(0, len(mapped_items), batch_size)
+        )
+        with self.table.batch_writer() as writer:
+            for batch in batched_data:
+                if prefixer.enabled:
+                    prefixed_batch = prefixer.apply_items(batch, add=True)
+                    items_to_store = prefixed_batch if prefixed_batch is not None else batch
+                else:
+                    items_to_store = batch
+                for item in items_to_store:
+                    writer.put_item(Item=item)
+
+    def delete(self, **kwargs: Any) -> DynamoItem:
+        prefixer = self.__build_prefixer(kwargs)
+        request_args = self.__prepare_request_arguments(prefixer, kwargs)
+        request_args["ReturnValues"] = "ALL_OLD"
+        result = self.table.delete_item(**request_args).get("Attributes", {})
+        if prefixer.enabled:
+            cleaned = prefixer.apply_item(result, add=False)
+            cleaned_item = cleaned if isinstance(cleaned, dict) else result
+        else:
+            cleaned_item = result
+        super().publish("delete", cleaned_item, **kwargs)
+        return cleaned_item if isinstance(cleaned_item, dict) else {}
+
+    def batch_delete(self, **kwargs: Any) -> None:
+        batch_size: int = kwargs.get("batch_size", 25)
+        if not isinstance(kwargs["data"], list):
+            raise BatchItemException("Batched data must be contained within a list")
+        batched_data: Iterable[DynamoItems] = (
+            kwargs["data"][pos: pos + batch_size]
+            for pos in range(0, len(kwargs["data"]), batch_size)
+        )
+        prefixer = self.__build_prefixer(kwargs)
+        with self.table.batch_writer() as writer:
+            for batch in batched_data:
+                if prefixer.enabled:
+                    prefixed_batch = prefixer.apply_items(batch, add=True)
+                    items_to_delete = prefixed_batch if prefixed_batch is not None else batch
+                else:
+                    items_to_delete = batch
+                for item in items_to_delete:
+                    writer.delete_item(Key=item)
+
+    def update(self, **kwargs: Any) -> DynamoItem:
+        prefixer = self.__build_prefixer(kwargs)
+        original_data = self.__get_original_data(**kwargs)
+        merged_data = merge(original_data, kwargs["data"], **kwargs)
+        payload = self.__map_with_schema(merged_data, kwargs)
+        if prefixer.enabled:
+            prefixed_item = prefixer.apply_item(payload, add=True)
+            data_to_store = prefixed_item if isinstance(prefixed_item, dict) else payload
+            response_template_raw = prefixer.apply_item(data_to_store, add=False)
+            response_template = (
+                response_template_raw if isinstance(response_template_raw, dict) else payload
+            )
+        else:
+            data_to_store = payload
+            response_template = payload
+        key_name = self.idempotence_key
+        original_value = (
+            original_data.get(key_name)
+            if isinstance(original_data, dict) and key_name
+            else None
+        )
+        new_value = (
+            response_template.get(key_name)
+            if isinstance(response_template, dict) and key_name
+            else None
+        )
+        if self.__should_use_latest(original_value, new_value):
+            return self.__clean_for_response(prefixer, original_data)
+        put_kwargs = self.__build_put_kwargs(original_value, data_to_store)
+        self.table.put_item(**put_kwargs)
+        cleaned_item = self.__clean_for_response(prefixer, data_to_store)
+        super().publish("update", cleaned_item, **kwargs)
+        return cleaned_item
+
+    @lru_cache(maxsize=128)
+    def __get_dynamo_table(self, table: str, endpoint: Optional[str] = None) -> Any:
+        return boto3.resource("dynamodb", endpoint_url=endpoint).Table(table)
+
+    def __build_put_kwargs(self, original_value: Any, data_to_store: Dict[str, Any]) -> Dict[str, Any]:
+        put_kwargs: Dict[str, Any] = {"Item": data_to_store}
+        if not self.idempotence_key:
+            return put_kwargs
+        if original_value is None:
+            if self.raise_idempotence_error:
+                raise ValueError(
+                    f"idempotence key '{self.idempotence_key}' not found in original item"
+                )
+            return put_kwargs
+        if (
+            original_value != data_to_store.get(self.idempotence_key)
+            and self.raise_idempotence_error
+        ):
+            raise ValueError("update: idempotence key value has changed")
+        put_kwargs["ConditionExpression"] = Attr(self.idempotence_key).eq(original_value)
+        return put_kwargs
+
+    def __clean_for_response(self, prefixer: DynamodbPrefixer, item: Dict[str, Any]) -> Dict[str, Any]:
+        if not prefixer.enabled:
+            return item
+        cleaned = prefixer.apply_item(item, add=False)
+        return cleaned if isinstance(cleaned, dict) else item
+
+    def __get_original_data(self, **kwargs: Any) -> DynamoItem:
+        if kwargs["operation"] == "get":
+            original_data = self.get(**kwargs)
+        else:
+            query_result = self.query(**kwargs)
+            if isinstance(query_result, list):
+                items = query_result
+            elif isinstance(query_result, dict):
+                items = query_result.get("Items", [])
+            else:
+                items = []
+            if not items:
+                raise ValueError("update: no data found to update")
+            original_data = items[0]
+        if not original_data:
+            raise ValueError("update: no data found to update")
+        return original_data
+
+    def __should_use_latest(self, original_value: Any, new_value: Any) -> bool:
+        if not self.idempotence_use_latest or not self.idempotence_key:
+            return False
+        if original_value is None or new_value is None:
+            return False
+        try:
+            original_dt = datetime.fromisoformat(str(original_value))
+            new_dt = datetime.fromisoformat(str(new_value))
+        except ValueError as exc:
+            raise ValueError("idempotence_use_latest requires ISO date-compatible values") from exc
+        return original_dt > new_dt
+
+    def __map_with_schema(self, data: Dict[str, Any], call_kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        schema_file = call_kwargs.get("schema_file") or self.schema_file
+        schema_name = call_kwargs.get("schema")
+        if schema_file and schema_name:
+            return map_to_schema(data, schema_file, schema_name)
+        return deepcopy(data)
+
+    def __build_prefixer(self, call_kwargs: Dict[str, Any]) -> DynamodbPrefixer:
+        config = dict(self._default_prefix_config)
+        for key in self._prefix_keys:
+            value = call_kwargs.get(key)
+            if value is not None:
+                config[key] = value
+        return DynamodbPrefixer(**config)
+
+    def __extract_prefix_config(self, source: Dict[str, Any]) -> Dict[str, Any]:
+        config: Dict[str, Any] = {}
+        for key in ("hash_key", "hash_prefix", "range_key", "range_prefix"):
+            value = source.get(key)
+            if value is not None:
+                config[key] = value
+        return config
+
+    def __prepare_request_arguments(
+        self,
+        prefixer: DynamodbPrefixer,
+        call_kwargs: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        raw_query = call_kwargs.get("query") or {}
+        if not isinstance(raw_query, dict):
+            return {}
+        if prefixer.enabled:
+            transformed = prefixer.apply_request(raw_query, add=True)
+            return transformed if isinstance(transformed, dict) else {}
+        return deepcopy(raw_query)

daplug_ddb/common/__init__.py

@@ -0,0 +1,11 @@
+"""Shared utilities used by the DynamoDB adapter."""
+
+from .base_adapter import BaseAdapter
+from .dict_merger import merge
+from .schema_mapper import map_to_schema
+
+__all__ = [
+    "BaseAdapter",
+    "map_to_schema",
+    "merge",
+]

daplug_ddb/common/base_adapter.py

@@ -0,0 +1,69 @@
+"""Shared base adapter that wraps SNS publishing."""
+
+from typing import Any, Dict, Optional
+
+from daplug_ddb.types import MessageAttributes
+
+from . import publisher
+
+
+class BaseAdapter:
+    """Provides shared publish helper logic for adapters."""
+
+    def __init__(self, **kwargs: Any) -> None:
+        self.sns_arn: Optional[str] = kwargs.get("sns_arn")
+        self.sns_custom: Dict[str, Any] = kwargs.get("sns_attributes", {})
+        self.sns_defaults: bool = kwargs.get("sns_default_attributes", True)
+        self.sns_endpoint: Optional[str] = kwargs.get("sns_endpoint")
+        self.publisher = publisher
+        self.default_attributes: Dict[str, Any] = {
+            "identifier": kwargs.get("identifier"),
+            "idempotence_key": kwargs.get("idempotence_key"),
+            "author_identifier": kwargs.get("author_identifier"),
+        }
+
+    def publish(self, db_operation: str, db_data: Dict[str, Any], **kwargs: Any) -> None:
+        attributes = self.create_format_attibutes(db_operation, kwargs.get("sns_attributes", {}))
+        self.publisher.publish(
+            endpoint=self.sns_endpoint,
+            arn=self.sns_arn,
+            attributes=attributes,
+            data=db_data,
+            fifo_group_id=kwargs.get("fifo_group_id"),
+            fifo_duplication_id=kwargs.get("fifo_duplication_id"),
+        )
+
+    def create_format_attibutes(self, operation: str, call_attributes: dict) -> MessageAttributes:
+        combined = self.__combined_attributes(operation, call_attributes)
+        formatted_attributes: MessageAttributes = {}
+        for key, value in combined.items():
+            if value is not None:
+                data_type = "String" if isinstance(value, str) else "Number"
+                formatted_attributes[key] = {
+                    "DataType": data_type,
+                    "StringValue": value,
+                }
+        return formatted_attributes
+
+    def __combined_attributes(self, operation: str, call_attributes: dict) -> Dict[str, Any]:
+        pieces = []
+        base: Dict[str, Any] = {}
+        if self.sns_defaults:
+            base.update(self.default_attributes)
+        base["operation"] = operation
+        pieces.append(base)
+
+        if self.sns_custom:
+            pieces.append(self.sns_custom)
+        if call_attributes:
+            pieces.append(call_attributes)
+
+        return self.__merge_attributes(*pieces)
+
+    def __merge_attributes(self, *dicts: Dict[str, Any]) -> Dict[str, Any]:
+        merged: Dict[str, Any] = {}
+        for data in dicts:
+            if not data:
+                continue
+            merged.update(data)
+        return merged

daplug_ddb/common/dict_merger.py

@@ -0,0 +1,66 @@
+"""Utilities to merge dictionaries and lists with optional strategies."""
+
+from __future__ import annotations
+
+import copy
+from typing import Any, List
+
+import simplejson as json
+
+from daplug_ddb.types import DynamoItem
+
+
+def merge(original_data: DynamoItem, new_data: DynamoItem, **kwargs: Any) -> DynamoItem:
+    updated_data = copy.deepcopy(original_data)
+    _walk_dict(updated_data, new_data, **kwargs)
+    return updated_data
+
+
+def _walk_dict(old_data: DynamoItem, new_data: DynamoItem, **kwargs: Any) -> None:
+    for new_key in new_data.keys():
+        if old_data.get(new_key) and isinstance(new_data[new_key], dict):
+            _walk_dict(old_data[new_key], new_data[new_key], **kwargs)
+        elif isinstance(old_data.get(new_key), list) and isinstance(new_data[new_key], list):
+            old_data[new_key] = _merge_lists(
+                old_data[new_key], new_data[new_key], kwargs.get("update_list_operation", "add")
+            )
+        else:
+            _merge_dicts(new_key, old_data, new_data, kwargs.get("update_dict_operation", "upsert"))
+
+
+def _merge_dicts(
+    dict_key: str,
+    old_dict: DynamoItem,
+    new_dict: DynamoItem,
+    update_dict_operation: str = "upsert",
+) -> None:
+    if update_dict_operation == "remove":
+        old_dict.pop(dict_key, None)
+    else:
+        old_dict[dict_key] = new_dict[dict_key]
+
+
+def _merge_lists(
+    old_list: List[Any], new_list: List[Any], update_list_operation: str = "add"
+) -> List[Any]:
+    if update_list_operation == "remove":
+        _remove_item_in_list(old_list, new_list)
+    elif update_list_operation == "add":
+        _add_unique_item_in_list(old_list, new_list)
+    elif update_list_operation == "replace":
+        old_list = new_list
+    return old_list
+
+
+def _remove_item_in_list(old_list: List[Any], new_list: List[Any]) -> None:
+    for old_item in old_list:
+        old = sorted(old_item.items()) if isinstance(old_item, dict) else old_item
+        new = sorted(new_list[0].items()) if isinstance(new_list[0], dict) else new_list[0]
+        if json.dumps(old) == json.dumps(new):
+            old_list.remove(old_item)
+
+
+def _add_unique_item_in_list(old_list: List[Any], new_list: List[Any]) -> None:
+    for item in new_list:
+        if item not in old_list:
+            old_list.append(item)

daplug_ddb/common/json_helper.py

@@ -0,0 +1,19 @@
+"""Safe JSON encoding/decoding helpers."""
+
+from typing import Any
+
+import json
+
+
+def try_decode_json(possible_json: Any) -> Any:
+    try:
+        return json.loads(possible_json)
+    except Exception:  # pylint: disable=broad-except
+        return possible_json
+
+
+def try_encode_json(possible_json: Any) -> Any:
+    try:
+        return json.dumps(possible_json)
+    except Exception:  # pylint: disable=broad-except
+        return possible_json

daplug_ddb/common/logger.py

@@ -0,0 +1,16 @@
+"""Lightweight stdout logger honoring RUN_MODE."""
+
+from typing import Any, Dict
+
+import os
+
+from . import json_helper
+
+
+def log(**kwargs: Any) -> None:
+    if os.getenv("RUN_MODE") != "unittest":
+        payload: Dict[str, Any] = {
+            "level": kwargs.get("level", "INFO"),
+            "log": kwargs.get("log", {}),
+        }
+        print(json_helper.try_encode_json(payload))

daplug_ddb/common/publisher.py

@@ -0,0 +1,29 @@
+"""SNS publisher helper used by adapters for event fan-out."""
+
+from typing import Any, Dict
+
+import boto3
+import simplejson as json
+
+from . import logger
+
+
+def publish(**kwargs: Any) -> None:
+    if not kwargs.get("arn") or not kwargs.get("data"):
+        return
+    try:
+        sns_client = boto3.client(
+            "sns", region_name=kwargs.get("region"), endpoint_url=kwargs.get("endpoint")
+        )
+        publish_kwargs: Dict[str, Any] = {
+            "TopicArn": kwargs["arn"],
+            "Message": json.dumps(kwargs["data"]),
+            "MessageAttributes": kwargs.get("attributes", {}),
+        }
+        if kwargs.get("fifo_group_id"):
+            publish_kwargs["MessageGroupId"] = kwargs["fifo_group_id"]
+        if kwargs.get("fifo_duplication_id"):
+            publish_kwargs["MessageDeduplicationId"] = kwargs["fifo_duplication_id"]
+        sns_client.publish(**publish_kwargs)
+    except Exception as exc:  # pylint: disable=broad-except
+        logger.log(level="WARN", log={"error": f"publish_sns_error: {exc}"})

daplug_ddb/common/schema_loader.py

@@ -0,0 +1,13 @@
+"""Loads schema definitions with JSON reference resolution."""
+
+from typing import Any, Dict
+
+import jsonref
+import simplejson as json
+import yaml
+
+
+def load_schema(schema_file: str, schema_key: str) -> Dict[str, Any]:
+    with open(schema_file, encoding="UTF-8") as openapi:
+        api_doc = yaml.load(openapi, Loader=yaml.FullLoader)
+    return jsonref.loads(json.dumps(api_doc))["components"]["schemas"][schema_key]

daplug_ddb/common/schema_mapper.py

@@ -0,0 +1,51 @@
+"""Helpers that map arbitrary data to JSON schema-defined structures."""
+
+from typing import Any, Dict, List
+
+from daplug_ddb.types import DynamoItem
+
+from . import schema_loader
+
+
+def map_to_schema(data: DynamoItem, schema_file: str, schema_key: str) -> DynamoItem:
+    model_data: DynamoItem = {}
+    model_schema = schema_loader.load_schema(schema_file, schema_key)
+    schemas = model_schema["allOf"] if model_schema.get("allOf") else [model_schema]
+    for model in schemas:
+        if model.get("type") == "object":
+            _populate_model_data(model.get("properties", {}), data, model_data)
+    return model_data
+
+
+def _populate_model_data(
+    properties: Dict[str, Any], data: Any, model_data: DynamoItem
+) -> DynamoItem:
+    if data and isinstance(data, dict):
+        _populate_model_dict(properties, data, model_data)
+    return model_data
+
+
+def _populate_model_dict(properties: Dict[str, Any], data: Dict[str, Any], model_data: DynamoItem) -> None:
+    for property_key, property_value in properties.items():
+        model_data[property_key] = {}
+        if property_value.get("properties"):
+            _populate_model_data(
+                property_value["properties"], data.get(property_key), model_data[property_key]
+            )
+        elif property_value.get("items", {}).get("properties"):
+            _populate_model_list(model_data, property_key, property_value, data)
+        else:
+            model_data[property_key] = data.get(property_key)
+
+
+def _populate_model_list(
+    model_data: DynamoItem, property_key: str, property_value: Dict[str, Any], data: Dict[str, Any]
+) -> None:
+    model_data[property_key] = []
+    items: List[Any] = data.get(property_key, [])
+    for index in range(len(items)):  # pylint: disable=consider-using-enumerate
+        if data.get(property_key) and isinstance(items, list) and index < len(items):
+            pop = _populate_model_data(
+                property_value["items"]["properties"], items[index], {}
+            )
+            model_data[property_key].append(pop)

daplug_ddb/exception.py

@@ -0,0 +1,2 @@
+class BatchItemException(Exception):
+    """Raised when batched operations receive invalid input."""