boto3-assist 0.32.0__py3-none-any.whl

boto3_assist/dynamodb/dynamodb_model_base.py

@@ -0,0 +1,382 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from __future__ import annotations
+import datetime as dt
+
+# import decimal
+# import inspect
+# import uuid
+from typing import TypeVar, List, Dict, Any
+from boto3.dynamodb.types import TypeSerializer, TypeDeserializer
+from boto3_assist.utilities.serialization_utility import Serialization
+from boto3_assist.utilities.decimal_conversion_utility import DecimalConversionUtility
+from boto3_assist.dynamodb.dynamodb_helpers import DynamoDBHelpers
+from boto3_assist.dynamodb.dynamodb_index import (
+    DynamoDBIndexes,
+    DynamoDBIndex,
+)
+from boto3_assist.dynamodb.dynamodb_reserved_words import DynamoDBReservedWords
+from boto3_assist.utilities.datetime_utility import DatetimeUtility
+from boto3_assist.models.serializable_model import SerializableModel
+from boto3_assist.utilities.string_utility import StringUtility
+
+
+def exclude_from_serialization(method):
+    """
+    Decorator to mark methods or properties to be excluded from serialization.
+    """
+    method.exclude_from_serialization = True
+    return method
+
+
+def exclude_indexes_from_serialization(method):
+    """
+    Decorator to mark methods or properties to be excluded from serialization.
+    """
+    method.exclude_indexes_from_serialization = True
+    return method
+
+
+class DynamoDBModelBase(SerializableModel):
+    """DynamoDb Model Base"""
+
+    T = TypeVar("T", bound="DynamoDBModelBase")
+
+    def __init__(self, auto_generate_projections: bool = True) -> None:
+        self.__projection_expression: str | None = None
+        self.__projection_expression_attribute_names: dict | None = None
+        self.__helpers: DynamoDBHelpers | None = None
+        self.__indexes: DynamoDBIndexes | None = None
+        self.__reserved_words: DynamoDBReservedWords = DynamoDBReservedWords()
+        self.__auto_generate_projections: bool = auto_generate_projections
+        self.__actively_serializing_data__: bool = False
+
+    def serialization_in_progress(self) -> bool:
+        return self.__actively_serializing_data__
+
+    @property
+    @exclude_from_serialization
+    def indexes(self) -> DynamoDBIndexes:
+        """Gets the indexes"""
+        # although this is marked as excluded, the indexes are add
+        # but in a more specialized way
+        if self.__indexes is None:
+            self.__indexes = DynamoDBIndexes()
+        return self.__indexes
+
+    @property
+    @exclude_from_serialization
+    def projection_expression(self) -> str | None:
+        """Gets the projection expression"""
+        prop_list: List[str] = []
+        if self.__projection_expression is None and self.auto_generate_projections:
+            props = self.to_dictionary()
+            # turn props to a list[str]
+            prop_list = list(props.keys())
+        else:
+            if self.__projection_expression:
+                prop_list = self.__projection_expression.split(",")
+                prop_list = [p.strip() for p in prop_list]
+
+        if len(prop_list) == 0:
+            return None
+
+        transformed_list = self.__reserved_words.tranform_projections(prop_list)
+        self.projection_expression = ",".join(transformed_list)
+
+        return self.__projection_expression
+
+    @projection_expression.setter
+    def projection_expression(self, value: str | None):
+        self.__projection_expression = value
+
+    @property
+    @exclude_from_serialization
+    def auto_generate_projections(self) -> bool:
+        """Gets the auto generate projections"""
+        return self.__auto_generate_projections
+
+    @auto_generate_projections.setter
+    def auto_generate_projections(self, value: bool):
+        self.__auto_generate_projections = value
+
+    @property
+    @exclude_from_serialization
+    def projection_expression_attribute_names(self) -> dict | None:
+        """
+        Gets the projection expression attribute names
+
+        """
+        if (
+            self.__projection_expression_attribute_names is None
+            and self.auto_generate_projections
+        ):
+            props = self.to_dictionary()
+            # turn props to a list[str]
+            prop_list = list(props.keys())
+            self.projection_expression_attribute_names = (
+                self.__reserved_words.transform_attributes(prop_list)
+            )
+        else:
+            if self.projection_expression:
+                expression_list = self.projection_expression.replace("#", "").split(",")
+                self.projection_expression_attribute_names = (
+                    self.__reserved_words.transform_attributes(expression_list)
+                )
+
+        return self.__projection_expression_attribute_names
+
+    @projection_expression_attribute_names.setter
+    def projection_expression_attribute_names(self, value: dict | None):
+        self.__projection_expression_attribute_names = value
+
+    def map(self: T, item: Dict[str, Any] | DynamoDBModelBase | None) -> T:
+        """
+        Map the item to the instance.  If the item is a DynamoDBModelBase,
+        it will be converted to a dictionary first and then mapped.
+
+        Args:
+            self (T): The Type of object you are converting it to.
+            item (dict | DynamoDBModelBase): _description_
+
+        Raises:
+            ValueError: If the object is not a dictionary or DynamoDBModelBase
+
+        Returns:
+            T | None: An object of type T with properties set matching
+            that of the dictionary object or None
+        """
+        if item is None:
+            item = {}
+
+        if isinstance(item, DynamoDBModelBase):
+            item = item.to_resource_dictionary()
+
+        if isinstance(item, dict):
+            # Handle DynamoDB response structures
+            if "ResponseMetadata" in item:
+                # Full DynamoDB response with metadata
+                response: dict | None = item.get("Item")
+                if response is None:
+                    response = {}
+                item = response
+            elif "Item" in item and not any(
+                key in item for key in ["id", "name", "pk", "sk"]
+            ):
+                # Response with Item key but no direct model attributes (likely a DynamoDB response)
+                # This handles cases like {'Item': {...}} or {'Item': {...}, 'Count': 1}
+                item = item.get("Item", {})
+
+            # Convert any Decimal objects to native Python types for easier handling
+            item = DecimalConversionUtility.convert_decimals_to_native_types(item)
+
+        else:
+            raise ValueError("Item must be a dictionary or DynamoDBModelBase")
+        # attempt to map it
+        return DynamoDBSerializer.map(source=item, target=self)
+
+    def to_client_dictionary(self, include_indexes: bool = True):
+        """
+        Convert the instance to a dictionary suitable for DynamoDB client.
+        """
+        return DynamoDBSerializer.to_client_dictionary(
+            self, include_indexes=include_indexes
+        )
+
+    def to_resource_dictionary(
+        self, include_indexes: bool = True, include_none: bool = False
+    ):
+        """
+        Convert the instance to a dictionary suitable for DynamoDB resource.
+        """
+        return DynamoDBSerializer.to_resource_dictionary(
+            self, include_indexes=include_indexes, include_none=include_none
+        )
+
+    def to_dict(self, include_none: bool = True):
+        """
+        Convert the instance to a dictionary suitable for DynamoDB client.
+        """
+        return self.to_dictionary(include_none=include_none)
+
+    def to_dictionary(self, include_none: bool = True):
+        """
+        Convert the instance to a dictionary without an indexes/keys.
+        Useful for turning an object into a dictionary for serialization.
+        This is the same as to_resource_dictionary(include_indexes=False)
+        """
+        return DynamoDBSerializer.to_resource_dictionary(
+            self, include_indexes=False, include_none=include_none
+        )
+
+    def get_key(self, index_name: str) -> DynamoDBIndex:
+        """Get the index name and key"""
+
+        if index_name is None:
+            raise ValueError("Index name cannot be None")
+
+        return self.indexes.get(index_name)
+
+    @staticmethod
+    def generate_uuid(sortable: bool = True) -> str:
+        if sortable:
+            return StringUtility.generate_sortable_uuid()
+
+        return StringUtility.generate_uuid()
+
+    @property
+    @exclude_from_serialization
+    def helpers(self) -> DynamoDBHelpers:
+        """Get the helpers"""
+        if self.__helpers is None:
+            self.__helpers = DynamoDBHelpers()
+        return self.__helpers
+
+    def list_keys(self, exclude_pk: bool = False) -> List[DynamoDBIndex]:
+        """List the keys"""
+        values = self.indexes.values()
+        if exclude_pk:
+            values = [v for v in values if not v.name == DynamoDBIndexes.PRIMARY_INDEX]
+
+        return values
+
+    def to_timestamp_or_none(self, value: str | dt.datetime | None) -> float | None:
+        """
+        Convert a value to a timestamp (float) or None
+
+        Exceptions:
+            ValueError: If the value is not a datetime string or datetime
+        """
+
+        if isinstance(value, str):
+            # value = dt.datetime.fromisoformat(value)
+            value = DatetimeUtility.to_datetime_utc(value)
+
+        if value is None:
+            return None
+
+        if isinstance(value, dt.datetime):
+            return value.timestamp()
+
+        raise ValueError(
+            "Value must be a None, a string in a valid datetime format or datetime"
+        )
+
+    def to_utc(self, value: str | dt.datetime | None) -> dt.datetime | None:
+        """
+        Convert a datetime to UTC. This ensures all datetimes are stored in UTC format
+
+        Exceptions:
+            ValueError: If the value is not a datetime string or datetime
+        """
+
+        value = DatetimeUtility.to_datetime_utc(value)
+        return value
+
+
+class DynamoDBSerializer:
+    """Library to Serialize object to a DynamoDB Format"""
+
+    T = TypeVar("T", bound=DynamoDBModelBase)
+
+    @staticmethod
+    def map(source: dict, target: T) -> T:
+        """
+        Map the source dictionary to the target object.
+
+        Args:
+        - source: The dictionary to map from.
+        - target: The object to map to.
+        """
+        mapped = Serialization.map(source, target)
+        if mapped is None:
+            raise ValueError("Unable to map source to target")
+
+        return mapped
+
+    @staticmethod
+    def to_client_dictionary(
+        instance: DynamoDBModelBase, include_indexes: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Convert a Python class instance to a dictionary suitable for DynamoDB client.
+
+        Args:
+        - instance: The class instance to be converted.
+
+        Returns:
+        - dict: A dictionary representation of the class instance suitable for DynamoDB client.
+        """
+        serializer = TypeSerializer()
+        d = Serialization.to_dict(instance, serializer.serialize)
+
+        if include_indexes:
+            d = DynamoDBSerializer._add_indexes(instance=instance, instance_dict=d)
+
+        return d
+
+    @staticmethod
+    def to_resource_dictionary(
+        instance: DynamoDBModelBase,
+        include_indexes: bool = True,
+        include_none: bool = False,
+    ) -> Dict[str, Any]:
+        """
+        Convert a Python class instance to a dictionary suitable for DynamoDB resource.
+
+        Args:
+        - instance: The class instance to be converted.
+
+        Returns:
+        - dict: A dictionary representation of the class instance suitable for DynamoDB resource.
+        """
+        d = Serialization.to_dict(
+            instance,
+            lambda x: x,
+            include_none=include_none,
+        )
+
+        if include_indexes:
+            d = DynamoDBSerializer._add_indexes(instance=instance, instance_dict=d)
+
+        return d
+
+    @staticmethod
+    def _add_indexes(instance: DynamoDBModelBase, instance_dict: dict) -> dict:
+        if not issubclass(type(instance), DynamoDBModelBase):
+            return instance_dict
+
+        if instance.indexes is None:
+            return instance_dict
+
+        primary = instance.indexes.primary
+
+        if primary:
+            instance_dict[primary.partition_key.attribute_name] = (
+                primary.partition_key.value
+            )
+            if (
+                primary.sort_key.attribute_name is not None
+                and primary.sort_key.value is not None
+            ):
+                instance_dict[primary.sort_key.attribute_name] = primary.sort_key.value
+
+        secondaries = instance.indexes.secondaries
+
+        key: DynamoDBIndex
+        for _, key in secondaries.items():
+            if (
+                key.partition_key.attribute_name is not None
+                and key.partition_key.value is not None
+            ):
+                instance_dict[key.partition_key.attribute_name] = (
+                    key.partition_key.value
+                )
+            if key.sort_key.value is not None and key.sort_key.value is not None:
+                instance_dict[key.sort_key.attribute_name] = key.sort_key.value
+
+        return instance_dict

boto3_assist/dynamodb/dynamodb_model_base_interfaces.py

@@ -0,0 +1,34 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+from typing import Protocol, Optional
+from boto3.dynamodb.conditions import (
+    And,
+    Equals,
+    # NotEquals,
+    # Or,
+    # GreaterThan,
+    # GreaterThanEquals,
+    # LessThan,
+    # LessThanEquals,
+    # In,
+    # Between,
+    # Contains,
+    # BeginsWith,
+)
+
+
+class HasKeys(Protocol):
+    """Interface for classes that have primary and sort keys"""
+
+    def get_pk(self, index_name: str) -> Optional[str]:
+        """Interface to get_pk"""
+
+    def get_sk(self, index_name: str) -> Optional[str]:
+        """Interface to get_sk"""
+
+    def get_key(self, index_name: str) -> And | Equals:
+        """Get the index name and key"""

boto3_assist/dynamodb/dynamodb_re_indexer.py

@@ -0,0 +1,165 @@
+"""
+Geek Cafe, LLC
+Maintainers: Eric Wilson
+MIT License.  See Project Root for the license information.
+"""
+
+import json
+from typing import Any, Dict, Optional, List, Type
+from aws_lambda_powertools import Logger
+from boto3_assist.dynamodb.dynamodb import DynamoDB
+from boto3_assist.dynamodb.dynamodb_model_base import DynamoDBModelBase
+from boto3_assist.utilities.serialization_utility import Serialization
+from boto3_assist.dynamodb.dynamodb_index import DynamoDBIndex
+from boto3_assist.dynamodb.dynamodb_iservice import IDynamoDBService
+
+logger = Logger()
+
+
+class DynamoDBReIndexer:
+    """ReIndexing your database"""
+
+    def __init__(
+        self,
+        table_name: str,
+        *,
+        db: Optional[DynamoDB] = None,
+        aws_profile: Optional[str] = None,
+        aws_region: Optional[str] = None,
+        aws_end_point_url: Optional[str] = None,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+    ):
+        self.table_name = table_name
+        self.db: DynamoDB = db or DynamoDB(
+            aws_profile=aws_profile,
+            aws_region=aws_region,
+            aws_end_point_url=aws_end_point_url,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+
+    def reindex_item(
+        self,
+        original_primary_key: dict,
+        model: DynamoDBModelBase,
+        *,
+        dry_run: bool = False,
+        inplace: bool = True,
+        leave_original_record: bool = False,
+        service_cls: Type[IDynamoDBService] | None,
+    ):
+        """
+        Reindex the record
+
+        Args:
+            original_primary_key (dict): The original primary key of the record to be reindexed.
+                This is either the partition_key or a composite key (partition_key, sort_key)
+            model (DynamoDBModelBase): A model instance that will be used to serialize the new keys
+                into a dictionary. It must inherit from DynamoDBModelBase
+
+            dry_run (bool, optional): Ability to log the actions without executing them. Defaults to False.
+            inplace (bool, optional): Ability to just update the indexes only.
+                No other fields will be updated, however you can't update the primary_key (partition/sort key)
+                with this action since they are immutable.
+                Defaults to True.
+            leave_original_record (bool, optional): _description_. Defaults to False.
+        """
+
+        if inplace:
+            keys: List[DynamoDBIndex] = model.list_keys()
+            # Update the item in DynamoDB with new keys
+            self.update_item_in_dynamodb(
+                original_primary_key=original_primary_key, keys=keys, dry_run=dry_run
+            )
+            # todo: add some additional error handling here and throw a more
+            # descriptive error if they try to use a different primary
+            # pk or sk, which you can't do.  If that's the case
+        else:
+            # add the new one first and optionally delete the older one
+            # once we are successful
+            try:
+                # save the new one first
+                service_instance: Optional[IDynamoDBService] = (
+                    service_cls(db=self.db) if callable(service_cls) else None
+                )
+
+                if service_instance:
+                    service_instance.save(model=model)
+                else:
+                    self.db.save(
+                        item=model, table_name=self.table_name, source="reindex"
+                    )
+
+                # then delete the old on
+                if not leave_original_record:
+                    self.db.delete(
+                        table_name=self.table_name, primary_key=original_primary_key
+                    )
+            except Exception as e:  # pylint: disable=broad-except
+                logger.error(str(e))
+                raise RuntimeError(str(e)) from e
+            # this gets a little more trick as we need to delete the item
+
+    def load_model(
+        self, db_item: dict, db_model: DynamoDBModelBase
+    ) -> DynamoDBModelBase | None:
+        """load the model which will serialze the dynamodb dictionary to an instance of an object"""
+
+        base_model = Serialization.map(db_item, db_model)
+        return base_model
+
+    def update_item_in_dynamodb(
+        self,
+        original_primary_key: dict,
+        keys: List[DynamoDBIndex],
+        dry_run: bool = False,
+    ):
+        """Update the dynamodb item"""
+        dictionary = self.db.helpers.keys_to_dictionary(keys=keys)
+
+        update_expression = self.build_update_expression(dictionary)
+        expression_attribute_values = self.build_expression_attribute_values(dictionary)
+
+        if not dry_run:
+            self.db.update_item(
+                table_name=self.table_name,
+                key=original_primary_key,
+                update_expression=update_expression,
+                expression_attribute_values=expression_attribute_values,
+            )
+        else:
+            print("Dry run: Skipping Update item")
+            print(f"{json.dumps(original_primary_key, indent=4)}")
+            print(f"{update_expression}")
+            print(f"{json.dumps(expression_attribute_values, indent=4)}")
+
+    def build_update_expression(self, updated_keys: Dict[str, Any]) -> str:
+        """
+        Build the expression for updating the item
+
+        Args:
+            updated_keys (Dict[str, Any]): _description_
+
+        Returns:
+            str: _description_
+        """
+        update_expression = "SET " + ", ".join(
+            f"{k} = :{k}" for k in updated_keys.keys()
+        )
+        return update_expression
+
+    def build_expression_attribute_values(
+        self, updated_keys: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        Build the expression attribute values for the update expression
+
+        Args:
+            updated_keys (Dict[str, Any]): _description_
+
+        Returns:
+            Dict[str, Any]: _description_
+        """
+        expression_attribute_values = {f":{k}": v for k, v in updated_keys.items()}
+        return expression_attribute_values