clear-skies-aws 1.10.2__py3-none-any.whl → 2.0.1__py3-none-any.whl

clearskies_aws/backends/dynamo_db_backend_test.py

@@ -1,300 +0,0 @@
-import unittest
-from collections import OrderedDict
-from decimal import Decimal
-from types import SimpleNamespace
-from unittest.mock import MagicMock
-
-import clearskies
-from boto3.dynamodb import conditions as dynamodb_conditions
-
-from ..di import StandardDependencies
-
-
-class User(clearskies.Model):
-    def __init__(self, dynamo_db_backend, columns):
-        super().__init__(dynamo_db_backend, columns)
-
-    def columns_configuration(self):
-        return OrderedDict([
-            clearskies.column_types.string('name'),
-            clearskies.column_types.string('category_id'),
-            clearskies.column_types.integer('age'),
-        ])
-class Users(clearskies.Models):
-    def __init__(self, dynamo_db_backend, columns):
-        super().__init__(dynamo_db_backend, columns)
-
-    def model_class(self):
-        return User
-class DynamoDBBackendTest(unittest.TestCase):
-    def setUp(self):
-        self.di = StandardDependencies()
-        self.di.bind('environment', {'AWS_REGION': 'us-east-2'})
-        self.dynamo_db_table = SimpleNamespace(
-            key_schema=[{
-                'KeyType': 'HASH',
-                'AttributeName': 'id'
-            }],
-            global_secondary_indexes=[
-                {
-                    'IndexName':
-                    'category_id-name-index',
-                    'KeySchema': [
-                        {
-                            'KeyType': 'HASH',
-                            'AttributeName': 'category_id'
-                        },
-                        {
-                            'KeyType': 'RANGE',
-                            'AttributeName': 'name'
-                        },
-                    ]
-                },
-                {
-                    'IndexName':
-                    'category_id-age-index',
-                    'KeySchema': [
-                        {
-                            'KeyType': 'RANGE',
-                            'AttributeName': 'age'
-                        },
-                        {
-                            'KeyType': 'HASH',
-                            'AttributeName': 'category_id'
-                        },
-                    ]
-                },
-            ],
-            local_secondary_indexes=[{
-                'IndexName':
-                'id-category_id-index',
-                'KeySchema': [
-                    {
-                        'KeyType': 'RANGE',
-                        'AttributeName': 'category_id'
-                    },
-                    {
-                        'KeyType': 'HASH',
-                        'AttributeName': 'id'
-                    },
-                ]
-            }],
-        )
-        self.dynamo_db = SimpleNamespace(Table=MagicMock(return_value=self.dynamo_db_table), )
-        self.boto3 = SimpleNamespace(resource=MagicMock(return_value=self.dynamo_db), )
-        self.di.bind('boto3', self.boto3)
-
-    def test_create(self):
-        self.dynamo_db_table.put_item = MagicMock()
-        user = self.di.build(User)
-        user.save({'name': 'sup', 'age': 5, 'category_id': '1-2-3-4'})
-        self.boto3.resource.assert_called_with('dynamodb', region_name='us-east-2')
-        self.dynamo_db.Table.assert_called_with('users')
-        self.assertEqual(1, len(self.dynamo_db_table.put_item.call_args_list))
-        call = self.dynamo_db_table.put_item.call_args_list[0]
-        self.assertEqual((), call.args)
-        self.assertEqual(1, len(call.kwargs))
-        self.assertTrue('Item' in call.kwargs)
-        saved_data = call.kwargs['Item']
-        # we're doing this a bit weird because the UUIDs will generate random values.
-        # I could mock it, or I could just be lazy and grab it from the data I was given.
-        self.assertEqual(
-            {
-                "id": saved_data["id"],
-                "name": "sup",
-                "age": 5,
-                "category_id": "1-2-3-4",
-            },
-            saved_data,
-        )
-        self.assertEqual(saved_data["id"], user.id)
-        self.assertEqual(5, user.age)
-        self.assertEqual("1-2-3-4", user.category_id)
-        self.assertEqual("sup", user.name)
-
-    def test_update(self):
-        self.dynamo_db_table.update_item = MagicMock(
-            return_value={
-                'Attributes': {
-                    'id': '1-2-3-4',
-                    'name': 'hello',
-                    'age': Decimal('10'),
-                    'category_id': '1-2-3-5',
-                }
-            }
-        )
-        user = self.di.build(User)
-        user.data = {'id': '1-2-3-4', 'name': 'sup', 'age': 5, 'category_id': '1-2-3-5'}
-        user.save({'name': 'hello', 'age': 10})
-        self.boto3.resource.assert_called_with('dynamodb', region_name='us-east-2')
-        self.dynamo_db.Table.assert_called_with('users')
-        self.dynamo_db_table.update_item.assert_called_with(
-            Key={'id': '1-2-3-4'},
-            UpdateExpression='SET #name = :name, #age = :age',
-            ExpressionAttributeValues={
-                ':name': 'hello',
-                ':age': 10
-            },
-            ExpressionAttributeNames={
-                '#name': 'name',
-                '#age': 'age'
-            },
-            ReturnValues="ALL_NEW",
-        )
-        self.assertEqual("1-2-3-4", user.id)
-        self.assertEqual("hello", user.name)
-        self.assertEqual(10, user.age)
-        self.assertEqual("1-2-3-5", user.category_id)
-
-    def test_delete(self):
-        self.dynamo_db_table.delete_item = MagicMock()
-        user = self.di.build(User)
-        user.data = {'id': '1-2-3-4', 'name': 'sup', 'age': 5, 'category_id': '1-2-3-5'}
-        user.delete()
-        self.dynamo_db_table.delete_item.assert_called_with(Key={'id': '1-2-3-4'}, )
-
-    def test_fetch_by_id(self):
-        self.dynamo_db_table.query = MagicMock(
-            return_value={'Items': [{
-                'id': '1-2-3-4',
-                'age': Decimal(10),
-                'category_id': '4-5-6'
-            }]}
-        )
-        users = self.di.build(Users)
-        user = users.where('id=1-2-3-4').first()
-        self.assertTrue(user.exists)
-        self.assertEqual("1-2-3-4", user.id)
-        self.assertEqual(10, user.age)
-        self.assertEqual("4-5-6", user.category_id)
-        self.assertEqual(1, len(self.dynamo_db_table.query.call_args_list))
-        call = self.dynamo_db_table.query.call_args_list[0]
-        self.assertEqual(3, len(call.kwargs))
-        self.assertEqual("ALL_ATTRIBUTES", call.kwargs["Select"])
-        self.assertEqual(True, call.kwargs["ScanIndexForward"])
-        key_condition = call.kwargs['KeyConditionExpression']
-
-        # dynamodb does not make it easy to verify that the key expression was built properly,
-        # and I don't think that mocking will make this any better.  Hoepfully they don't
-        # change their library often...
-        key_column = key_condition.get_expression()['values'][0]
-        self.assertEqual("id", key_column.name)
-        self.assertTrue(isinstance(key_column, dynamodb_conditions.Key))
-        self.assertEqual("=", key_condition.expression_operator)
-        self.assertEqual("1-2-3-4", key_condition.get_expression()["values"][1])
-
-    def test_fetch_by_id_with_sort(self):
-        self.dynamo_db_table.query = MagicMock(
-            return_value={'Items': [{
-                'id': '1-2-3-4',
-                'age': Decimal(10),
-                'category_id': '4-5-6'
-            }]}
-        )
-        users = self.di.build(Users)
-        users = users.where('id=1-2-3-4').sort_by('category_id', 'desc').__iter__()
-        self.assertEqual(1, len(self.dynamo_db_table.query.call_args_list))
-        call = self.dynamo_db_table.query.call_args_list[0]
-        self.assertEqual(4, len(call.kwargs))
-        self.assertEqual("ALL_ATTRIBUTES", call.kwargs["Select"])
-        self.assertEqual("id-category_id-index", call.kwargs["IndexName"])
-        self.assertEqual(False, call.kwargs["ScanIndexForward"])
-        key_condition = call.kwargs['KeyConditionExpression']
-
-        # dynamodb does not make it easy to verify that the key expression was built properly,
-        # and I don't think that mocking will make this any better.  Hoepfully they don't
-        # change their library often...
-        key_column = key_condition.get_expression()['values'][0]
-        self.assertEqual("id", key_column.name)
-        self.assertTrue(isinstance(key_column, dynamodb_conditions.Key))
-        self.assertEqual("=", key_condition.expression_operator)
-        self.assertEqual("1-2-3-4", key_condition.get_expression()["values"][1])
-
-    def test_fetch_by_secondary_index_twice(self):
-        self.dynamo_db_table.query = MagicMock(
-            return_value={'Items': [{
-                'id': '1-2-3-4',
-                'age': Decimal(10),
-                'category_id': '4-5-6'
-            }]}
-        )
-        users = self.di.build(Users)
-        users = users.where('category_id=1-2-3-4').where('age>10').__iter__()
-        self.assertEqual(1, len(self.dynamo_db_table.query.call_args_list))
-        call = self.dynamo_db_table.query.call_args_list[0]
-        self.assertEqual(4, len(call.kwargs))
-        self.assertEqual("ALL_ATTRIBUTES", call.kwargs["Select"])
-        self.assertEqual("category_id-age-index", call.kwargs["IndexName"])
-        self.assertEqual(True, call.kwargs["ScanIndexForward"])
-        key_condition = call.kwargs['KeyConditionExpression']
-        self.assertTrue(isinstance(key_condition, dynamodb_conditions.And))
-        gt_condition = key_condition.get_expression()['values'][0]
-        equal_condition = key_condition.get_expression()['values'][1]
-
-        self.assertTrue(isinstance(equal_condition, dynamodb_conditions.Equals))
-        key_column = equal_condition.get_expression()['values'][0]
-        self.assertEqual("category_id", key_column.name)
-        self.assertTrue(isinstance(key_column, dynamodb_conditions.Key))
-        self.assertEqual("=", equal_condition.expression_operator)
-        self.assertEqual("1-2-3-4", equal_condition.get_expression()["values"][1])
-
-        self.assertTrue(isinstance(gt_condition, dynamodb_conditions.GreaterThan))
-        key_column = gt_condition.get_expression()['values'][0]
-        self.assertEqual("age", key_column.name)
-        self.assertTrue(isinstance(key_column, dynamodb_conditions.Key))
-        self.assertEqual(">", gt_condition.expression_operator)
-        self.assertEqual(Decimal("10"), gt_condition.get_expression()["values"][1])
-
-    def test_index_and_scan(self):
-        self.dynamo_db_table.query = MagicMock(
-            return_value={'Items': [{
-                'id': '1-2-3-4',
-                'age': Decimal(10),
-                'category_id': '4-5-6'
-            }]}
-        )
-        users = self.di.build(Users)
-        users = users.where('category_id=1-2-3-4').where('age is not null').__iter__()
-        self.assertEqual(1, len(self.dynamo_db_table.query.call_args_list))
-        call = self.dynamo_db_table.query.call_args_list[0]
-        self.assertEqual(5, len(call.kwargs))
-        self.assertEqual("ALL_ATTRIBUTES", call.kwargs["Select"])
-        self.assertEqual("category_id-name-index", call.kwargs["IndexName"])
-        self.assertEqual(True, call.kwargs["ScanIndexForward"])
-
-        # our key condition should be an equal search on category_id
-        key_condition = call.kwargs['KeyConditionExpression']
-        key_column = key_condition.get_expression()['values'][0]
-        self.assertEqual("category_id", key_column.name)
-        self.assertTrue(isinstance(key_column, dynamodb_conditions.Key))
-        self.assertEqual("=", key_condition.expression_operator)
-        self.assertEqual("1-2-3-4", key_condition.get_expression()["values"][1])
-
-        # and we should have a FilterExpression which is an 'is not null' condition
-        filter_condition = call.kwargs['FilterExpression']
-        key_column = filter_condition.get_expression()['values'][0]
-        self.assertEqual("age", key_column.name)
-        self.assertTrue(isinstance(key_column, dynamodb_conditions.Attr))
-        self.assertEqual("attribute_not_exists", filter_condition.expression_operator)
-
-    def test_scan_only(self):
-        self.dynamo_db_table.scan = MagicMock(
-            return_value={'Items': [{
-                'id': '1-2-3-4',
-                'age': Decimal(10),
-                'category_id': '4-5-6'
-            }]}
-        )
-        users = self.di.build(Users)
-        users = users.where('category_id>1-2-3-4').__iter__()
-        self.assertEqual(1, len(self.dynamo_db_table.scan.call_args_list))
-        call = self.dynamo_db_table.scan.call_args_list[0]
-        self.assertEqual(2, len(call.kwargs))
-        self.assertEqual("ALL_ATTRIBUTES", call.kwargs["Select"])
-
-        # and we should have a FilterExpression which is an 'is not null' condition
-        filter_condition = call.kwargs['FilterExpression']
-        key_column = filter_condition.get_expression()['values'][0]
-        self.assertEqual("category_id", key_column.name)
-        self.assertTrue(isinstance(key_column, dynamodb_conditions.Attr))
-        self.assertEqual(">", filter_condition.expression_operator)

clearskies_aws/backends/dynamo_db_condition_parser.py

@@ -1,365 +0,0 @@
-import base64
-import json
-import logging
-from decimal import Decimal, DecimalException
-from typing import Any, Dict, List, Tuple
-
-from clearskies import ConditionParser
-
-# Ensure AttributeValueTypeDef is imported from the correct boto3 types package
-# This is crucial for the "ideal fix".
-from types_boto3_dynamodb.type_defs import AttributeValueTypeDef
-
-logger = logging.getLogger(__name__)
-
-
-class DynamoDBConditionParser(ConditionParser):
-    """
-    Parses string conditions into a structured format suitable for DynamoDB PartiQL queries.
-
-    This class handles various SQL-like operators and translates them into
-    PartiQL compatible expressions and parameters. It also includes a utility
-    to convert Python values into the DynamoDB AttributeValue format.
-    """
-
-    operator_lengths: Dict[str, int] = {
-        "<>": 2,
-        "<=": 2,
-        ">=": 2,
-        "!=": 2,
-        ">": 1,
-        "<": 1,
-        "=": 1,
-        "in": 4,
-        "is not null": 12,
-        "is null": 8,
-        "is not": 8,
-        "is": 4,
-        "like": 6,
-        "is not missing": 15,
-        "is missing": 11,
-        "contains": 9,
-        "begins_with": 12,
-    }
-    operators: List[str] = [
-        # Longer operators first to help with matching
-        "is not null",
-        "is not missing",
-        "is null",
-        "is missing",
-        "begins_with",
-        "contains",
-        "<>",
-        "!=",
-        "<=",
-        ">=",
-        "is not",
-        "is",
-        "like",
-        ">",
-        "<",
-        "=",
-        "in",
-    ]
-    operators_for_matching: Dict[str, str] = {
-        "like": " like ",
-        "in": " in ",
-        "is not missing": " is not missing",
-        "is missing": " is missing",
-        "is not null": " is not null",
-        "is null": " is null",
-        "is": " is ",
-        "is not": " is not ",
-        "begins_with": " begins_with",
-        "contains": " contains",
-    }
-    operators_with_simple_placeholders: Dict[str, bool] = {
-        "<>": True,
-        "<=": True,
-        ">=": True,
-        "!=": True,
-        "=": True,
-        "<": True,
-        ">": True,
-        "is": True,
-        "is not": True,
-    }
-    operators_without_placeholders: set[str] = {
-        "is not missing",
-        "is missing",
-    }
-    operator_needs_remap: Dict[str, str] = {
-        "is not null": "is not missing",
-        "is null": "is missing",
-    }
-    operators_with_special_placeholders: set[str] = {"begins_with", "contains"}
-
-    def parse_condition(self, condition: str) -> Dict[str, Any]:
-        """
-        Parses a string condition into a structured dictionary.
-
-        The "values" key in the returned dictionary will contain List[AttributeValueTypeDef].
-
-        Args:
-            condition: The condition string to parse.
-
-        Returns:
-            A dictionary with keys: "table", "column", "operator", "values" (DynamoDB formatted),
-            and "parsed" (the SQL fragment).
-        """
-        lowercase_condition: str = condition.lower()
-        matching_operator: str = ""
-        matching_index: int = -1
-        current_best_match_len: int = 0
-
-        for operator in self.operators:
-            try:
-                operator_for_match: str = self.operators_for_matching.get(
-                    operator, operator
-                )
-                index: int = lowercase_condition.index(operator_for_match)
-
-                if matching_index == -1 or index < matching_index:
-                    matching_index = index
-                    matching_operator = operator
-                    current_best_match_len = len(operator_for_match)
-                elif index == matching_index:
-                    if len(operator_for_match) > current_best_match_len:
-                        matching_operator = operator
-                        current_best_match_len = len(operator_for_match)
-            except ValueError:
-                continue
-
-        if not matching_operator:
-            raise ValueError(f"No supported operators found in condition {condition}")
-
-        column: str = condition[:matching_index].strip()
-        value: str = condition[
-            matching_index + self.operator_lengths[matching_operator] :
-        ].strip()
-
-        if len(value) >= 2:
-            first_char = value[0]
-            last_char = value[-1]
-            if (first_char == "'" and last_char == "'") or (
-                first_char == '"' and last_char == '"'
-            ):
-                value = value[1:-1]
-
-        raw_values: List[str] = []
-
-        if matching_operator == "in":
-            raw_values = self._parse_condition_list(value) if value else []
-        elif matching_operator not in self.operators_without_placeholders and not (
-            matching_operator in self.operator_needs_remap
-            and self.operator_needs_remap[matching_operator]
-            in self.operators_without_placeholders
-        ):
-            raw_values = [value]
-
-        if matching_operator.lower() == "like":
-            if value.startswith("%") and value.endswith("%") and len(value) > 1:
-                matching_operator = "contains"
-                raw_values = [value[1:-1]]
-            elif value.endswith("%") and not value.startswith("%"):
-                matching_operator = "begins_with"
-                raw_values = [value[:-1]]
-            elif value.startswith("%") and not value.endswith("%"):
-                raise ValueError(
-                    "DynamoDB PartiQL does not directly support 'ends_with'"
-                )
-            else:
-                matching_operator = "="
-                raw_values = [value]
-
-        matching_operator = self.operator_needs_remap.get(
-            matching_operator.lower(), matching_operator
-        )
-
-        table_name: str = ""
-        final_column_name: str = column
-        if "." in column:
-            table_prefix, column_name_part = column.split(".", 1)
-            table_name = table_prefix.strip().replace('"', "").replace("`", "")
-            final_column_name = (
-                column_name_part.strip().replace('"', "").replace("`", "")
-            )
-        else:
-            final_column_name = column.replace('"', "").replace("`", "")
-
-        # This list will now correctly be List[AttributeValueTypeDef]
-        parameters: List[AttributeValueTypeDef] = []
-        if matching_operator.lower() not in self.operators_without_placeholders:
-            for val_item in raw_values:
-                parameters.append(self.to_dynamodb_attribute_value(val_item))
-
-        column_for_parsed: str = (
-            f"{table_name}.{final_column_name}" if table_name else final_column_name
-        )
-
-        return {
-            "table": table_name,
-            "column": final_column_name,
-            "operator": matching_operator.upper(),
-            "values": parameters,  # This is now correctly typed for MyPy
-            "parsed": self._with_placeholders(
-                column_for_parsed,
-                matching_operator,
-                parameters,
-            ),
-        }
-
-    def _with_placeholders(
-        self,
-        column: str,
-        operator: str,
-        values: List[
-            AttributeValueTypeDef
-        ],  # Parameter 'values' is List[AttributeValueTypeDef]
-        escape: bool = True,
-        escape_character: str = '"',
-    ) -> str:
-        """
-        Formats a SQL fragment with placeholders for a given column, operator, and parameters.
-        """
-        quoted_column = column
-        if escape:
-            parts: List[str] = column.split(".", 1)
-            cleaned_parts: List[str] = [part.strip('"`') for part in parts]
-            if len(cleaned_parts) == 2:
-                quoted_column = (
-                    f"{escape_character}{cleaned_parts[0]}{escape_character}"
-                    "."
-                    f"{escape_character}{cleaned_parts[1]}{escape_character}"
-                )
-            else:
-                quoted_column = (
-                    f"{escape_character}{cleaned_parts[0]}{escape_character}"
-                )
-
-        upper_case_operator: str = operator.upper()
-        lower_case_operator: str = operator.lower()
-
-        if lower_case_operator in self.operators_with_simple_placeholders:
-            return f"{quoted_column} {upper_case_operator} ?"
-        if lower_case_operator in self.operators_without_placeholders:
-            return f"{quoted_column} {upper_case_operator}"
-        if lower_case_operator in self.operators_with_special_placeholders:
-            return f"{lower_case_operator}({quoted_column}, ?)"
-
-        if lower_case_operator == "in":
-            placeholders_str: str = ", ".join(["?" for _ in values])
-            return f"{quoted_column} IN ({placeholders_str})"
-
-        raise ValueError(f"Unsupported operator for placeholder generation: {operator}")
-
-    def to_dynamodb_attribute_value(
-        self, value: Any
-    ) -> AttributeValueTypeDef:  # Return type changed
-        """
-        Converts a Python variable into a DynamoDB-formatted attribute value dictionary.
-        """
-        if isinstance(value, str):
-            if value.lower() == "true":
-                return {"BOOL": True}
-            if value.lower() == "false":
-                return {"BOOL": False}
-            if value.lower() == "null":
-                return {"NULL": True}
-            try:
-                if value.isdigit() or (value.startswith("-") and value[1:].isdigit()):
-                    return {"N": str(int(value))}
-                return {"N": str(Decimal(value))}
-            except (ValueError, TypeError, json.JSONDecodeError, DecimalException):
-                return {"S": value}
-        elif isinstance(value, bool):
-            return {"BOOL": value}
-        elif isinstance(value, (int, float, Decimal)):
-            return {"N": str(value)}
-        elif value is None:
-            return {"NULL": True}
-        elif isinstance(value, bytes):
-            return {"B": base64.b64encode(value).decode("utf-8")}
-        elif isinstance(value, list):
-            # Each item will be AttributeValueTypeDef, so the list is List[AttributeValueTypeDef]
-            return {"L": [self.to_dynamodb_attribute_value(item) for item in value]}
-        elif isinstance(value, dict):
-            # Each value in the map will be AttributeValueTypeDef
-            return {
-                "M": {
-                    str(k): self.to_dynamodb_attribute_value(v)
-                    for k, v in value.items()
-                }
-            }
-        elif isinstance(value, set):
-            if not value:
-                raise ValueError(
-                    "Cannot determine DynamoDB Set type from an empty Python set."
-                )
-            if all(isinstance(item, str) for item in value):
-                return {"SS": sorted(list(value))}
-            elif all(isinstance(item, (int, float, Decimal)) for item in value):
-                return {"NS": sorted([str(item) for item in value])}
-            elif all(isinstance(item, bytes) for item in value):
-                return {
-                    "BS": sorted(
-                        [base64.b64encode(item).decode("utf-8") for item in value]
-                    )
-                }
-            raise ValueError(
-                "Set contains mixed types or unsupported types for DynamoDB Sets."
-            )
-        else:
-            raise TypeError(
-                f"Unsupported Python type for DynamoDB conversion: {type(value)}"
-            )
-
-    def _parse_condition_list(self, list_string: str) -> List[str]:
-        """
-        Parses a string representation of a list into a list of strings.
-        """
-        if not list_string.strip():
-            return []
-
-        if list_string.startswith("(") and list_string.endswith(")"):
-            list_string = list_string[1:-1]
-            if not list_string.strip():
-                return []
-
-        items: List[str] = []
-        current_item: str = ""
-        in_quotes: bool = False
-        quote_char: str = ""
-        for char in list_string:
-            if char in ("'", '"'):
-                if in_quotes and char == quote_char:
-                    in_quotes = False
-                elif not in_quotes:
-                    in_quotes = True
-                    quote_char = char
-                else:
-                    current_item += char
-            elif char == "," and not in_quotes:
-                stripped_item = current_item.strip()
-                if stripped_item:
-                    items.append(stripped_item)
-                current_item = ""
-            else:
-                current_item += char
-
-        stripped_current_item = current_item.strip()
-        if stripped_current_item:
-            items.append(stripped_current_item)
-
-        final_items = []
-        for item in items:
-            processed_item = item
-            if len(processed_item) >= 2:
-                if processed_item.startswith("'") and processed_item.endswith("'"):
-                    processed_item = processed_item[1:-1]
-                elif processed_item.startswith('"') and processed_item.endswith('"'):
-                    processed_item = processed_item[1:-1]
-
-            if processed_item:
-                final_items.append(processed_item)
-        return final_items