in-layers-data 0.4.20__py3-none-any.whl

in_layers/__init__.py

@@ -0,0 +1,7 @@
+"""Namespace package for in_layers."""
+
+from __future__ import annotations
+
+import pkgutil
+
+__path__ = pkgutil.extend_path(__path__, __name__)  # type: ignore[name-defined]

in_layers/data/__init__.py

@@ -0,0 +1,21 @@
+from . import services
+from .protocols import (
+    BackendConfig,
+    DataNamespace,
+    DynamoDBBackendConfig,
+    InLayersDataConfig,
+    MongoBackendConfig,
+    SupportedBackend,
+)
+
+name = DataNamespace.root.value
+
+__all__ = [
+    "BackendConfig",
+    "DynamoDBBackendConfig",
+    "InLayersDataConfig",
+    "MongoBackendConfig",
+    "SupportedBackend",
+    "name",
+    "services",
+]

in_layers/data/backends/dynamodb/__init__.py

@@ -0,0 +1,5 @@
+"""DynamoDB backend for InLayers models."""
+
+from .services import DynamoDBBackend
+
+__all__ = ["DynamoDBBackend"]

in_layers/data/backends/dynamodb/libs.py

@@ -0,0 +1,108 @@
+"""Pure functional utilities for DynamoDB query conversion and data formatting."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from datetime import datetime
+from decimal import Decimal
+from typing import Any
+
+from in_layers.core.models.protocols import ModelDefinition
+
+
+def get_table_name_for_model(
+    environment: str, model_definition: ModelDefinition
+) -> str:
+    """Generate a DynamoDB table name from a model definition."""
+    name = model_definition.plural_name.replace("@", "").replace("/", "-")
+    # Convert to kebab-case: insert hyphens before uppercase letters (except first)
+    # and handle sequences of uppercase letters
+    name = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", name)
+    name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1-\2", name)
+    return f"{name}-{environment}".lower()
+
+
+def convert_all_floats_to_decimals(obj):
+    """
+    Recursively convert all float values in a nested structure to Decimal.
+    This is useful for preparing data for DynamoDB, which requires numbers to be
+    represented as Decimals to avoid precision issues.
+
+    Args:
+        obj: The input object, which can be a dict, list, float, or other types.
+    Returns:
+        The modified object with all float values converted to Decimal.
+    """
+    if isinstance(obj, float):
+        return Decimal(str(obj))
+    elif isinstance(obj, dict):
+        return {k: convert_all_floats_to_decimals(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [convert_all_floats_to_decimals(item) for item in obj]
+    else:
+        return obj
+
+
+def format_for_dynamodb(data: Mapping[str, Any]) -> dict[str, Any]:
+    """Format data for DynamoDB storage.
+
+    DynamoDB natively handles:
+    - Strings
+    - Numbers (int, float, Decimal)
+    - Binary data
+    - Boolean
+    - Null
+    - Lists
+    - Maps (dicts)
+    - Sets (string sets, number sets, binary sets)
+
+    The boto3 DynamoDBDocumentClient will handle conversion automatically,
+    but we ensure datetime objects are converted to ISO format strings.
+    """
+    result = dict(data)
+    # Convert datetime objects to ISO format strings for DynamoDB
+
+    for key, value in result.items():
+        if isinstance(value, datetime):
+            result[key] = value.isoformat()
+    return convert_all_floats_to_decimals(result)
+
+
+def from_dynamodb(item: dict[str, Any] | None) -> dict[str, Any]:
+    """Convert a DynamoDB item to a plain dictionary.
+
+    The boto3 DynamoDBDocumentClient already converts AttributeValue format
+    to native Python types, so this is mainly for consistency and future-proofing.
+    """
+    if item is None:
+        return {}
+    return dict(item)
+
+
+def build_scan_params(
+    table_name: str, exclusive_start_key: dict[str, Any] | None = None
+) -> dict[str, Any]:
+    """Build parameters for a DynamoDB Scan operation."""
+    params: dict[str, Any] = {
+        "TableName": table_name,
+    }
+    if exclusive_start_key:
+        params["ExclusiveStartKey"] = exclusive_start_key
+    return params
+
+
+def split_array_into_batches(array: list[Any], max_batch_size: int) -> list[list[Any]]:
+    """Split an array into batches of maximum size.
+
+    DynamoDB has limits on batch operations (e.g., BatchWriteItem max 25 items).
+    """
+    if not isinstance(array, list):
+        raise ValueError("Input must be a list")
+    if max_batch_size < 1:
+        raise ValueError("max_batch_size must be at least 1")
+
+    batches = []
+    for i in range(0, len(array), max_batch_size):
+        batches.append(array[i : i + max_batch_size])
+    return batches

in_layers/data/backends/dynamodb/services.py

@@ -0,0 +1,395 @@
+"""DynamoDB backend implementation for InLayers models."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from uuid import uuid4
+
+from box import Box
+from in_layers.core.models.backends import (
+    _apply_sort,
+    _apply_take,
+    _matches_query_tokens,
+)
+from in_layers.core.models.protocols import (
+    InLayersModel,
+    ModelSearch,
+    ModelSearchResult,
+    PrimaryKeyType,
+)
+
+from in_layers.data.protocols import DynamoDBBackendConfig
+
+from .libs import (
+    format_for_dynamodb,
+    from_dynamodb,
+    get_table_name_for_model,
+    split_array_into_batches,
+)
+
+# DynamoDB batch operation limits
+MAX_BATCH_WRITE_SIZE = 25
+SCAN_RETURN_THRESHOLD = 1000
+
+
+class DynamoDBBackend:
+    """DynamoDB backend implementation."""
+
+    def __init__(self, context, config: DynamoDBBackendConfig):
+        self.__context = context
+        self.__config = config
+        self.__client: Any = None
+        self.__table_client: Any = None
+
+    @staticmethod
+    def create_unique_connection_string(config: DynamoDBBackendConfig) -> str:
+        """Create a unique connection string from config."""
+        region = config.region or "us-east-1"
+        endpoint_url = config.endpoint_url or ""
+
+        parts = [f"region={region}"]
+        if endpoint_url:
+            parts.append(f"endpoint={endpoint_url}")
+
+        return "|".join(parts)
+
+    def __get_config_value(self, key: str, default_value: Any = None) -> Any:
+        if key in self.__config:
+            return self.__config[key]
+        return default_value
+
+    def __connect(self) -> None:
+        """Connect to DynamoDB (private method)."""
+        # Use boto3 from config if provided (for testing), otherwise import it
+        if self.__get_config_value("boto3") is not None:
+            boto3 = self.__get_config_value("boto3")
+        else:
+            import boto3  # pragma: no cover # noqa: PLC0415
+
+        region = self.__get_config_value("region")
+        endpoint_url = self.__get_config_value("endpoint_url")
+        aws_access_key_id = self.__get_config_value("aws_access_key_id")
+        aws_secret_access_key = self.__get_config_value("aws_secret_access_key")
+        client_kwargs: dict[str, Any] = {}
+        if region:
+            client_kwargs["region_name"] = region
+        if endpoint_url:
+            client_kwargs["endpoint_url"] = endpoint_url
+        if aws_access_key_id:
+            client_kwargs["aws_access_key_id"] = aws_access_key_id
+        if aws_secret_access_key:
+            client_kwargs["aws_secret_access_key"] = aws_secret_access_key
+
+        # Create DynamoDB client
+        self.__client = boto3.client("dynamodb", **client_kwargs)
+
+        # Create DynamoDB resource for easier table operations
+        dynamodb_resource = boto3.resource("dynamodb", **client_kwargs)
+        self.__table_client = dynamodb_resource
+
+    def __disconnect(self) -> None:
+        """Disconnect from DynamoDB (private method)."""
+        # boto3 clients don't need explicit closing, but we can clear references
+        self.__client = None
+        self.__table_client = None
+
+    def __ensure_connected(self) -> None:
+        """Ensure DynamoDB connection is established."""
+        if self.__client is None:
+            self.__connect()
+
+    def get_raw_client(self) -> Any:
+        self.__ensure_connected()
+        return self.__table_client
+
+    def get_backend_name(self) -> str:
+        return "dynamodb"
+
+    def create(self, model: InLayersModel, data: Mapping) -> Mapping:
+        """Create a new item in DynamoDB."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(
+            self.__context.config.environment, model.get_model_definition()
+        )
+        table = self.__table_client.Table(table_name)
+
+        payload = dict(data)
+        formatted = format_for_dynamodb(payload)
+
+        pk_name = model.get_primary_key_name()
+        pk_value = formatted.get(pk_name)
+        if pk_value is None:
+            pk_value = str(uuid4())
+            formatted[pk_name] = pk_value
+
+        # Ensure primary key is a string (DynamoDB requirement)
+        formatted[pk_name] = str(pk_value)
+
+        # Put item in DynamoDB
+        table.put_item(Item=formatted)
+        return formatted
+
+    def retrieve(self, model: InLayersModel, id: PrimaryKeyType) -> Mapping | None:
+        """Retrieve an item by ID."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(
+            self.__context.config.environment, model.get_model_definition()
+        )
+        table = self.__table_client.Table(table_name)
+
+        pk_name = model.get_primary_key_name()
+        key = {pk_name: str(id)}
+
+        response = table.get_item(Key=key)
+        item = response.get("Item")
+        if not item:
+            return None
+
+        return from_dynamodb(item)
+
+    def update(
+        self, model: InLayersModel, id: PrimaryKeyType, data: Mapping
+    ) -> Mapping:
+        """Update an item by ID.
+
+        Supports partial updates - only the fields provided in data will be updated.
+        Returns the full merged object (original + updates).
+        """
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(
+            self.__context.config.environment, model.get_model_definition()
+        )
+        table = self.__table_client.Table(table_name)
+
+        pk_name = model.get_primary_key_name()
+        key = {pk_name: str(id)}
+
+        # Check if item exists
+        existing = table.get_item(Key=key)
+        if "Item" not in existing:
+            raise KeyError(f"Instance with id {id!r} not found")
+
+        payload = dict(data)
+        formatted = format_for_dynamodb(payload)
+
+        # Ensure primary key field remains consistent
+        formatted[pk_name] = str(id)
+
+        # Build UpdateExpression for partial update
+        # DynamoDB requires SET expressions for each attribute
+        update_expressions = []
+        expression_attribute_names = {}
+        expression_attribute_values = {}
+
+        for attr_name, attr_value in formatted.items():
+            # Skip the primary key as it's in the Key parameter
+            if attr_name == pk_name:
+                continue
+
+            # Use attribute name placeholders to handle reserved words
+            name_placeholder = f"#attr_{attr_name}"
+            value_placeholder = f":val_{attr_name}"
+
+            expression_attribute_names[name_placeholder] = attr_name
+            expression_attribute_values[value_placeholder] = attr_value
+            update_expressions.append(f"{name_placeholder} = {value_placeholder}")
+
+        if not update_expressions:
+            # No fields to update (only primary key was provided)
+            # Just return the existing item
+            return from_dynamodb(existing["Item"])
+
+        # Build the update expression
+        update_expression = f"SET {', '.join(update_expressions)}"
+
+        # Perform partial update using update_item
+        table.update_item(
+            Key=key,
+            UpdateExpression=update_expression,
+            ExpressionAttributeNames=expression_attribute_names,
+            ExpressionAttributeValues=expression_attribute_values,
+        )
+
+        # Retrieve the updated item to return the full merged object
+        updated = table.get_item(Key=key)
+        if "Item" not in updated:
+            raise KeyError(f"Instance with id {id!r} not found after update")
+
+        return from_dynamodb(updated["Item"])
+
+    def delete(self, model: InLayersModel, id: PrimaryKeyType) -> None:
+        """Delete an item by ID."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(
+            self.__context.config.environment, model.get_model_definition()
+        )
+        table = self.__table_client.Table(table_name)
+
+        pk_name = model.get_primary_key_name()
+        key = {pk_name: str(id)}
+
+        table.delete_item(Key=key)
+
+    def search(self, model: InLayersModel, query: ModelSearch) -> ModelSearchResult:
+        """Search for items matching the query.
+
+        Note: DynamoDB Scan operations are expensive and should be used sparingly.
+        For production use, consider using Query operations with Global Secondary Indexes (GSI)
+        or Local Secondary Indexes (LSI) for better performance.
+
+        This implementation continues scanning across pages until:
+        - Threshold is met (SCAN_RETURN_THRESHOLD if no take specified)
+        - No more keys (LastEvaluatedKey is null)
+        - Take limit is reached (if take is specified)
+        """
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(
+            self.__context.config.environment, model.get_model_definition()
+        )
+        table = self.__table_client.Table(table_name)
+
+        # Start recursive scanning
+        result = self._do_search_until_threshold_or_no_last_evaluated_key(
+            table, query, []
+        )
+
+        return Box(instances=result["instances"], page=result["page"])
+
+    def _do_search_until_threshold_or_no_last_evaluated_key(
+        self,
+        table: Any,
+        search: ModelSearch,
+        old_instances_found: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        """Recursively scan DynamoDB until threshold is met or no more keys.
+
+        This matches the TypeScript implementation's behavior of continuing
+        to scan across pages until enough filtered results are found.
+        """
+        # Build scan parameters
+        scan_kwargs: dict[str, Any] = {}
+        # Ensure query is always a list (handle None case)
+        query = getattr(search, "query", []) or []
+        take = getattr(search, "take", None)
+        sort = getattr(search, "sort", None)
+        if getattr(search, "page", None):
+            scan_kwargs["ExclusiveStartKey"] = search.page
+
+        # Execute scan
+        response = table.scan(**scan_kwargs)
+        items = response.get("Items", [])
+
+        # Convert DynamoDB items to plain dicts
+        unfiltered = [from_dynamodb(item) for item in items]
+
+        # Apply filtering using the same logic as MemoryBackend
+        filtered = [r for r in unfiltered if _matches_query_tokens(r, query)]
+
+        # Combine with previously found instances
+        all_filtered = filtered + old_instances_found
+
+        # Determine threshold
+        using_take = take is not None and take > 0
+        threshold = take if using_take else SCAN_RETURN_THRESHOLD
+
+        # Get pagination key
+        last_evaluated_key = response.get("LastEvaluatedKey")
+
+        # Check stopping conditions:
+        # 1. We have enough results (more than threshold)
+        # 2. No more keys to evaluate
+        # Note: TypeScript uses > (strictly greater), meaning we continue if we have exactly 'threshold' items
+        stop_for_threshold = len(all_filtered) > threshold
+        stop_for_no_more = last_evaluated_key is None
+
+        if stop_for_threshold or stop_for_no_more:
+            # Apply sorting and take limit
+            sorted_instances = _apply_sort(all_filtered, sort)
+            # Apply take limit (use original take value, not threshold)
+            limited_instances = _apply_take(sorted_instances, take)
+
+            # Return page: null when using take, otherwise return LastEvaluatedKey
+            page = None if using_take else last_evaluated_key
+
+            return {
+                "instances": [dict(x) for x in limited_instances],
+                "page": page,
+            }
+
+        # Continue scanning with the new page key
+        # Create a new ModelSearch with updated page (frozen dataclass requires new instance)
+        new_query = ModelSearch(
+            query=query,
+            take=take,
+            sort=sort,
+            page=last_evaluated_key,
+        )
+        return self._do_search_until_threshold_or_no_last_evaluated_key(
+            table, new_query, all_filtered
+        )
+
+    def bulk_insert(self, model: InLayersModel, data: list[Mapping]) -> None:
+        """Bulk insert items."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(
+            self.__context.config.environment, model.get_model_definition()
+        )
+        table = self.__table_client.Table(table_name)
+        pk_name = model.get_primary_key_name()
+
+        # Prepare items
+        items = []
+        for item in data:
+            payload = dict(item)
+            formatted = format_for_dynamodb(payload)
+
+            pk_value = formatted.get(pk_name)
+            if pk_value is None:
+
+                pk_value = str(uuid4())
+                formatted[pk_name] = pk_value
+
+            formatted[pk_name] = str(pk_value)
+            items.append(formatted)
+
+        # Split into batches (DynamoDB BatchWriteItem limit is 25)
+        batches = split_array_into_batches(items, MAX_BATCH_WRITE_SIZE)
+
+        # Write batches
+        for batch in batches:
+            with table.batch_writer() as writer:
+                for item in batch:
+                    writer.put_item(Item=item)
+
+    def bulk_delete(self, model: InLayersModel, ids: list[PrimaryKeyType]) -> None:
+        """Bulk delete items by IDs."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(
+            self.__context.config.environment, model.get_model_definition()
+        )
+        table = self.__table_client.Table(table_name)
+        pk_name = model.get_primary_key_name()
+
+        # Prepare keys
+        keys = [{pk_name: str(id)} for id in ids]
+
+        # Split into batches
+        batches = split_array_into_batches(keys, MAX_BATCH_WRITE_SIZE)
+
+        # Delete batches
+        for batch in batches:
+            with table.batch_writer() as writer:
+                for key in batch:
+                    writer.delete_item(Key=key)
+
+    def dispose(self) -> None:
+        """Clean up resources."""
+        self.__disconnect()

in_layers/data/backends/mongodb/libs.py

@@ -0,0 +1,251 @@
+"""Pure functional utilities for MongoDB query conversion and data formatting."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from datetime import datetime
+from decimal import Decimal
+from typing import Any
+
+from in_layers.core.models.protocols import (
+    BooleanQuery,
+    DatastoreValueType,
+    EqualitySymbol,
+    ModelDefinition,
+    PropertyQuery,
+    QueryTokens,
+)
+
+
+def get_collection_name_for_model(model_definition: ModelDefinition) -> str:
+    """Generate a MongoDB collection name from a model definition."""
+    name = model_definition.plural_name.replace("@", "").replace("/", "-")
+    # Convert to kebab-case: insert hyphens before uppercase letters (except first)
+    # and handle sequences of uppercase letters
+    name = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", name)
+    name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1-\2", name)
+    return f"{name}".lower()
+
+
+def escape_regex(s: str) -> str:
+    """Escape special regex characters in a string."""
+    return re.escape(s)
+
+
+def build_string_pattern(
+    raw: str, starts_with: bool, ends_with: bool, includes: bool
+) -> str:
+    """Build a regex pattern for string matching."""
+    escaped = escape_regex(raw)
+    if starts_with:
+        return f"^{escaped}"
+    if ends_with:
+        return f"{escaped}$"
+    if includes:
+        return escaped
+    # default: exact match
+    return f"^{escaped}$"
+
+
+def regex_object_for_pattern(pattern: str, case_sensitive: bool) -> dict[str, Any]:
+    """Create a MongoDB regex object from a pattern."""
+    if case_sensitive:
+        return {"$regex": pattern}
+    return {"$regex": pattern, "$options": "i"}
+
+
+def build_mongo_find_value(query: PropertyQuery) -> dict[str, Any]:  # noqa: PLR0911
+    """Convert a PropertyQuery to a MongoDB query value."""
+    value = query.value
+    if value is None:
+        return {query.key: None}
+
+    # Handle date objects
+    if hasattr(value, "isoformat"):  # datetime objects
+        return {query.key: value}
+
+    if query.value_type == DatastoreValueType.string:
+        case_sensitive = bool(query.options.case_sensitive)
+        starts_with = bool(query.options.starts_with)
+        ends_with = bool(query.options.ends_with)
+        includes = bool(query.options.includes)
+
+        if query.equality_symbol not in (EqualitySymbol.eq, EqualitySymbol.ne):
+            raise ValueError(
+                f"Symbol {query.equality_symbol} is unhandled for string type"
+            )
+
+        raw = str(value)
+        pattern = build_string_pattern(raw, starts_with, ends_with, includes)
+        regex_obj = regex_object_for_pattern(pattern, case_sensitive)
+
+        if query.equality_symbol == EqualitySymbol.ne:
+            is_plain_exact = (
+                not starts_with and not ends_with and not includes and case_sensitive
+            )
+            if is_plain_exact:
+                return {query.key: {"$ne": raw}}
+            return {query.key: {"$not": regex_obj}}
+
+        use_regex = starts_with or ends_with or includes or not case_sensitive
+        if use_regex:
+            return {query.key: regex_obj}
+        return {query.key: raw}
+
+    if query.value_type == DatastoreValueType.number:
+        equality_symbol_to_mongo = {
+            EqualitySymbol.eq: "$eq",
+            EqualitySymbol.gt: "$gt",
+            EqualitySymbol.gte: "$gte",
+            EqualitySymbol.lt: "$lt",
+            EqualitySymbol.lte: "$lte",
+            EqualitySymbol.ne: "$ne",
+        }
+        mongo_symbol = equality_symbol_to_mongo.get(query.equality_symbol)
+        if not mongo_symbol:
+            raise ValueError(f"Symbol {query.equality_symbol} is unhandled")
+        return {query.key: {mongo_symbol: query.value}}
+
+    return {query.key: value}
+
+
+def threeitize(
+    data: list[QueryTokens],
+) -> list[tuple[QueryTokens, BooleanQuery, QueryTokens]]:
+    """Convert a list of tokens into three-tuples of (left, link, right)."""
+    if len(data) in (0, 1):
+        return []
+    if len(data) % 2 == 0:
+        raise ValueError("Must be an odd number of 3 or greater.")
+    three = (data[0], _as_link(data[1]), data[2])
+    rest = data[2:]
+    more = threeitize(rest)
+    return [three, *more]
+
+
+def _as_link(value: QueryTokens) -> BooleanQuery:
+    """Convert a token to a BooleanQuery link."""
+    if value in {"AND", "OR"}:
+        return value
+    raise ValueError("Must have AND/OR between statements")
+
+
+def process_mongo_array(
+    tokens: list[QueryTokens],
+) -> dict[str, Any]:
+    """Process an array of query tokens into a MongoDB query."""
+    # If we don't have any AND/OR, it's all an AND
+    if all(t != "AND" and t != "OR" for t in tokens):  # noqa: PLR1714
+        return {"$and": [handle_mongo_query(t) for t in tokens]}
+
+    # Process with threeitize
+    threes = threeitize(tokens)
+    threes.reverse()
+    result: dict[str, Any] = {}
+    for a, link, b in threes:
+        a_query = handle_mongo_query(a)
+        if result:
+            result = {f"${link.lower()}": [a_query, result]}
+        else:
+            b_query = handle_mongo_query(b)
+            result = {f"${link.lower()}": [a_query, b_query]}
+    return result
+
+
+def handle_mongo_query(token: QueryTokens | list[QueryTokens]) -> dict[str, Any]:
+    """Convert a query token to a MongoDB query object."""
+    # Handle list of tokens (the main query)
+    if isinstance(token, list):
+        return process_mongo_array(token)
+
+    # Check by shape (duck typing) rather than isinstance
+    # PropertyQuery has type="property" and value attribute
+    if (
+        hasattr(token, "type")
+        and getattr(token, "type", None) == "property"
+        and hasattr(token, "value")
+    ):
+        return build_mongo_find_value(token)
+
+    # DatesBeforeQuery has type="datesBefore" and date attribute
+    if (
+        hasattr(token, "type")
+        and getattr(token, "type", None) == "datesBefore"
+        and hasattr(token, "date")
+    ):
+        date_value = token.date
+        if (
+            hasattr(token, "value_type") and token.value_type == DatastoreValueType.date
+        ) and isinstance(date_value, str):
+            # Convert string to datetime if needed
+            date_value = datetime.fromisoformat(date_value.replace("Z", "+00:00"))
+        operator = (
+            "$lte"
+            if (hasattr(token, "options") and token.options.equal_to_and_before)
+            else "$lt"
+        )
+        return {token.key: {operator: date_value}}
+
+    # DatesAfterQuery has type="datesAfter" and date attribute
+    if (
+        hasattr(token, "type")
+        and getattr(token, "type", None) == "datesAfter"
+        and hasattr(token, "date")
+    ):
+        date_value = token.date
+        if (
+            hasattr(token, "value_type") and token.value_type == DatastoreValueType.date
+        ) and isinstance(date_value, str):
+            date_value = datetime.fromisoformat(date_value.replace("Z", "+00:00"))
+        operator = (
+            "$gte"
+            if (hasattr(token, "options") and token.options.equal_to_and_after)
+            else "$gt"
+        )
+        return {token.key: {operator: date_value}}
+
+    raise ValueError(f"Unhandled query token {token}")
+
+
+def to_mongo(query: list[QueryTokens]) -> list[dict[str, Any]]:
+    """Convert a query list to MongoDB aggregation pipeline stages."""
+    if not query:
+        return [{"$match": {}}]
+    # Pass the list directly to handle_mongo_query, which will process it as an array
+    match_query = handle_mongo_query(query)
+    return [{"$match": match_query}]
+
+
+def convert_decimals_to_float(value: Any) -> Any:
+    """Recursively convert Decimal values to float for MongoDB serialization.
+
+    MongoDB cannot serialize Decimal objects, so we need to convert them to float.
+    This function walks through nested structures (dicts, lists, etc.) and converts
+    any Decimal values it finds.
+    """
+    if isinstance(value, Decimal):
+        return float(value)
+    if isinstance(value, dict):
+        return {k: convert_decimals_to_float(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [convert_decimals_to_float(item) for item in value]
+    if isinstance(value, tuple):
+        return tuple(convert_decimals_to_float(item) for item in value)
+    return value
+
+
+def format_for_mongo(data: Mapping[str, Any]) -> dict[str, Any]:
+    """Format data for MongoDB storage, converting dates, decimals, and other types."""
+    result = dict(data)
+    # Convert datetime objects (they're already in the right format for MongoDB)
+    # In the TypeScript version, this iterates over model properties to find Datetime types
+    # For Python, we'll preserve datetime objects as-is (MongoDB handles them natively)
+    # and convert string ISO dates if they're clearly datetime values
+    for key, value in result.items():
+        if isinstance(value, datetime):
+            result[key] = value
+        else:
+            # Convert any Decimal values to float (recursively)
+            result[key] = convert_decimals_to_float(value)
+    return result

in_layers/data/backends/mongodb/services.py

@@ -0,0 +1,251 @@
+"""MongoDB backend implementation for InLayers models."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from uuid import uuid4
+
+from box import Box
+from in_layers.core.models.protocols import (
+    InLayersModel,
+    ModelSearch,
+    ModelSearchResult,
+    PrimaryKeyType,
+    SortOrder,
+)
+
+from in_layers.data.protocols import MongoBackendConfig
+
+from .libs import (
+    format_for_mongo,
+    get_collection_name_for_model,
+    to_mongo,
+)
+
+
+class MongoBackend:
+    """MongoDB backend implementation."""
+
+    def __init__(self, context, config: MongoBackendConfig):
+        self.__context = context
+        self.__config = config
+        self.__client: Any = None
+
+    @staticmethod
+    def create_unique_connection_string(config: MongoBackendConfig) -> str:
+        """Create a unique connection string from config."""
+        host = config.host
+        port = getattr(config, "port", None)
+        username = getattr(config, "username", None)
+        password = getattr(config, "password", None)
+        database = getattr(config, "database", None)
+
+        connection_string = "mongodb://"
+        if username:
+            connection_string += f"{username}:{password}@"
+        connection_string += f"{host}"
+        if port:
+            connection_string += f":{port}"
+        if database:
+            connection_string += f"/{database}"
+        return connection_string
+
+    def __connect(self) -> None:
+        """Connect to MongoDB (private method)."""
+        from pymongo import MongoClient  # noqa: PLC0415
+
+        connection_string = self.create_unique_connection_string(self.__config)
+        self.__client = MongoClient(connection_string)
+
+    def __get_database(self):
+        default_database_name = f"{self.__context.config.system_name}-{self.__context.config.environment}".lower()
+        database_name = getattr(self.__config, "database", default_database_name)
+        return self.__client[database_name]
+
+    def __disconnect(self) -> None:
+        """Disconnect from MongoDB (private method)."""
+        if self.__client:
+            self.__client.close()
+            self.__client = None
+
+    def __ensure_connected(self) -> None:
+        """Ensure MongoDB connection is established."""
+        if self.__client is None:
+            self.__connect()
+
+    def get_raw_client(self) -> Any:
+        self.__ensure_connected()
+        return self.__get_database()
+
+    def get_backend_name(self) -> str:
+        return "mongodb"
+
+    def create(self, model: InLayersModel, data: Mapping) -> Mapping:
+        """Create a new document in MongoDB."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        database = self.__get_database()
+        collection = database[collection_name]
+        payload = dict(data)
+        formatted = format_for_mongo(payload)
+
+        pk_name = model.get_primary_key_name()
+        pk_value = formatted.get(pk_name)
+        if pk_value is None:
+            # Generate a simple ID - in production you might want UUID
+
+            pk_value = str(uuid4())
+            formatted[pk_name] = pk_value
+
+        # Use _id as MongoDB's primary key, mapping from model's primary key
+        insert_data = {**formatted, "_id": pk_value}
+        collection.insert_one(insert_data)
+        # Return without _id
+        result = {k: v for k, v in insert_data.items() if k != "_id"}
+        return result
+
+    def retrieve(self, model: InLayersModel, id: PrimaryKeyType) -> Mapping | None:
+        """Retrieve a document by ID."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        database = self.__get_database()
+        collection = database[collection_name]
+        doc = collection.find_one({"_id": id})
+        if not doc:
+            return None
+        # Remove _id and return
+        result = {k: v for k, v in doc.items() if k != "_id"}
+        return result
+
+    def update(
+        self, model: InLayersModel, id: PrimaryKeyType, data: Mapping
+    ) -> Mapping:
+        """Update a document by ID.
+
+        Supports partial updates - only the fields provided in data will be updated.
+        Returns the full merged object (original + updates).
+        """
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        database = self.__get_database()
+        collection = database[collection_name]
+
+        # Check if document exists
+        existing = collection.find_one({"_id": id})
+        if not existing:
+            raise KeyError(f"Instance with id {id!r} not found")
+
+        payload = dict(data)
+        formatted = format_for_mongo(payload)
+
+        # Ensure primary key field remains consistent
+        pk_name = model.get_primary_key_name()
+        formatted[pk_name] = id
+
+        # Update with _id mapping - use $set for partial update
+        update_data = {**formatted, "_id": id}
+        collection.update_one({"_id": id}, {"$set": update_data})
+
+        # Retrieve the updated document to return the full merged object
+        updated = collection.find_one({"_id": id})
+        if not updated:
+            raise KeyError(f"Instance with id {id!r} not found after update")
+
+        # Return without _id
+        result = {k: v for k, v in updated.items() if k != "_id"}
+        return result
+
+    def delete(self, model: InLayersModel, id: PrimaryKeyType) -> None:
+        """Delete a document by ID."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        database = self.__get_database()
+        collection = database[collection_name]
+        collection.delete_one({"_id": id})
+
+    def search(self, model: InLayersModel, query: ModelSearch) -> ModelSearchResult:
+        """Search for documents matching the query."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(
+            self.__context.config.environment, model.get_model_definition()
+        )
+        database = self.__get_database()
+        collection = database[collection_name]
+
+        # Build aggregation pipeline
+        pipeline = []
+
+        # Add match stage if there's a query
+        if query.query:
+            mongo_query = to_mongo(query.query)
+            pipeline.extend(mongo_query)
+        else:
+            pipeline.append({"$match": {}})
+
+        # Add sort stage if needed
+        if query.sort:
+            sort_direction = 1 if query.sort.order == SortOrder.asc else -1
+            pipeline.append({"$sort": {query.sort.key: sort_direction}})
+
+        # Add limit stage if needed
+        if query.take:
+            pipeline.append({"$limit": query.take})
+
+        # Execute aggregation
+        results = list(collection.aggregate(pipeline))
+        instances = [{k: v for k, v in doc.items() if k != "_id"} for doc in results]
+
+        return Box(instances=instances, page=query.page)
+
+    def bulk_insert(self, model: InLayersModel, data: list[Mapping]) -> None:
+        """Bulk insert documents."""
+        from pymongo.operations import UpdateOne  # noqa: PLC0415
+
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        database = self.__get_database()
+        collection = database[collection_name]
+        pk_name = model.get_primary_key_name()
+
+        # Prepare bulk write operations
+        operations = []
+        for item in data:
+            payload = dict(item)
+            formatted = format_for_mongo(payload)
+
+            pk_value = formatted.get(pk_name)
+            if pk_value is None:
+                pk_value = str(uuid4())
+                formatted[pk_name] = pk_value
+
+            doc = {**formatted, "_id": pk_value}
+            operations.append(
+                UpdateOne(
+                    {"_id": pk_value},
+                    {"$set": doc},
+                    upsert=True,
+                )
+            )
+
+        if operations:
+            collection.bulk_write(operations)
+
+    def bulk_delete(self, model: InLayersModel, ids: list[PrimaryKeyType]) -> None:
+        """Bulk delete documents by IDs."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        database = self.__get_database()
+        collection = database[collection_name]
+        collection.delete_many({"_id": {"$in": ids}})
+
+    def dispose(self) -> None:
+        """Clean up resources."""
+        self.__disconnect()

in_layers/data/protocols.py

@@ -0,0 +1,43 @@
+from collections.abc import Mapping
+from enum import Enum
+from typing import Any, Literal, Protocol
+
+
+class SupportedBackend(Enum):
+    MongoDB = "mongodb"
+    DynamoDB = "dynamodb"
+
+
+class MongoBackendConfig(Protocol):
+    type: Literal[SupportedBackend.MongoDB]
+    host: str
+    port: int | None
+    username: str | None
+    password: str | None
+    database: str | None
+
+
+class DynamoDBBackendConfig(Protocol):
+    type: Literal[SupportedBackend.DynamoDB]
+    region: str | None
+    endpoint_url: str | None
+    aws_access_key_id: str | None
+    aws_secret_access_key: str | None
+    boto3: Any | None
+
+
+BackendConfig = MongoBackendConfig | DynamoDBBackendConfig
+
+
+class DataNamespace(Enum):
+    root = "in_layers_data"
+    backends = "in_layers_data_backends"
+
+
+class InLayersDataConfig(Protocol):
+    default: BackendConfig
+    model_to_backend: Mapping[str, BackendConfig] | None
+
+
+class WithInLayersDataConfig(Protocol):
+    config: InLayersDataConfig

in_layers/data/services.py

@@ -0,0 +1,91 @@
+from typing import Protocol
+
+from in_layers.core import create_error_object
+from in_layers.core.models.protocols import BackendProtocol, ModelDefinition
+
+from .backends.dynamodb.services import DynamoDBBackend
+from .backends.mongodb.services import MongoBackend
+from .protocols import (
+    BackendConfig,
+    SupportedBackend,
+    WithInLayersDataConfig,
+)
+
+
+class _ExpectedContext(Protocol):
+    config: WithInLayersDataConfig
+
+
+class InLayersDataServices:
+    def __init__(self, context: _ExpectedContext):
+        self.__context = context
+        self.__backend_by_unique_key = {}
+        self.__backends = None
+
+    def __initialize_backend(self, config: BackendConfig) -> BackendProtocol:
+        if config.type in [SupportedBackend.MongoDB, SupportedBackend.MongoDB.value]:
+            unique = MongoBackend.create_unique_connection_string(config)
+            if unique in self.__backend_by_unique_key:
+                return self.__backend_by_unique_key[unique]
+            backend = MongoBackend(self.__context, config)
+            self.__backend_by_unique_key[unique] = backend
+            return backend
+        elif config.type in [
+            SupportedBackend.DynamoDB,
+            SupportedBackend.DynamoDB.value,
+        ]:
+            unique = DynamoDBBackend.create_unique_connection_string(config)
+            if unique in self.__backend_by_unique_key:
+                return self.__backend_by_unique_key[unique]
+            backend = DynamoDBBackend(self.__context, config)
+            self.__backend_by_unique_key[unique] = backend
+            return backend
+        else:
+            raise ValueError(f"Unsupported backend type: {config.type}")
+
+    def __initialize_backends(self):
+        if self.__backends is not None:
+            return
+        config = self.__context.config.in_layers_data
+        default_backend_config = config.default
+        model_to_backend_config = getattr(config, "model_to_backend", {}) or {}
+
+        self.__backends = {"default": self.__initialize_backend(default_backend_config)}
+        for model_key, backend_config in model_to_backend_config.items():
+            backend_instance = self.__initialize_backend(backend_config)
+            self.__backends[model_key] = backend_instance
+
+    def __get_backend_for_model(self, meta: ModelDefinition) -> BackendProtocol:
+        self.__initialize_backends()
+        model_key_full = f"{meta.domain}.{meta.plural_name}"
+        if model_key_full in self.__backends:
+            return self.__backends[model_key_full]
+        if meta.domain in self.__backends:
+            return self.__backends[meta.domain]
+        return self.__backends["default"]
+
+    def get_model_backend(self, model_definition: ModelDefinition) -> BackendProtocol:
+        backend = self.__get_backend_for_model(model_definition)
+        if not backend:
+            raise ValueError(
+                f"No backend found for model {model_definition.domain}.{model_definition.plural_name}"
+            )
+        return backend
+
+    def dispose(self):
+        for backend in self.__backends.values():
+            try:
+                backend.dispose()
+            except Exception as e:
+                error_obj = create_error_object(
+                    "DATA_SERVICES_DISPOSE_ERROR", "Error disposing backend", e
+                )
+                log = self.__context.log.get_inner_logger("dispose")
+                log.warn("Error disposing backend", error_obj)
+
+        self.__backends = None
+        self.__backend_by_unique_key = {}
+
+
+def create(context: _ExpectedContext) -> InLayersDataServices:
+    return InLayersDataServices(context)

in_layers_data-0.4.20.dist-info/METADATA

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: in-layers-data
+Version: 0.4.20
+Summary: A data layer for In Layers Core. (in-layers-core). Part of the Node in Layers ecosystem
+License: GPLv3
+Keywords: layers,python,in-layers,node,domains,data,models,orm
+Author: Mike Cornwell
+Author-email: mike@mikecornwell.com
+Requires-Python: >=3.13,<4.0
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: in-layers-core (>=0.4.2,<0.5.0)
+Requires-Dist: pydantic (>=2.12.5,<3.0.0)
+Requires-Dist: python-box (>=7,<9)
+Description-Content-Type: text/markdown
+
+# In Layers Data 
+A data layer for the In Layers Core framework. 
+
+NOTE: There are no explicit dependencies on any database. To use a specific database you must install it in your own system. These databases are "imported in" as the databases are actually used at runtime.
+
+## How To Use
+1. Install in-layers-data
+1. Set `"in_layers_data"` to the `in_layers_core.models.model_backend` property
+1. Add `in_layers_data` configuration to your config
+1. Install database libraries to use. Example: `pymongo` or `boto3`
+
+### Configuration Example
+```python
+# config_base.py
+from box import Box
+def get_base_config():
+    return Box(
+        ...,
+        in_layers_core=Box(
+            ...
+            models=Box(
+                model_backend="in_layers_data",
+            )
+        ),
+        in_layers_data=Box(
+            default=Box(
+                type="mongodb"
+                # Connection information here
+            ),
+            # Optional: Set "domain" or "domain.ModelPluralNames" to a specific database configuration.
+            # model_to_backend=Box(
+            #     "domain.ModelPluralNames"=Box(
+            #         type="mongodb",
+            #         host="different-host"
+            #     )
+            # )
+        )
+    )
+
+
+```
+
+## Key Features
+- Drop in, Swappable Databases
+- Multi-database support
+- Low dependencies 
+
+## Databases Supported
+- Mongodb
+- Dynamodb
+
+## Database Info
+### Mongo 
+Mongodb requires `pymongo`
+
+### Dynamodb
+Dynamodb requires `boto3`
+
+#### Important
+Dynamodb is very poor at performing search queries. While this is implemented, it is not-recommended for use. Instead use the retrieve.

in_layers_data-0.4.20.dist-info/RECORD

@@ -0,0 +1,14 @@
+in_layers/__init__.py,sha256=VyrcAibWxTcBorfOipl5a1Vg9ZY8hOQt2T-lsWOGFps,173
+in_layers/data/__init__.py,sha256=eDl_a3x2QVMUauUSuTw0if180sdSER6DHYAip546d5M,387
+in_layers/data/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+in_layers/data/backends/dynamodb/__init__.py,sha256=q5OQ4ygMLYHGV_L30K4HSrVk10dROgNTrjUg32Ki2XM,114
+in_layers/data/backends/dynamodb/libs.py,sha256=wqzipG8FkMLmlt05jNmRwxvK6ycTVcfv3Wjra_r_sfU,3574
+in_layers/data/backends/dynamodb/services.py,sha256=d6UEX-xuh2sWJwqjxdZxvTiJUJ1NG_nLdg3DZU8GCe4,13838
+in_layers/data/backends/mongodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+in_layers/data/backends/mongodb/libs.py,sha256=HC0iSd2TibdXaQmXdO5YJP0Q4ItAMJdTaJ_w9c9GZmE,9001
+in_layers/data/backends/mongodb/services.py,sha256=pxF4P82ayaTGsnYoyqOHf3OYGKMdTHu8lMdQNmfUn8I,8607
+in_layers/data/protocols.py,sha256=yIH2Bu6K11cTBws-IrnopKB4-L_ltf6yLWDZAWec844,960
+in_layers/data/services.py,sha256=qPJCt4KzL7qfXWzXNzi8pnoeD7QSoZY-NpCB6KGKi74,3616
+in_layers_data-0.4.20.dist-info/METADATA,sha256=GhGC-KpdSWZiDG5vdDFoONHRDJJjtxLcCEiJ5ywnxQQ,2367
+in_layers_data-0.4.20.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+in_layers_data-0.4.20.dist-info/RECORD,,

in_layers_data-0.4.20.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.2.1
+Root-Is-Purelib: true
+Tag: py3-none-any