in-layers-data 0.4.0__tar.gz

in_layers_data-0.4.0/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: in-layers-data
+Version: 0.4.0
+Summary: A data layer for In Layers Core. (in-layers-core). Part of the Node in Layers ecosystem
+License: GPLv3
+Keywords: layers,python,in-layers,node,domains,data,models,orm
+Author: Mike Cornwell
+Author-email: mike@mikecornwell.com
+Requires-Python: >=3.13,<4.0
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: in-layers-core (>=0.4.2,<0.5.0)
+Requires-Dist: pydantic (>=2.12.5,<3.0.0)
+Requires-Dist: python-box (>=7,<9)
+Description-Content-Type: text/markdown
+
+# In Layers Data 
+A data layer for the In Layers Core framework. 
+
+NOTE: There are no explicit dependencies on any database. To use a specific database you must install it in your own system. These databases are "imported in" as the databases are actually used at runtime.
+
+## How To Use
+1. Install in-layers-data
+1. Set `"in_layers_data"` to the `in_layers_core.models.model_backend` property
+1. Add `in_layers_data` configuration to your config
+1. Install database libraries to use. Example: `pymongo` or `boto3`
+
+### Configuration Example
+```python
+# config_base.py
+from box import Box
+def get_base_config():
+    return Box(
+        ...,
+        in_layers_core=Box(
+            ...
+            models=Box(
+                model_backend="in_layers_data",
+            )
+        ),
+        in_layers_data=Box(
+            default=Box(
+                type="mongodb"
+                # Connection information here
+            ),
+            # Optional: Set "domain" or "domain.ModelPluralNames" to a specific database configuration.
+            # model_to_backend=Box(
+            #     "domain.ModelPluralNames"=Box(
+            #         type="mongodb",
+            #         host="different-host"
+            #     )
+            # )
+        )
+    )
+
+
+```
+
+## Key Features
+- Drop in, Swappable Databases
+- Multi-database support
+- Low dependencies 
+
+## Databases Supported
+- Mongodb
+- Dynamodb
+
+## Database Info
+### Mongo 
+Mongodb requires `pymongo`
+
+### Dynamodb
+Dynamodb requires `boto3`
+
+#### Important
+Dynamodb is very poor at performing search queries. While this is implemented, it is not-recommended for use. Instead use the retrieve.

in_layers_data-0.4.0/README.md

@@ -0,0 +1,60 @@
+# In Layers Data 
+A data layer for the In Layers Core framework. 
+
+NOTE: There are no explicit dependencies on any database. To use a specific database you must install it in your own system. These databases are "imported in" as the databases are actually used at runtime.
+
+## How To Use
+1. Install in-layers-data
+1. Set `"in_layers_data"` to the `in_layers_core.models.model_backend` property
+1. Add `in_layers_data` configuration to your config
+1. Install database libraries to use. Example: `pymongo` or `boto3`
+
+### Configuration Example
+```python
+# config_base.py
+from box import Box
+def get_base_config():
+    return Box(
+        ...,
+        in_layers_core=Box(
+            ...
+            models=Box(
+                model_backend="in_layers_data",
+            )
+        ),
+        in_layers_data=Box(
+            default=Box(
+                type="mongodb"
+                # Connection information here
+            ),
+            # Optional: Set "domain" or "domain.ModelPluralNames" to a specific database configuration.
+            # model_to_backend=Box(
+            #     "domain.ModelPluralNames"=Box(
+            #         type="mongodb",
+            #         host="different-host"
+            #     )
+            # )
+        )
+    )
+
+
+```
+
+## Key Features
+- Drop in, Swappable Databases
+- Multi-database support
+- Low dependencies 
+
+## Databases Supported
+- Mongodb
+- Dynamodb
+
+## Database Info
+### Mongo 
+Mongodb requires `pymongo`
+
+### Dynamodb
+Dynamodb requires `boto3`
+
+#### Important
+Dynamodb is very poor at performing search queries. While this is implemented, it is not-recommended for use. Instead use the retrieve.

in_layers_data-0.4.0/pyproject.toml

@@ -0,0 +1,204 @@
+[project]
+name = "in-layers-data"
+version = "0.4.0"
+description = "A data layer for In Layers Core. (in-layers-core). Part of the Node in Layers ecosystem"
+authors = [
+    {name = "Mike Cornwell",email = "mike@mikecornwell.com"}
+]
+license = {text = "GPLv3"}
+readme = "README.md"
+homepage = "https://github.com/node-in-layers/python-in-layers-data"  
+repository = "https://github.com/node-in-layers/python-in-layers-data"
+keywords = ["layers", "python", "in-layers", "node", "domains", "data", "models", "orm"]
+requires-python = ">=3.13,<4.0"
+dependencies = [
+    "httpx (>=0.28.1,<0.29.0)",
+    "python-box (>=7,<9)",
+    "pydantic (>=2.12.5,<3.0.0)",
+    "in-layers-core (>=0.4.2,<0.5.0)"
+]
+
+[tool.poetry]
+name = "in-layers-data"
+version = "0.1.0"
+description = ""
+authors = []
+packages = [
+    { include = "in_layers", from = "src" }
+]
+
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[dependency-groups]
+dev = [
+    "black>=24.0.0",
+    "ruff>=0.7.0",
+    "isort>=5.12.0",
+    "mypy>=1.12.0",
+    "pytest>=8.0.0",
+    "commitizen (>=3.13,<4.0)",
+    "pytest-cov (>=7.0.0,<8.0.0)",
+    "build (>=1.3.0,<2.0.0)",
+    "twine (>=6.2.0,<7.0.0)",
+    "pymongo (>=4.15.5,<5.0.0)",
+    "behave>=1.2.6",
+    "boto3 (>=1.42.13,<2.0.0)"
+]
+
+[tool.poetry.dependencies]
+python = ">=3.13,<4.0"
+python-box = ">=7,<9"
+httpx = ">=0.28.1,<0.29.0"
+
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+lines_after_imports = 2
+
+
+# ================================================================
+# BLACK (code formatting)
+# ================================================================
+[tool.black]
+line-length = 88
+target-version = ["py313"]
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.venv
+  | venv
+  | build
+  | dist
+  | __pycache__
+  | \.mypy_cache
+  | \.ruff_cache
+)/
+'''
+
+# ================================================================
+# RUFF (linting + import sorting + auto-fixes)
+# ================================================================
+[tool.ruff]
+target-version = "py313"
+line-length = 88
+# Run Ruff from the root; it will traverse all subdirs except excluded.
+exclude = [
+  ".git",
+  ".venv",
+  "venv",
+  "./features",
+  "__pycache__",
+  "build",
+  "dist",
+  ".mypy_cache",
+  ".ruff_cache",
+  "tests",
+  "*/tests",
+  "**/tests"
+]
+
+[tool.ruff.lint]
+select = [
+  "E",   # pycodestyle errors
+  "W",   # pycodestyle warnings
+  "F",   # Pyflakes
+  "B",   # flake8-bugbear (common bugs / bad patterns)
+  "I",   # isort (import sorting)
+  "UP",  # pyupgrade (modern syntax)
+  "N",   # pep8-naming (function, class, etc. naming)
+  "C90", # mccabe (complexity)
+  "SIM", # flake8-simplify
+  "S",   # flake8-bandit (basic security checks)
+  "DTZ", # flake8-datetimez (timezone-aware datetimes)
+  "T20", # flake8-print (no stray prints in non-test code)
+  "ARG", # flake8-unused-arguments
+  "PL",  # pylint-like checks (subset)
+  "RUF", # Ruff-specific rules
+]
+
+# Rules we choose to ignore or relax:
+ignore = [
+  "E501",  # Let Black handle line length
+  "W291",  # Trailing whitespace (Black fixes this)
+]
+
+# Per-file ignore examples (customize as desired):
+[tool.ruff.lint.per-file-ignores]
+"**/tests/**" = [
+  "S101",   # Allow 'assert' in tests
+  "T201",   # Allow 'print' in tests if you wish
+]
+"orchestration/scripts/**" = [
+  "T201",   # Allow print in CLI-style scripts
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 10
+
+[tool.ruff.lint.pylint]
+max-args = 8
+
+# Optionally enable Ruff as a formatter (complementing Black):
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+line-ending = "auto"
+
+# -------------------------------------------------------------------
+# Mypy configuration (type checking / "code lensing")
+# -------------------------------------------------------------------
+[tool.mypy]
+ignore_missing_imports = true
+warn_unused_ignores = true
+warn_redundant_casts = true
+warn_unreachable = true
+disallow_untyped_defs = true        # Functions must be typed
+disallow_incomplete_defs = true     # Arguments & returns must be fully typed
+disallow_untyped_calls = false      # Relaxed for now; can tighten later
+disallow_untyped_decorators = false
+no_implicit_optional = true
+strict_equality = true
+show_error_codes = true
+pretty = true
+exclude = '''
+/(
+    \.git
+  | \.venv
+  | build
+  | dist
+  | __pycache__
+  | \.mypy_cache
+  | tmp
+)/
+'''
+
+# Optional: per-module/per-package overrides
+[[tool.mypy.overrides]]
+module = "tests.*"
+ignore_errors = true
+
+# -------------------------------------------------------------------
+# Pytest configuration (unit tests)
+# -------------------------------------------------------------------
+[tool.pytest.ini_options]
+minversion = "8.0"
+addopts = "-ra -q"
+pythonpath = ["src"]
+python_files = [
+    "test_*.py",
+    "*_test.py",
+    "test_*.py",
+    "*-test.py",
+]
+[tool.commitizen]
+name = "cz_conventional_commits"
+tag_format = "$version"
+version_scheme = "pep440"
+version_provider = "pep621"

in_layers_data-0.4.0/src/in_layers/__init__.py

@@ -0,0 +1,7 @@
+"""Namespace package for in_layers."""
+
+from __future__ import annotations
+
+import pkgutil
+
+__path__ = pkgutil.extend_path(__path__, __name__)  # type: ignore[name-defined]

in_layers_data-0.4.0/src/in_layers/data/__init__.py

@@ -0,0 +1,21 @@
+from . import services
+from .protocols import (
+    BackendConfig,
+    DataNamespace,
+    DynamoDBBackendConfig,
+    InLayersDataConfig,
+    MongoBackendConfig,
+    SupportedBackend,
+)
+
+name = DataNamespace.root
+
+__all__ = [
+    "BackendConfig",
+    "DynamoDBBackendConfig",
+    "InLayersDataConfig",
+    "MongoBackendConfig",
+    "SupportedBackend",
+    "name",
+    "services",
+]

in_layers_data-0.4.0/src/in_layers/data/backends/dynamodb/__init__.py

@@ -0,0 +1,5 @@
+"""DynamoDB backend for InLayers models."""
+
+from .services import DynamoDBBackend
+
+__all__ = ["DynamoDBBackend"]

in_layers_data-0.4.0/src/in_layers/data/backends/dynamodb/libs.py

@@ -0,0 +1,84 @@
+"""Pure functional utilities for DynamoDB query conversion and data formatting."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any
+
+from in_layers.core.models.protocols import ModelDefinition
+
+
+def get_table_name_for_model(model_definition: ModelDefinition) -> str:
+    """Generate a DynamoDB table name from a model definition."""
+    name = model_definition.plural_name.replace("@", "").replace("/", "-")
+    # Convert to kebab-case: insert hyphens before uppercase letters (except first)
+    # and handle sequences of uppercase letters
+    name = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", name)
+    name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1-\2", name)
+    return name.lower()
+
+
+def format_for_dynamodb(data: Mapping[str, Any]) -> dict[str, Any]:
+    """Format data for DynamoDB storage.
+
+    DynamoDB natively handles:
+    - Strings
+    - Numbers (int, float, Decimal)
+    - Binary data
+    - Boolean
+    - Null
+    - Lists
+    - Maps (dicts)
+    - Sets (string sets, number sets, binary sets)
+
+    The boto3 DynamoDBDocumentClient will handle conversion automatically,
+    but we ensure datetime objects are converted to ISO format strings.
+    """
+    result = dict(data)
+    # Convert datetime objects to ISO format strings for DynamoDB
+
+    for key, value in result.items():
+        if isinstance(value, datetime):
+            result[key] = value.isoformat()
+    return result
+
+
+def from_dynamodb(item: dict[str, Any] | None) -> dict[str, Any]:
+    """Convert a DynamoDB item to a plain dictionary.
+
+    The boto3 DynamoDBDocumentClient already converts AttributeValue format
+    to native Python types, so this is mainly for consistency and future-proofing.
+    """
+    if item is None:
+        return {}
+    return dict(item)
+
+
+def build_scan_params(
+    table_name: str, exclusive_start_key: dict[str, Any] | None = None
+) -> dict[str, Any]:
+    """Build parameters for a DynamoDB Scan operation."""
+    params: dict[str, Any] = {
+        "TableName": table_name,
+    }
+    if exclusive_start_key:
+        params["ExclusiveStartKey"] = exclusive_start_key
+    return params
+
+
+def split_array_into_batches(array: list[Any], max_batch_size: int) -> list[list[Any]]:
+    """Split an array into batches of maximum size.
+
+    DynamoDB has limits on batch operations (e.g., BatchWriteItem max 25 items).
+    """
+    if not isinstance(array, list):
+        raise ValueError("Input must be a list")
+    if max_batch_size < 1:
+        raise ValueError("max_batch_size must be at least 1")
+
+    batches = []
+    for i in range(0, len(array), max_batch_size):
+        batches.append(array[i : i + max_batch_size])
+    return batches

in_layers_data-0.4.0/src/in_layers/data/backends/dynamodb/services.py

@@ -0,0 +1,318 @@
+"""DynamoDB backend implementation for InLayers models."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from uuid import uuid4
+
+from box import Box
+from in_layers.core.models.backends import (
+    _apply_sort,
+    _apply_take,
+    _matches_query_tokens,
+)
+from in_layers.core.models.protocols import (
+    InLayersModel,
+    ModelSearch,
+    ModelSearchResult,
+    PrimaryKeyType,
+)
+
+from in_layers.data.protocols import DynamoDBBackendConfig
+
+from .libs import (
+    format_for_dynamodb,
+    from_dynamodb,
+    get_table_name_for_model,
+    split_array_into_batches,
+)
+
+# DynamoDB batch operation limits
+MAX_BATCH_WRITE_SIZE = 25
+SCAN_RETURN_THRESHOLD = 1000
+
+
+class DynamoDBBackend:
+    """DynamoDB backend implementation."""
+
+    def __init__(self, config: DynamoDBBackendConfig):
+        self.__config = config
+        self.__client: Any = None
+        self.__table_client: Any = None
+
+    @staticmethod
+    def create_unique_connection_string(config: DynamoDBBackendConfig) -> str:
+        """Create a unique connection string from config."""
+        region = config.region or "us-east-1"
+        endpoint_url = config.endpoint_url or ""
+
+        parts = [f"region={region}"]
+        if endpoint_url:
+            parts.append(f"endpoint={endpoint_url}")
+
+        return "|".join(parts)
+
+    def __connect(self) -> None:
+        """Connect to DynamoDB (private method)."""
+        # Use boto3 from config if provided (for testing), otherwise import it
+        if self.__config.boto3 is not None:
+            boto3 = self.__config.boto3
+        else:
+            import boto3  # pragma: no cover # noqa: PLC0415
+
+        # Build client configuration
+        client_kwargs: dict[str, Any] = {}
+        if self.__config.region:
+            client_kwargs["region_name"] = self.__config.region
+        if self.__config.endpoint_url:
+            client_kwargs["endpoint_url"] = self.__config.endpoint_url
+        if self.__config.aws_access_key_id and self.__config.aws_secret_access_key:
+            client_kwargs["aws_access_key_id"] = self.__config.aws_access_key_id
+            client_kwargs["aws_secret_access_key"] = self.__config.aws_secret_access_key
+
+        # Create DynamoDB client
+        self.__client = boto3.client("dynamodb", **client_kwargs)
+
+        # Create DynamoDB resource for easier table operations
+        dynamodb_resource = boto3.resource("dynamodb", **client_kwargs)
+        self.__table_client = dynamodb_resource
+
+    def __disconnect(self) -> None:
+        """Disconnect from DynamoDB (private method)."""
+        # boto3 clients don't need explicit closing, but we can clear references
+        self.__client = None
+        self.__table_client = None
+
+    def __ensure_connected(self) -> None:
+        """Ensure DynamoDB connection is established."""
+        if self.__client is None:
+            self.__connect()
+
+    def create(self, model: InLayersModel, data: Mapping) -> Mapping:
+        """Create a new item in DynamoDB."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(model.get_model_definition())
+        table = self.__table_client.Table(table_name)
+
+        payload = dict(data)
+        formatted = format_for_dynamodb(payload)
+
+        pk_name = model.get_primary_key_name()
+        pk_value = formatted.get(pk_name)
+        if pk_value is None:
+            pk_value = str(uuid4())
+            formatted[pk_name] = pk_value
+
+        # Ensure primary key is a string (DynamoDB requirement)
+        formatted[pk_name] = str(pk_value)
+
+        # Put item in DynamoDB
+        table.put_item(Item=formatted)
+        return formatted
+
+    def retrieve(self, model: InLayersModel, id: PrimaryKeyType) -> Mapping | None:
+        """Retrieve an item by ID."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(model.get_model_definition())
+        table = self.__table_client.Table(table_name)
+
+        pk_name = model.get_primary_key_name()
+        key = {pk_name: str(id)}
+
+        response = table.get_item(Key=key)
+        item = response.get("Item")
+        if not item:
+            return None
+
+        return from_dynamodb(item)
+
+    def update(
+        self, model: InLayersModel, id: PrimaryKeyType, data: Mapping
+    ) -> Mapping:
+        """Update an item by ID."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(model.get_model_definition())
+        table = self.__table_client.Table(table_name)
+
+        pk_name = model.get_primary_key_name()
+        key = {pk_name: str(id)}
+
+        # Check if item exists
+        existing = table.get_item(Key=key)
+        if "Item" not in existing:
+            raise KeyError(f"Instance with id {id!r} not found")
+
+        payload = dict(data)
+        formatted = format_for_dynamodb(payload)
+
+        # Ensure primary key field remains consistent
+        formatted[pk_name] = str(id)
+
+        # Update item in DynamoDB
+        table.put_item(Item=formatted)
+        return formatted
+
+    def delete(self, model: InLayersModel, id: PrimaryKeyType) -> None:
+        """Delete an item by ID."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(model.get_model_definition())
+        table = self.__table_client.Table(table_name)
+
+        pk_name = model.get_primary_key_name()
+        key = {pk_name: str(id)}
+
+        table.delete_item(Key=key)
+
+    def search(self, model: InLayersModel, query: ModelSearch) -> ModelSearchResult:
+        """Search for items matching the query.
+
+        Note: DynamoDB Scan operations are expensive and should be used sparingly.
+        For production use, consider using Query operations with Global Secondary Indexes (GSI)
+        or Local Secondary Indexes (LSI) for better performance.
+
+        This implementation continues scanning across pages until:
+        - Threshold is met (SCAN_RETURN_THRESHOLD if no take specified)
+        - No more keys (LastEvaluatedKey is null)
+        - Take limit is reached (if take is specified)
+        """
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(model.get_model_definition())
+        table = self.__table_client.Table(table_name)
+
+        # Start recursive scanning
+        result = self._do_search_until_threshold_or_no_last_evaluated_key(
+            table, query, []
+        )
+
+        return Box(instances=result["instances"], page=result["page"])
+
+    def _do_search_until_threshold_or_no_last_evaluated_key(
+        self,
+        table: Any,
+        query: ModelSearch,
+        old_instances_found: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        """Recursively scan DynamoDB until threshold is met or no more keys.
+
+        This matches the TypeScript implementation's behavior of continuing
+        to scan across pages until enough filtered results are found.
+        """
+        # Build scan parameters
+        scan_kwargs: dict[str, Any] = {}
+        if query.page:
+            scan_kwargs["ExclusiveStartKey"] = query.page
+
+        # Execute scan
+        response = table.scan(**scan_kwargs)
+        items = response.get("Items", [])
+
+        # Convert DynamoDB items to plain dicts
+        unfiltered = [from_dynamodb(item) for item in items]
+
+        # Apply filtering using the same logic as MemoryBackend
+        filtered = [r for r in unfiltered if _matches_query_tokens(r, query.query)]
+
+        # Combine with previously found instances
+        all_filtered = filtered + old_instances_found
+
+        # Determine threshold
+        using_take = query.take is not None and query.take > 0
+        take = query.take if using_take else SCAN_RETURN_THRESHOLD
+
+        # Get pagination key
+        last_evaluated_key = response.get("LastEvaluatedKey")
+
+        # Check stopping conditions:
+        # 1. We have enough results (more than threshold)
+        # 2. No more keys to evaluate
+        # 3. If using take, we've hit our max
+        # Note: TypeScript uses > (strictly greater), meaning we continue if we have exactly 'take' items
+        stop_for_threshold = len(all_filtered) > take
+        stop_for_no_more = last_evaluated_key is None
+
+        if stop_for_threshold or stop_for_no_more:
+            # Apply sorting and take limit
+            sorted_instances = _apply_sort(all_filtered, query.sort)
+            limited_instances = _apply_take(sorted_instances, query.take)
+
+            # Return page: null when using take, otherwise return LastEvaluatedKey
+            page = None if using_take else last_evaluated_key
+
+            return {
+                "instances": [dict(x) for x in limited_instances],
+                "page": page,
+            }
+
+        # Continue scanning with the new page key
+        # Create a new ModelSearch with updated page (frozen dataclass requires new instance)
+        new_query = ModelSearch(
+            query=query.query,
+            take=query.take,
+            sort=query.sort,
+            page=last_evaluated_key,
+        )
+        return self._do_search_until_threshold_or_no_last_evaluated_key(
+            table, new_query, all_filtered
+        )
+
+    def bulk_insert(self, model: InLayersModel, data: list[Mapping]) -> None:
+        """Bulk insert items."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(model.get_model_definition())
+        table = self.__table_client.Table(table_name)
+        pk_name = model.get_primary_key_name()
+
+        # Prepare items
+        items = []
+        for item in data:
+            payload = dict(item)
+            formatted = format_for_dynamodb(payload)
+
+            pk_value = formatted.get(pk_name)
+            if pk_value is None:
+
+                pk_value = str(uuid4())
+                formatted[pk_name] = pk_value
+
+            formatted[pk_name] = str(pk_value)
+            items.append(formatted)
+
+        # Split into batches (DynamoDB BatchWriteItem limit is 25)
+        batches = split_array_into_batches(items, MAX_BATCH_WRITE_SIZE)
+
+        # Write batches
+        for batch in batches:
+            with table.batch_writer() as writer:
+                for item in batch:
+                    writer.put_item(Item=item)
+
+    def bulk_delete(self, model: InLayersModel, ids: list[PrimaryKeyType]) -> None:
+        """Bulk delete items by IDs."""
+        self.__ensure_connected()
+
+        table_name = get_table_name_for_model(model.get_model_definition())
+        table = self.__table_client.Table(table_name)
+        pk_name = model.get_primary_key_name()
+
+        # Prepare keys
+        keys = [{pk_name: str(id)} for id in ids]
+
+        # Split into batches
+        batches = split_array_into_batches(keys, MAX_BATCH_WRITE_SIZE)
+
+        # Delete batches
+        for batch in batches:
+            with table.batch_writer() as writer:
+                for key in batch:
+                    writer.delete_item(Key=key)
+
+    def dispose(self) -> None:
+        """Clean up resources."""
+        self.__disconnect()

in_layers_data-0.4.0/src/in_layers/data/backends/mongodb/libs.py

@@ -0,0 +1,229 @@
+"""Pure functional utilities for MongoDB query conversion and data formatting."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any
+
+from in_layers.core.models.protocols import (
+    BooleanQuery,
+    DatastoreValueType,
+    EqualitySymbol,
+    ModelDefinition,
+    PropertyQuery,
+    QueryTokens,
+)
+
+
+def get_collection_name_for_model(model_definition: ModelDefinition) -> str:
+    """Generate a MongoDB collection name from a model definition."""
+    name = model_definition.plural_name.replace("@", "").replace("/", "-")
+    # Convert to kebab-case: insert hyphens before uppercase letters (except first)
+    # and handle sequences of uppercase letters
+    name = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", name)
+    name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1-\2", name)
+    return name.lower()
+
+
+def escape_regex(s: str) -> str:
+    """Escape special regex characters in a string."""
+    return re.escape(s)
+
+
+def build_string_pattern(
+    raw: str, starts_with: bool, ends_with: bool, includes: bool
+) -> str:
+    """Build a regex pattern for string matching."""
+    escaped = escape_regex(raw)
+    if starts_with:
+        return f"^{escaped}"
+    if ends_with:
+        return f"{escaped}$"
+    if includes:
+        return escaped
+    # default: exact match
+    return f"^{escaped}$"
+
+
+def regex_object_for_pattern(pattern: str, case_sensitive: bool) -> dict[str, Any]:
+    """Create a MongoDB regex object from a pattern."""
+    if case_sensitive:
+        return {"$regex": pattern}
+    return {"$regex": pattern, "$options": "i"}
+
+
+def build_mongo_find_value(query: PropertyQuery) -> dict[str, Any]:  # noqa: PLR0911
+    """Convert a PropertyQuery to a MongoDB query value."""
+    value = query.value
+    if value is None:
+        return {query.key: None}
+
+    # Handle date objects
+    if hasattr(value, "isoformat"):  # datetime objects
+        return {query.key: value}
+
+    if query.value_type == DatastoreValueType.string:
+        case_sensitive = bool(query.options.case_sensitive)
+        starts_with = bool(query.options.starts_with)
+        ends_with = bool(query.options.ends_with)
+        includes = bool(query.options.includes)
+
+        if query.equality_symbol not in (EqualitySymbol.eq, EqualitySymbol.ne):
+            raise ValueError(
+                f"Symbol {query.equality_symbol} is unhandled for string type"
+            )
+
+        raw = str(value)
+        pattern = build_string_pattern(raw, starts_with, ends_with, includes)
+        regex_obj = regex_object_for_pattern(pattern, case_sensitive)
+
+        if query.equality_symbol == EqualitySymbol.ne:
+            is_plain_exact = (
+                not starts_with and not ends_with and not includes and case_sensitive
+            )
+            if is_plain_exact:
+                return {query.key: {"$ne": raw}}
+            return {query.key: {"$not": regex_obj}}
+
+        use_regex = starts_with or ends_with or includes or not case_sensitive
+        if use_regex:
+            return {query.key: regex_obj}
+        return {query.key: raw}
+
+    if query.value_type == DatastoreValueType.number:
+        equality_symbol_to_mongo = {
+            EqualitySymbol.eq: "$eq",
+            EqualitySymbol.gt: "$gt",
+            EqualitySymbol.gte: "$gte",
+            EqualitySymbol.lt: "$lt",
+            EqualitySymbol.lte: "$lte",
+            EqualitySymbol.ne: "$ne",
+        }
+        mongo_symbol = equality_symbol_to_mongo.get(query.equality_symbol)
+        if not mongo_symbol:
+            raise ValueError(f"Symbol {query.equality_symbol} is unhandled")
+        return {query.key: {mongo_symbol: query.value}}
+
+    return {query.key: value}
+
+
+def threeitize(
+    data: list[QueryTokens],
+) -> list[tuple[QueryTokens, BooleanQuery, QueryTokens]]:
+    """Convert a list of tokens into three-tuples of (left, link, right)."""
+    if len(data) in (0, 1):
+        return []
+    if len(data) % 2 == 0:
+        raise ValueError("Must be an odd number of 3 or greater.")
+    three = (data[0], _as_link(data[1]), data[2])
+    rest = data[2:]
+    more = threeitize(rest)
+    return [three, *more]
+
+
+def _as_link(value: QueryTokens) -> BooleanQuery:
+    """Convert a token to a BooleanQuery link."""
+    if value in {"AND", "OR"}:
+        return value
+    raise ValueError("Must have AND/OR between statements")
+
+
+def process_mongo_array(
+    tokens: list[QueryTokens],
+) -> dict[str, Any]:
+    """Process an array of query tokens into a MongoDB query."""
+    # If we don't have any AND/OR, it's all an AND
+    if all(t != "AND" and t != "OR" for t in tokens):  # noqa: PLR1714
+        return {"$and": [handle_mongo_query(t) for t in tokens]}
+
+    # Process with threeitize
+    threes = threeitize(tokens)
+    threes.reverse()
+    result: dict[str, Any] = {}
+    for a, link, b in threes:
+        a_query = handle_mongo_query(a)
+        if result:
+            result = {f"${link.lower()}": [a_query, result]}
+        else:
+            b_query = handle_mongo_query(b)
+            result = {f"${link.lower()}": [a_query, b_query]}
+    return result
+
+
+def handle_mongo_query(token: QueryTokens | list[QueryTokens]) -> dict[str, Any]:
+    """Convert a query token to a MongoDB query object."""
+    # Handle list of tokens (the main query)
+    if isinstance(token, list):
+        return process_mongo_array(token)
+
+    # Check by shape (duck typing) rather than isinstance
+    # PropertyQuery has type="property" and value attribute
+    if (
+        hasattr(token, "type")
+        and getattr(token, "type", None) == "property"
+        and hasattr(token, "value")
+    ):
+        return build_mongo_find_value(token)
+
+    # DatesBeforeQuery has type="datesBefore" and date attribute
+    if (
+        hasattr(token, "type")
+        and getattr(token, "type", None) == "datesBefore"
+        and hasattr(token, "date")
+    ):
+        date_value = token.date
+        if (
+            hasattr(token, "value_type") and token.value_type == DatastoreValueType.date
+        ) and isinstance(date_value, str):
+            # Convert string to datetime if needed
+            date_value = datetime.fromisoformat(date_value.replace("Z", "+00:00"))
+        operator = (
+            "$lte"
+            if (hasattr(token, "options") and token.options.equal_to_and_before)
+            else "$lt"
+        )
+        return {token.key: {operator: date_value}}
+
+    # DatesAfterQuery has type="datesAfter" and date attribute
+    if (
+        hasattr(token, "type")
+        and getattr(token, "type", None) == "datesAfter"
+        and hasattr(token, "date")
+    ):
+        date_value = token.date
+        if (
+            hasattr(token, "value_type") and token.value_type == DatastoreValueType.date
+        ) and isinstance(date_value, str):
+            date_value = datetime.fromisoformat(date_value.replace("Z", "+00:00"))
+        operator = (
+            "$gte"
+            if (hasattr(token, "options") and token.options.equal_to_and_after)
+            else "$gt"
+        )
+        return {token.key: {operator: date_value}}
+
+    raise ValueError(f"Unhandled query token {token}")
+
+
+def to_mongo(query: list[QueryTokens]) -> list[dict[str, Any]]:
+    """Convert a query list to MongoDB aggregation pipeline stages."""
+    if not query:
+        return [{"$match": {}}]
+    # Pass the list directly to handle_mongo_query, which will process it as an array
+    match_query = handle_mongo_query(query)
+    return [{"$match": match_query}]
+
+
+def format_for_mongo(data: Mapping[str, Any]) -> dict[str, Any]:
+    """Format data for MongoDB storage, converting dates and other types."""
+    result = dict(data)
+    # Convert datetime objects (they're already in the right format for MongoDB)
+    # In the TypeScript version, this iterates over model properties to find Datetime types
+    # For Python, we'll preserve datetime objects as-is (MongoDB handles them natively)
+    # and convert string ISO dates if they're clearly datetime values
+    for key, value in result.items():
+        if isinstance(value, datetime):
+            result[key] = value
+    return result

in_layers_data-0.4.0/src/in_layers/data/backends/mongodb/services.py

@@ -0,0 +1,225 @@
+"""MongoDB backend implementation for InLayers models."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+from uuid import uuid4
+
+from box import Box
+from in_layers.core.models.protocols import (
+    InLayersModel,
+    ModelSearch,
+    ModelSearchResult,
+    PrimaryKeyType,
+    SortOrder,
+)
+
+from in_layers.data.protocols import MongoBackendConfig
+
+from .libs import (
+    format_for_mongo,
+    get_collection_name_for_model,
+    to_mongo,
+)
+
+
+class MongoBackend:
+    """MongoDB backend implementation."""
+
+    def __init__(self, config: MongoBackendConfig):
+        self.__config = config
+        self.__client: Any = None
+        self.__db: Any = None
+
+    @staticmethod
+    def create_unique_connection_string(config: MongoBackendConfig) -> str:
+        """Create a unique connection string from config."""
+        host = config.host
+        port = config.port
+        username = config.username
+        password = config.password
+        database = config.database
+
+        if username and password:
+            return f"mongodb://{username}:{password}@{host}:{port}/{database}"
+        return f"mongodb://{host}:{port}/{database}"
+
+    def __connect(self) -> None:
+        """Connect to MongoDB (private method)."""
+        from pymongo import MongoClient  # noqa: PLC0415
+
+        connection_string = self.create_unique_connection_string(
+            {
+                "host": self.__config.host,
+                "port": self.__config.port,
+                "username": self.__config.username,
+                "password": self.__config.password,
+                "database": self.__config.database,
+            }
+        )
+        self.__client = MongoClient(connection_string)
+        self.__db = self.__client[self.__config.database]
+
+    def __disconnect(self) -> None:
+        """Disconnect from MongoDB (private method)."""
+        if self.__client:
+            self.__client.close()
+            self.__client = None
+            self.__db = None
+
+    def __ensure_connected(self) -> None:
+        """Ensure MongoDB connection is established."""
+        if self.__db is None:
+            self.__connect()
+
+    def create(self, model: InLayersModel, data: Mapping) -> Mapping:
+        """Create a new document in MongoDB."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        collection = self.__db[collection_name]
+        payload = dict(data)
+        formatted = format_for_mongo(payload)
+
+        pk_name = model.get_primary_key_name()
+        pk_value = formatted.get(pk_name)
+        if pk_value is None:
+            # Generate a simple ID - in production you might want UUID
+
+            pk_value = str(uuid4())
+            formatted[pk_name] = pk_value
+
+        # Use _id as MongoDB's primary key, mapping from model's primary key
+        insert_data = {**formatted, "_id": pk_value}
+        collection.insert_one(insert_data)
+        # Return without _id
+        result = {k: v for k, v in insert_data.items() if k != "_id"}
+        return result
+
+    def retrieve(self, model: InLayersModel, id: PrimaryKeyType) -> Mapping | None:
+        """Retrieve a document by ID."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        collection = self.__db[collection_name]
+        doc = collection.find_one({"_id": id})
+        if not doc:
+            return None
+        # Remove _id and return
+        result = {k: v for k, v in doc.items() if k != "_id"}
+        return result
+
+    def update(
+        self, model: InLayersModel, id: PrimaryKeyType, data: Mapping
+    ) -> Mapping:
+        """Update a document by ID."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        collection = self.__db[collection_name]
+
+        # Check if document exists
+        existing = collection.find_one({"_id": id})
+        if not existing:
+            raise KeyError(f"Instance with id {id!r} not found")
+
+        payload = dict(data)
+        formatted = format_for_mongo(payload)
+
+        # Ensure primary key field remains consistent
+        pk_name = model.get_primary_key_name()
+        formatted[pk_name] = id
+
+        # Update with _id mapping
+        update_data = {**formatted, "_id": id}
+        collection.update_one({"_id": id}, {"$set": update_data})
+
+        # Return without _id
+        result = {k: v for k, v in update_data.items() if k != "_id"}
+        return result
+
+    def delete(self, model: InLayersModel, id: PrimaryKeyType) -> None:
+        """Delete a document by ID."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        collection = self.__db[collection_name]
+        collection.delete_one({"_id": id})
+
+    def search(self, model: InLayersModel, query: ModelSearch) -> ModelSearchResult:
+        """Search for documents matching the query."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        collection = self.__db[collection_name]
+
+        # Build aggregation pipeline
+        pipeline = []
+
+        # Add match stage if there's a query
+        if query.query:
+            mongo_query = to_mongo(query.query)
+            pipeline.extend(mongo_query)
+        else:
+            pipeline.append({"$match": {}})
+
+        # Add sort stage if needed
+        if query.sort:
+            sort_direction = 1 if query.sort.order == SortOrder.asc else -1
+            pipeline.append({"$sort": {query.sort.key: sort_direction}})
+
+        # Add limit stage if needed
+        if query.take:
+            pipeline.append({"$limit": query.take})
+
+        # Execute aggregation
+        results = list(collection.aggregate(pipeline))
+        instances = [{k: v for k, v in doc.items() if k != "_id"} for doc in results]
+
+        return Box(instances=instances, page=query.page)
+
+    def bulk_insert(self, model: InLayersModel, data: list[Mapping]) -> None:
+        """Bulk insert documents."""
+        from pymongo.operations import UpdateOne  # noqa: PLC0415
+
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        collection = self.__db[collection_name]
+        pk_name = model.get_primary_key_name()
+
+        # Prepare bulk write operations
+        operations = []
+        for item in data:
+            payload = dict(item)
+            formatted = format_for_mongo(payload)
+
+            pk_value = formatted.get(pk_name)
+            if pk_value is None:
+                pk_value = str(uuid4())
+                formatted[pk_name] = pk_value
+
+            doc = {**formatted, "_id": pk_value}
+            operations.append(
+                UpdateOne(
+                    {"_id": pk_value},
+                    {"$set": doc},
+                    upsert=True,
+                )
+            )
+
+        if operations:
+            collection.bulk_write(operations)
+
+    def bulk_delete(self, model: InLayersModel, ids: list[PrimaryKeyType]) -> None:
+        """Bulk delete documents by IDs."""
+        self.__ensure_connected()
+
+        collection_name = get_collection_name_for_model(model.get_model_definition())
+        collection = self.__db[collection_name]
+        collection.delete_many({"_id": {"$in": ids}})
+
+    def dispose(self) -> None:
+        """Clean up resources."""
+        self.__disconnect()

in_layers_data-0.4.0/src/in_layers/data/protocols.py

@@ -0,0 +1,43 @@
+from collections.abc import Mapping
+from enum import Enum
+from typing import Any, Protocol
+
+
+class SupportedBackend(Enum):
+    MongoDB = "mongodb"
+    DynamoDB = "dynamodb"
+
+
+class MongoBackendConfig(Protocol):
+    type: SupportedBackend
+    host: str
+    port: int | None
+    username: str | None
+    password: str | None
+    database: str | None
+
+
+class DynamoDBBackendConfig(Protocol):
+    type: SupportedBackend
+    region: str | None
+    endpoint_url: str | None
+    aws_access_key_id: str | None
+    aws_secret_access_key: str | None
+    boto3: Any | None
+
+
+BackendConfig = MongoBackendConfig | DynamoDBBackendConfig
+
+
+class DataNamespace(Enum):
+    root = "in_layers_data"
+    backends = "in_layers_data_backends"
+
+
+class InLayersDataConfig(Protocol):
+    default: BackendConfig
+    model_to_backend: Mapping[str, BackendConfig] | None
+
+
+class WithInLayersDataConfig(Protocol):
+    config: InLayersDataConfig

in_layers_data-0.4.0/src/in_layers/data/services.py

@@ -0,0 +1,73 @@
+from typing import Protocol
+
+from in_layers.core.models.protocols import BackendProtocol, ModelDefinition
+
+from .backends.dynamodb.services import DynamoDBBackend
+from .backends.mongodb.services import MongoBackend
+from .protocols import (
+    BackendConfig,
+    SupportedBackend,
+    WithInLayersDataConfig,
+)
+
+
+class _ExpectedContext(Protocol):
+    config: WithInLayersDataConfig
+
+
+class InLayersDataServices:
+    def __init__(self, context: _ExpectedContext):
+        self.__context = context
+        self.__backend_by_unique_key = {}
+        self.__backends = None
+
+    def __initialize_backend(self, config: BackendConfig) -> BackendProtocol:
+        if config.type == SupportedBackend.MongoDB:
+            unique = MongoBackend.create_unique_connection_string(config)
+            if unique in self.__backend_by_unique_key:
+                return self.__backend_by_unique_key[unique]
+            backend = MongoBackend(config)
+            self.__backend_by_unique_key[unique] = backend
+            return backend
+        elif config.type == SupportedBackend.DynamoDB:
+            unique = DynamoDBBackend.create_unique_connection_string(config)
+            if unique in self.__backend_by_unique_key:
+                return self.__backend_by_unique_key[unique]
+            backend = DynamoDBBackend(config)
+            self.__backend_by_unique_key[unique] = backend
+            return backend
+        else:
+            raise ValueError(f"Unsupported backend type: {config.type}")
+
+    def __initialize_backends(self):
+        if self.__backends is not None:
+            return
+        config = self.__context.config.simple_models
+        default_backend_config = config.default
+        model_to_backend_config = getattr(config, "model_to_backend", {}) or {}
+
+        self.__backends = {"default": self.__initialize_backend(default_backend_config)}
+        for model_key, backend_config in model_to_backend_config.items():
+            backend_instance = self.__initialize_backend(backend_config)
+            self.__backends[model_key] = backend_instance
+
+    def __get_backend_for_model(self, meta: ModelDefinition) -> BackendProtocol:
+        self.__initialize_backends()
+        model_key_full = f"{meta.domain}.{meta.plural_name}"
+        if model_key_full in self.__backends:
+            return self.__backends[model_key_full]
+        if meta.domain in self.__backends:
+            return self.__backends[meta.domain]
+        return self.__backends["default"]
+
+    def get_model_backend(self, model_definition: ModelDefinition) -> BackendProtocol:
+        backend = self.__get_backend_for_model(model_definition)
+        if not backend:
+            raise ValueError(
+                f"No backend found for model {model_definition.domain}.{model_definition.plural_name}"
+            )
+        return backend
+
+
+def create(context: _ExpectedContext) -> InLayersDataServices:
+    return InLayersDataServices(context)