mdb-engine 0.1.6__py3-none-any.whl → 0.4.12__py3-none-any.whl

mdb_engine/config.py

@@ -7,7 +7,6 @@ with direct parameters as before.
 """
 
 import os
-from typing import Optional
 
 try:
     from pydantic import BaseSettings, Field
@@ -46,12 +45,12 @@ class EngineConfig:
 
     def __init__(
         self,
-        mongo_uri: Optional[str] = None,
-        db_name: Optional[str] = None,
-        max_pool_size: Optional[int] = None,
-        min_pool_size: Optional[int] = None,
-        server_selection_timeout_ms: Optional[int] = None,
-        authz_cache_ttl: Optional[int] = None,
+        mongo_uri: str | None = None,
+        db_name: str | None = None,
+        max_pool_size: int | None = None,
+        min_pool_size: int | None = None,
+        server_selection_timeout_ms: int | None = None,
+        authz_cache_ttl: int | None = None,
     ):
         """
         Initialize configuration.
@@ -66,18 +65,12 @@ class EngineConfig:
         """
         self.mongo_uri = mongo_uri or os.getenv("MONGO_URI", "")
         self.db_name = db_name or os.getenv("DB_NAME", "")
-        self.max_pool_size = max_pool_size or int(
-            os.getenv("MONGO_MAX_POOL_SIZE", "50")
-        )
-        self.min_pool_size = min_pool_size or int(
-            os.getenv("MONGO_MIN_POOL_SIZE", "10")
-        )
+        self.max_pool_size = max_pool_size or int(os.getenv("MONGO_MAX_POOL_SIZE", "50"))
+        self.min_pool_size = min_pool_size or int(os.getenv("MONGO_MIN_POOL_SIZE", "10"))
         self.server_selection_timeout_ms = server_selection_timeout_ms or int(
             os.getenv("MONGO_SERVER_SELECTION_TIMEOUT_MS", "5000")
         )
-        self.authz_cache_ttl = authz_cache_ttl or int(
-            os.getenv("AUTHZ_CACHE_TTL", "300")
-        )
+        self.authz_cache_ttl = authz_cache_ttl or int(os.getenv("AUTHZ_CACHE_TTL", "300"))
 
     def validate(self) -> None:
         """
@@ -115,9 +108,7 @@ class EngineConfig:
             )
 
         if self.authz_cache_ttl < 0:
-            raise ValueError(
-                f"authz_cache_ttl must be >= 0, got {self.authz_cache_ttl}"
-            )
+            raise ValueError(f"authz_cache_ttl must be >= 0, got {self.authz_cache_ttl}")
 
 
 # Pydantic-based configuration (optional, only if Pydantic is available)
@@ -138,9 +129,7 @@ if PYDANTIC_AVAILABLE:
             )
         """
 
-        mongo_uri: str = Field(
-            ..., env="MONGO_URI", description="MongoDB connection URI"
-        )
+        mongo_uri: str = Field(..., env="MONGO_URI", description="MongoDB connection URI")
         db_name: str = Field(..., env="DB_NAME", description="Database name")
         max_pool_size: int = Field(
             50,
@@ -203,15 +192,11 @@ ACCESS_TOKEN_TTL: int = int(os.getenv("ACCESS_TOKEN_TTL", "900"))  # 15 minutes
 REFRESH_TOKEN_TTL: int = int(os.getenv("REFRESH_TOKEN_TTL", "604800"))  # 7 days
 """Refresh token TTL in seconds (default: 604800 / 7 days)."""
 
-TOKEN_ROTATION_ENABLED: bool = (
-    os.getenv("TOKEN_ROTATION_ENABLED", "true").lower() == "true"
-)
+TOKEN_ROTATION_ENABLED: bool = os.getenv("TOKEN_ROTATION_ENABLED", "true").lower() == "true"
 """Whether to rotate refresh tokens on each use (default: true)."""
 
 MAX_SESSIONS_PER_USER: int = int(os.getenv("MAX_SESSIONS_PER_USER", "10"))
 """Maximum number of concurrent sessions per user (default: 10)."""
 
-SESSION_INACTIVITY_TIMEOUT: int = int(
-    os.getenv("SESSION_INACTIVITY_TIMEOUT", "1800")
-)  # 30 minutes
+SESSION_INACTIVITY_TIMEOUT: int = int(os.getenv("SESSION_INACTIVITY_TIMEOUT", "1800"))  # 30 minutes
 """Session inactivity timeout in seconds (default: 1800 / 30 minutes)."""

mdb_engine/constants.py

@@ -57,6 +57,25 @@ DEFAULT_MAX_IDLE_TIME_MS: Final[int] = 45000
 MAX_COLLECTION_NAME_LENGTH: Final[int] = 255
 """Maximum length for MongoDB collection names."""
 
+MIN_COLLECTION_NAME_LENGTH: Final[int] = 1
+"""Minimum length for MongoDB collection names."""
+
+# Reserved collection name prefixes (MongoDB system collections)
+RESERVED_COLLECTION_PREFIXES: Final[tuple[str, ...]] = (
+    "system",
+    "admin",
+    "config",
+    "local",
+)
+"""Reserved MongoDB collection name prefixes that cannot be used."""
+
+# Reserved collection names (engine-internal)
+RESERVED_COLLECTION_NAMES: Final[tuple[str, ...]] = (
+    "apps_config",  # Engine internal - app registration
+    "_mdb_engine_app_secrets",  # Engine internal - encrypted app secrets
+)
+"""Reserved collection names that cannot be accessed through scoped wrappers."""
+
 # App slug constraints
 MAX_APP_SLUG_LENGTH: Final[int] = 100
 """Maximum length for app slugs."""
@@ -158,3 +177,49 @@ SUPPORTED_APP_STATUSES: Final[tuple[str, ...]] = (
     APP_STATUS_ARCHIVED,
     APP_STATUS_INACTIVE,
 )
+
+# ============================================================================
+# QUERY SECURITY & RESOURCE LIMITS
+# ============================================================================
+
+# Query execution limits
+DEFAULT_MAX_TIME_MS: Final[int] = 30000
+"""Default query timeout in milliseconds (30 seconds)."""
+
+MAX_QUERY_TIME_MS: Final[int] = 300000
+"""Maximum allowed query timeout in milliseconds (5 minutes)."""
+
+MAX_QUERY_RESULT_SIZE: Final[int] = 10000
+"""Maximum number of documents that can be returned in a single query."""
+
+MAX_CURSOR_BATCH_SIZE: Final[int] = 1000
+"""Maximum batch size for cursor operations."""
+
+MAX_DOCUMENT_SIZE: Final[int] = 16 * 1024 * 1024
+"""Maximum document size in bytes (16MB MongoDB limit)."""
+
+# Pipeline limits
+MAX_PIPELINE_STAGES: Final[int] = 50
+"""Maximum number of stages allowed in an aggregation pipeline."""
+
+MAX_SORT_FIELDS: Final[int] = 10
+"""Maximum number of fields that can be sorted in a single query."""
+
+MAX_QUERY_DEPTH: Final[int] = 10
+"""Maximum nesting depth for query filters (prevents deeply nested queries)."""
+
+# Regex limits (prevent ReDoS attacks)
+MAX_REGEX_LENGTH: Final[int] = 1000
+"""Maximum length of regex patterns to prevent ReDoS attacks."""
+
+MAX_REGEX_COMPLEXITY: Final[int] = 50
+"""Maximum complexity score for regex patterns (prevents ReDoS)."""
+
+# Dangerous MongoDB operators that should be blocked
+DANGEROUS_OPERATORS: Final[tuple[str, ...]] = (
+    "$where",  # JavaScript execution (security risk)
+    "$eval",  # JavaScript evaluation (deprecated, security risk)
+    "$function",  # JavaScript functions (security risk)
+    "$accumulator",  # Can be abused for code execution
+)
+"""MongoDB operators that are blocked for security reasons."""

mdb_engine/core/README.md

@@ -2,6 +2,26 @@
 
 The central orchestration engine for MDB_ENGINE that manages database connections, app registration, manifest validation, index management, and resource lifecycle.
 
+## When to Use What
+
+### Integration Patterns
+
+| Pattern | Best For | Code |
+|---------|----------|------|
+| **`create_app()`** | New FastAPI apps | `app = engine.create_app(slug, manifest)` |
+| **`lifespan()`** | Custom FastAPI setup | `FastAPI(lifespan=engine.lifespan(...))` |
+| **Manual** | Existing apps, scripts | `await engine.initialize()` / `await engine.shutdown()` |
+| **Context Manager** | Scripts, tests | `async with engine: ...` |
+
+### Feature Activation
+
+| Feature | When to Enable | Configuration |
+|---------|---------------|---------------|
+| **Ray** | Distributed processing, isolated actors | `enable_ray=True` |
+| **Multi-site** | Cross-app data sharing | `read_scopes` in manifest |
+| **Auto-indexing** | Let engine optimize queries | Default ON (`auto_index=True`) |
+| **App Tokens** | Production security | Set `MDB_ENGINE_MASTER_KEY` |
+
 ## Features
 
 - **MongoDBEngine**: Central orchestration for all engine components
@@ -54,12 +74,57 @@ engine = MongoDBEngine(
     manifests_dir=Path("./manifests"),  # Optional
     authz_provider=authz_provider,      # Optional
     max_pool_size=10,                   # Optional
-    min_pool_size=1                     # Optional
+    min_pool_size=1,                    # Optional
+    enable_ray=False,                   # Optional: Enable Ray support
+    ray_namespace="modular_labs"        # Optional: Ray namespace
 )
 
 await engine.initialize()
 ```
 
+### FastAPI Integration
+
+The simplest way to create a FastAPI app with automatic lifecycle management:
+
+```python
+from mdb_engine import MongoDBEngine
+from pathlib import Path
+
+engine = MongoDBEngine(
+    mongo_uri="mongodb://localhost:27017",
+    db_name="my_database"
+)
+
+# Create FastAPI app with automatic initialization and cleanup
+app = engine.create_app(
+    slug="my_app",
+    manifest=Path("manifest.json")
+)
+
+@app.get("/items")
+async def get_items():
+    db = engine.get_scoped_db("my_app")
+    return await db.items.find({}).to_list(10)
+```
+
+This automatically:
+- Initializes the engine on startup
+- Loads and registers the manifest
+- Auto-detects multi-site mode from manifest
+- Auto-retrieves app tokens
+- Shuts down cleanly on app exit
+
+### Custom Lifespan
+
+For more control, use the `lifespan()` helper:
+
+```python
+from fastapi import FastAPI
+
+engine = MongoDBEngine(...)
+app = FastAPI(lifespan=engine.lifespan("my_app", Path("manifest.json")))
+```
+
 ### Get Scoped Database
 
 Get a database wrapper that automatically isolates data by app:
@@ -91,6 +156,33 @@ manifest = await engine.load_manifest("manifest.json")
 await engine.register_app(manifest)
 ```
 
+### Optional Ray Integration
+
+Enable Ray for distributed processing (Ray must be installed):
+
+```python
+from mdb_engine import MongoDBEngine
+
+# Enable Ray support
+engine = MongoDBEngine(
+    mongo_uri="mongodb://localhost:27017",
+    db_name="my_database",
+    enable_ray=True,
+    ray_namespace="my_namespace"
+)
+
+await engine.initialize()
+
+# Check if Ray is available
+if engine.has_ray:
+    print(f"Ray initialized in namespace: {engine.ray_namespace}")
+```
+
+Ray integration features:
+- Isolated app environments via Ray actors
+- Automatic graceful degradation if Ray not installed
+- App-specific namespaces for isolation
+
 ### Health Checks
 
 Check engine and MongoDB health:
@@ -98,10 +190,13 @@ Check engine and MongoDB health:
 ```python
 from mdb_engine.observability import check_engine_health, check_mongodb_health
 
-# Check MongoDB connection
+# Check MongoDB connection (mongo_client is for observability only, not data access)
 mongodb_status = await check_mongodb_health(engine.mongo_client)
 print(mongodb_status)
 
+# ⚠️ SECURITY: engine.mongo_client is for observability/admin operations only
+# Always use engine.get_scoped_db() for all data access operations
+
 # Check engine health
 engine_status = await check_engine_health(engine)
 print(engine_status)
@@ -299,11 +394,23 @@ is_valid, error, paths = await validate_manifest_with_db(
 - `get_app_info(app_slug)` - Get information about registered app
 - `unregister_app(app_slug)` - Unregister an app
 
+**FastAPI Integration Methods:**
+
+- `create_app(slug, manifest, title=None, **fastapi_kwargs)` - Create a FastAPI app with automatic lifecycle management
+- `lifespan(slug, manifest)` - Create a lifespan context manager for FastAPI
+
+**App Token Methods:**
+
+- `auto_retrieve_app_token(slug)` - Auto-retrieve app token from environment or database
+- `get_app_token(slug)` - Get cached app token for a slug
+
 #### Properties
 
-- `mongo_client` - MongoDB client instance
-- `mongo_db` - MongoDB database instance
+- `mongo_client` - MongoDB client instance (for observability only)
 - `initialized` - Whether engine is initialized
+- `has_ray` - Whether Ray is enabled and initialized
+- `enable_ray` - Whether Ray support is enabled
+- `ray_namespace` - Ray namespace for actor isolation
 
 ### ManifestValidator
 
@@ -354,6 +461,8 @@ export MONGO_SERVER_SELECTION_TIMEOUT_MS=5000
 - `authz_provider` (optional): Authorization provider instance
 - `max_pool_size` (optional): Maximum connection pool size (default: 10)
 - `min_pool_size` (optional): Minimum connection pool size (default: 1)
+- `enable_ray` (optional): Enable Ray support for distributed processing (default: False)
+- `ray_namespace` (optional): Ray namespace for actor isolation (default: "modular_labs")
 
 ## Manifest Schema Features
 
@@ -480,13 +589,15 @@ Define various index types:
 ```python
 from mdb_engine.exceptions import InitializationError
 
+from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError
+
 try:
     await engine.initialize()
 except InitializationError as e:
     print(f"Initialization failed: {e}")
     print(f"MongoDB URI: {e.mongo_uri}")
-except Exception as e:
-    print(f"Unexpected error: {e}")
+except (ConnectionFailure, ServerSelectionTimeoutError) as e:
+    print(f"MongoDB connection error: {e}")
 ```
 
 ## Integration Examples

mdb_engine/core/__init__.py

@@ -3,20 +3,52 @@ Core MongoDB Engine components.
 
 This module contains the main MongoDBEngine class and core
 orchestration logic for managing apps.
+
+The MongoDBEngine now includes:
+- FastAPI integration with create_app() method
+- Optional Ray support with enable_ray parameter
+- Automatic app token retrieval
+- Multi-site mode auto-detection from manifest
 """
 
 from .engine import MongoDBEngine
 from .manifest import (  # Classes; Constants; Functions (for backward compatibility); Schemas
-    CURRENT_SCHEMA_VERSION, DEFAULT_SCHEMA_VERSION, MANIFEST_SCHEMA,
-    MANIFEST_SCHEMA_V1, MANIFEST_SCHEMA_V2, SCHEMA_REGISTRY, ManifestParser,
-    ManifestValidator, clear_validation_cache, get_schema_for_version,
-    get_schema_version, migrate_manifest, validate_developer_id,
-    validate_index_definition, validate_managed_indexes, validate_manifest,
-    validate_manifest_with_db, validate_manifests_parallel)
+    CURRENT_SCHEMA_VERSION,
+    DEFAULT_SCHEMA_VERSION,
+    MANIFEST_SCHEMA,
+    MANIFEST_SCHEMA_V1,
+    MANIFEST_SCHEMA_V2,
+    SCHEMA_REGISTRY,
+    ManifestParser,
+    ManifestValidator,
+    clear_validation_cache,
+    get_schema_for_version,
+    get_schema_version,
+    migrate_manifest,
+    validate_developer_id,
+    validate_index_definition,
+    validate_managed_indexes,
+    validate_manifest,
+    validate_manifest_with_db,
+    validate_manifests_parallel,
+)
+
+# Optional Ray integration (gracefully handles missing Ray)
+from .ray_integration import (
+    RAY_AVAILABLE,
+    AppRayActor,
+    get_ray_actor_handle,
+    ray_actor_decorator,
+)
 
 __all__ = [
-    # MongoDB Engine
+    # MongoDB Engine (includes FastAPI integration and optional Ray)
     "MongoDBEngine",
+    # Ray Integration (optional - only active if Ray installed)
+    "RAY_AVAILABLE",
+    "AppRayActor",
+    "get_ray_actor_handle",
+    "ray_actor_decorator",
     # Classes
     "ManifestValidator",
     "ManifestParser",

mdb_engine/core/app_registration.py

@@ -10,16 +10,22 @@ This module is part of MDB_ENGINE - MongoDB Engine.
 import asyncio
 import logging
 import time
+from collections.abc import Callable
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Optional
 
+from jsonschema import SchemaError
+from jsonschema import ValidationError as JsonSchemaValidationError
 from motor.motor_asyncio import AsyncIOMotorDatabase
-from pymongo.errors import (ConnectionFailure, InvalidOperation,
-                            OperationFailure, ServerSelectionTimeoutError)
-
-from ..observability import clear_app_context
+from pymongo.errors import (
+    ConnectionFailure,
+    InvalidOperation,
+    OperationFailure,
+    ServerSelectionTimeoutError,
+)
+
+from ..observability import clear_app_context, record_operation, set_app_context
 from ..observability import get_logger as get_contextual_logger
-from ..observability import record_operation, set_app_context
 from .manifest import ManifestParser, ManifestValidator
 
 if TYPE_CHECKING:
@@ -53,11 +59,11 @@ class AppRegistrationManager:
         self._mongo_db = mongo_db
         self.manifest_validator = manifest_validator
         self.manifest_parser = manifest_parser
-        self._apps: Dict[str, Dict[str, Any]] = {}
+        self._apps: dict[str, dict[str, Any]] = {}
 
     async def validate_manifest(
         self, manifest: "ManifestDict"
-    ) -> Tuple[bool, Optional[str], Optional[List[str]]]:
+    ) -> tuple[bool, str | None, list[str] | None]:
         """
         Validate a manifest against the schema.
 
@@ -81,7 +87,7 @@ class AppRegistrationManager:
                 experiment_slug=slug,
             )
             return result
-        except Exception:
+        except (JsonSchemaValidationError, SchemaError, ValueError, TypeError, KeyError):
             duration_ms = (time.time() - start_time) * 1000
             record_operation(
                 "app_registration.validate_manifest",
@@ -110,19 +116,12 @@ class AppRegistrationManager:
     async def register_app(
         self,
         manifest: "ManifestDict",
-        create_indexes_callback: Optional[Callable[[str, "ManifestDict"], Any]] = None,
-        seed_data_callback: Optional[
-            Callable[[str, Dict[str, List[Dict[str, Any]]]], Any]
-        ] = None,
-        initialize_memory_callback: Optional[
-            Callable[[str, Dict[str, Any]], Any]
-        ] = None,
-        register_websockets_callback: Optional[
-            Callable[[str, Dict[str, Any]], Any]
-        ] = None,
-        setup_observability_callback: Optional[
-            Callable[[str, "ManifestDict", Dict[str, Any]], Any]
-        ] = None,
+        create_indexes_callback: Callable[[str, "ManifestDict"], Any] | None = None,
+        seed_data_callback: Callable[[str, dict[str, list[dict[str, Any]]]], Any] | None = None,
+        initialize_memory_callback: Callable[[str, dict[str, Any]], Any] | None = None,
+        register_websockets_callback: Callable[[str, dict[str, Any]], Any] | None = None,
+        setup_observability_callback: Callable[[str, "ManifestDict", dict[str, Any]], Any]
+        | None = None,
     ) -> bool:
         """
         Register an app from its manifest.
@@ -143,7 +142,7 @@ class AppRegistrationManager:
         """
         start_time = time.time()
 
-        slug: Optional[str] = manifest.get("slug")
+        slug: str | None = manifest.get("slug")
         if not slug:
             contextual_logger.error(
                 "Cannot register app: missing 'slug' in manifest",
@@ -210,9 +209,7 @@ class AppRegistrationManager:
                 )
                 # Continue even if persistence fails - app is still registered in memory
             except InvalidOperation as e:
-                logger.debug(
-                    f"Cannot persist app '{slug}': MongoDB client is closed: {e}"
-                )
+                logger.debug(f"Cannot persist app '{slug}': MongoDB client is closed: {e}")
                 # Continue - app is still registered in memory
 
             # Invalidate auth config cache for this app
@@ -228,32 +225,22 @@ class AppRegistrationManager:
 
             # Create indexes if requested
             if create_indexes_callback and "managed_indexes" in manifest:
-                logger.info(
-                    f"[{slug}] Creating managed indexes " f"(has_managed_indexes=True)"
-                )
+                logger.info(f"[{slug}] Creating managed indexes " f"(has_managed_indexes=True)")
                 callback_tasks.append(create_indexes_callback(slug, manifest))
 
             # Seed initial data if configured
             if seed_data_callback and "initial_data" in manifest:
-                callback_tasks.append(
-                    seed_data_callback(slug, manifest["initial_data"])
-                )
+                callback_tasks.append(seed_data_callback(slug, manifest["initial_data"]))
 
             # Initialize Memory service if configured
             memory_config = manifest.get("memory_config")
-            if (
-                initialize_memory_callback
-                and memory_config
-                and memory_config.get("enabled", False)
-            ):
+            if initialize_memory_callback and memory_config and memory_config.get("enabled", False):
                 callback_tasks.append(initialize_memory_callback(slug, memory_config))
 
             # Register WebSocket endpoints if configured
             websockets_config = manifest.get("websockets")
             if register_websockets_callback and websockets_config:
-                callback_tasks.append(
-                    register_websockets_callback(slug, websockets_config)
-                )
+                callback_tasks.append(register_websockets_callback(slug, websockets_config))
 
             # Set up observability (health checks, metrics, logging)
             observability_config = manifest.get("observability", {})
@@ -275,9 +262,7 @@ class AppRegistrationManager:
                             "register_websockets",
                             "setup_observability",
                         ]
-                        callback_name = (
-                            callback_names[i] if i < len(callback_names) else "unknown"
-                        )
+                        callback_name = callback_names[i] if i < len(callback_names) else "unknown"
                         logger.warning(
                             f"[{slug}] Callback '{callback_name}' failed during "
                             f"app registration: {result}",
@@ -295,9 +280,7 @@ class AppRegistrationManager:
                 "App registered successfully",
                 extra={
                     "app_slug": slug,
-                    "memory_enabled": bool(
-                        memory_config and memory_config.get("enabled", False)
-                    ),
+                    "memory_enabled": bool(memory_config and memory_config.get("enabled", False)),
                     "websockets_configured": bool(websockets_config),
                     "duration_ms": round(duration_ms, 2),
                 },
@@ -324,9 +307,7 @@ class AppRegistrationManager:
         try:
             # Fetch active apps
             active_cfgs = (
-                await self._mongo_db.apps_config.find({"status": "active"})
-                .limit(500)
-                .to_list(None)
+                await self._mongo_db.apps_config.find({"status": "active"}).limit(500).to_list(None)
             )
 
             logger.info(f"Found {len(active_cfgs)} active app(s).")
@@ -378,7 +359,7 @@ class AppRegistrationManager:
         """
         return self._apps.get(slug)
 
-    def list_apps(self) -> List[str]:
+    def list_apps(self) -> list[str]:
         """
         List all registered app slugs.