mdb-engine 0.1.6__py3-none-any.whl → 0.4.12__py3-none-any.whl

mdb_engine/core/types.py

@@ -8,8 +8,7 @@ throughout the codebase.
 This module is part of MDB_ENGINE - MongoDB Engine.
 """
 
-from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Optional,
-                    TypedDict, Union)
+from typing import TYPE_CHECKING, Any, Literal, TypedDict
 
 if TYPE_CHECKING:
     from ..memory import Mem0MemoryService
@@ -42,20 +41,20 @@ class IndexDefinitionDict(TypedDict, total=False):
         "partial",
         "hybrid",
     ]
-    keys: Union[Dict[str, Union[int, str]], List[List[Union[str, int]]]]
+    keys: dict[str, int | str] | list[list[str | int]]
     unique: bool
     sparse: bool
     background: bool
     expireAfterSeconds: int
-    partialFilterExpression: Dict[str, Any]
-    weights: Dict[str, int]  # For text indexes
+    partialFilterExpression: dict[str, Any]
+    weights: dict[str, int]  # For text indexes
     default_language: str  # For text indexes
     language_override: str  # For text indexes
     textIndexVersion: int  # For text indexes
     # Vector search specific
-    fields: List[Dict[str, Any]]  # For vectorSearch indexes
+    fields: list[dict[str, Any]]  # For vectorSearch indexes
     # Search index specific
-    mappings: Dict[str, Any]  # For search indexes
+    mappings: dict[str, Any]  # For search indexes
     # Geospatial specific
     bucketSize: float  # For geoHaystack
     # TTL specific
@@ -80,12 +79,12 @@ class AuthAuthorizationDict(TypedDict, total=False):
     model: str
     policies_collection: str
     link_users_roles: bool
-    default_roles: List[str]
+    default_roles: list[str]
     # OSO-specific
-    api_key: Optional[str]
-    url: Optional[str]
-    initial_roles: List[Dict[str, str]]
-    initial_policies: List[Dict[str, str]]
+    api_key: str | None
+    url: str | None
+    initial_roles: list[dict[str, str]]
+    initial_policies: list[dict[str, str]]
 
 
 class AuthPolicyDict(TypedDict, total=False):
@@ -94,12 +93,12 @@ class AuthPolicyDict(TypedDict, total=False):
     required: bool
     provider: Literal["casbin", "oso", "custom"]
     authorization: AuthAuthorizationDict
-    allowed_roles: List[str]
-    allowed_users: List[str]
-    denied_users: List[str]
-    required_permissions: List[str]
+    allowed_roles: list[str]
+    allowed_users: list[str]
+    denied_users: list[str]
+    required_permissions: list[str]
     custom_resource: str
-    custom_actions: List[Literal["access", "read", "write", "admin"]]
+    custom_actions: list[Literal["access", "read", "write", "admin"]]
     allow_anonymous: bool
     owner_can_access: bool
 
@@ -112,7 +111,7 @@ class DemoUserDict(TypedDict, total=False):
     role: str
     auto_create: bool
     link_to_platform: bool
-    extra_data: Dict[str, Any]
+    extra_data: dict[str, Any]
 
 
 class UsersConfigDict(TypedDict, total=False):
@@ -127,7 +126,7 @@ class UsersConfigDict(TypedDict, total=False):
     link_platform_users: bool
     anonymous_user_prefix: str
     user_id_field: str
-    demo_users: List[DemoUserDict]
+    demo_users: list[DemoUserDict]
     auto_link_platform_demo: bool
     demo_user_seed_strategy: Literal["auto", "manual", "disabled"]
     enable_demo_user_access: bool
@@ -313,7 +312,7 @@ class MetricsConfigDict(TypedDict, total=False):
     enabled: bool
     collect_operation_metrics: bool
     collect_performance_metrics: bool
-    custom_metrics: List[str]
+    custom_metrics: list[str]
 
 
 class LoggingConfigDict(TypedDict, total=False):
@@ -342,13 +341,11 @@ class CORSConfigDict(TypedDict, total=False):
     """CORS configuration."""
 
     enabled: bool
-    allow_origins: List[str]
+    allow_origins: list[str]
     allow_credentials: bool
-    allow_methods: List[
-        Literal["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD", "*"]
-    ]
-    allow_headers: List[str]
-    expose_headers: List[str]
+    allow_methods: list[Literal["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD", "*"]]
+    allow_headers: list[str]
+    expose_headers: list[str]
     max_age: int
 
 
@@ -374,22 +371,22 @@ class ManifestDict(TypedDict, total=False):
     schema_version: str
     slug: str
     name: str
-    description: Optional[str]
+    description: str | None
     status: Literal["active", "draft", "archived", "inactive"]
     auth_required: bool  # Backward compatibility
-    auth: Optional[AuthConfigDict]
-    token_management: Optional[TokenManagementDict]
-    data_scope: List[str]
-    pip_deps: List[str]
-    managed_indexes: Optional[ManagedIndexesDict]
-    collection_settings: Optional[Dict[str, Dict[str, Any]]]
-    websockets: Optional[WebSocketsDict]
-    embedding_config: Optional[EmbeddingConfigDict]
-    memory_config: Optional[MemoryConfigDict]
-    cors: Optional[CORSConfigDict]
-    observability: Optional[ObservabilityConfigDict]
-    initial_data: Optional[InitialDataDict]
-    developer_id: Optional[str]
+    auth: AuthConfigDict | None
+    token_management: TokenManagementDict | None
+    data_scope: list[str]
+    pip_deps: list[str]
+    managed_indexes: ManagedIndexesDict | None
+    collection_settings: dict[str, dict[str, Any]] | None
+    websockets: WebSocketsDict | None
+    embedding_config: EmbeddingConfigDict | None
+    memory_config: MemoryConfigDict | None
+    cors: CORSConfigDict | None
+    observability: ObservabilityConfigDict | None
+    initial_data: InitialDataDict | None
+    developer_id: str | None
 
 
 # ============================================================================
@@ -401,13 +398,13 @@ class HealthStatusDict(TypedDict, total=False):
     """Health status response."""
 
     status: Literal["healthy", "degraded", "unhealthy"]
-    checks: Dict[str, Dict[str, Any]]
+    checks: dict[str, dict[str, Any]]
     timestamp: str
 
 
 class MetricsDict(TypedDict, total=False):
     """Metrics response."""
 
-    operations: Dict[str, Dict[str, Any]]
-    summary: Dict[str, Any]
+    operations: dict[str, dict[str, Any]]
+    summary: dict[str, Any]
     timestamp: str

mdb_engine/database/README.md

@@ -180,9 +180,9 @@ Async-native interface for managing Atlas Search and Vector indexes.
 ```python
 from mdb_engine.database import AsyncAtlasIndexManager
 
-# Get index manager from collection
+# Get index manager from collection (automatically uses unscoped collection)
 collection = db.my_collection
-index_manager = AsyncAtlasIndexManager(collection._collection)  # Use unscoped collection
+index_manager = collection.index_manager  # Secure way to access index manager
 
 # Create vector search index
 await index_manager.create_vector_search_index(
@@ -277,25 +277,121 @@ await index_manager.create_search_index(
 )
 ```
 
-## Connection Pooling
+## Security Features
+
+The database module includes comprehensive security controls to prevent unauthorized access, NoSQL injection, resource exhaustion, and ensure data isolation:
+
+### Query Security
+
+All queries are automatically validated for security:
+- **Dangerous operator blocking**: Blocks `$where`, `$eval`, `$function`, and `$accumulator` operators that allow JavaScript execution
+- **Query depth limits**: Prevents deeply nested queries that could cause performance issues
+- **Regex complexity limits**: Prevents ReDoS (Regular Expression Denial of Service) attacks
+- **Pipeline validation**: Validates aggregation pipelines for safety and complexity
+
+```python
+# These queries will be blocked:
+db.collection.find({"$where": "this.status === 'active'"})  # ❌ Dangerous operator
+db.collection.aggregate([{"$match": {}}] * 100)  # ❌ Too many pipeline stages
 
-The database module provides shared MongoDB connection pooling for efficient resource usage.
+# These queries are safe:
+db.collection.find({"status": "active"})  # ✅ Safe
+db.collection.find({"age": {"$gt": 18}})  # ✅ Safe
+```
+
+### Resource Limits
 
-### Get Shared Client
+All operations have automatic resource limits to prevent resource exhaustion:
+- **Query timeouts**: All queries automatically have `maxTimeMS` set (default: 30 seconds, max: 5 minutes)
+- **Result size limits**: Maximum 10,000 documents per query (configurable)
+- **Batch size limits**: Maximum 1,000 documents per cursor batch
+- **Document size validation**: Documents are validated before insert (16MB MongoDB limit)
 
 ```python
-from mdb_engine.database import get_shared_mongo_client
+# Timeouts are automatically enforced:
+db.collection.find({"status": "active"})  # Automatically has maxTimeMS=30000
+
+# Result limits are enforced:
+db.collection.find({}, limit=20000)  # Automatically capped to 10,000
+
+# Document sizes are validated:
+large_doc = {"data": "x" * (20 * 1024 * 1024)}  # ❌ Exceeds 16MB limit
+await db.collection.insert_one(large_doc)  # Raises ResourceLimitExceeded
+```
+
+### Collection Name Validation
 
-# Get or create shared MongoDB client
-client = get_shared_mongo_client(
-    mongo_uri="mongodb://localhost:27017",
-    max_pool_size=10,
-    min_pool_size=1
+All collection names are validated for security:
+
+### Collection Name Validation
+
+All collection names are validated for security:
+- **Format validation**: Must match MongoDB naming rules (alphanumeric, underscore, dot, hyphen)
+- **Length limits**: 1-255 characters
+- **Reserved names**: System collections (`apps_config`) are blocked
+- **Reserved prefixes**: Collections starting with `system`, `admin`, `config`, or `local` are blocked
+- **Path traversal protection**: Blocks attempts to use `..`, `/`, or `\` in collection names
+
+```python
+# These will raise ValueError:
+db.system_users  # Reserved prefix
+db.apps_config    # Reserved name
+db["../other"]    # Path traversal attempt
+db["123invalid"] # Invalid format (starts with number)
+```
+
+### Cross-App Access Control
+
+Cross-app collection access is strictly controlled:
+- Apps can only read from collections of apps listed in their `read_scopes`
+- Unauthorized cross-app access attempts are logged and blocked
+- All cross-app access is logged for audit purposes
+
+```python
+# App can only read from authorized apps
+db = engine.get_scoped_db(
+    "my_app",
+    read_scopes=["my_app", "shared_app"]  # Can read from these apps
 )
 
-db = client["my_database"]
+# This works (authorized):
+collection = db.get_collection("shared_app_data")
+
+# This fails (unauthorized):
+collection = db.get_collection("other_app_data")  # Raises ValueError
 ```
 
+### Scope Validation
+
+The `get_scoped_db()` method validates all scopes:
+- `read_scopes` must be a non-empty list of valid app slugs
+- `write_scope` must be a non-empty string
+- Invalid scopes raise `ValueError` with clear error messages
+
+### Audit Logging
+
+All security-relevant events are logged:
+- Invalid collection name attempts
+- Unauthorized cross-app access attempts
+- Reserved name/prefix access attempts
+- Collection name validation failures
+
+Logs include app context (app_slug, collection_name, action) for security monitoring.
+
+### Best Practices
+
+1. **Always use scoped databases**: Never access raw MongoDB clients or databases
+2. **Validate collection names**: Use descriptive, valid collection names
+3. **Limit cross-app access**: Only grant `read_scopes` to apps that need cross-app data
+4. **Monitor audit logs**: Review security logs regularly for suspicious patterns
+5. **Follow naming conventions**: Use lowercase, underscore-separated names (e.g., `user_profiles`)
+
+## Connection Pooling
+
+The database module provides shared MongoDB connection pooling for efficient resource usage. Connection pooling is handled automatically by the engine - users should always use `engine.get_scoped_db()` for database access.
+
+**Security Note:** Direct MongoDB client creation functions are internal and not part of the public API. Always use scoped databases to ensure proper app isolation.
+
 ### Pool Metrics
 
 Monitor connection pool usage:
@@ -474,13 +570,40 @@ except OperationFailure as e:
     print(f"MongoDB operation failed: {e.details}")
 except AutoReconnect as e:
     print(f"MongoDB reconnection: {e}")
-except Exception as e:
-    print(f"Unexpected error: {e}")
+except (ConnectionFailure, ServerSelectionTimeoutError) as e:
+    print(f"Connection error: {e}")
 ```
 
 ## Integration Examples
 
-### FastAPI Integration
+### FastAPI Integration (Recommended)
+
+Use the request-scoped `get_scoped_db` dependency from `mdb_engine.dependencies`:
+
+```python
+from fastapi import Depends
+from mdb_engine import MongoDBEngine
+from mdb_engine.dependencies import get_scoped_db
+
+engine = MongoDBEngine(mongo_uri="...", db_name="...")
+app = engine.create_app(slug="my_app", manifest=Path("manifest.json"))
+
+@app.get("/data")
+async def get_data(db=Depends(get_scoped_db)):
+    # db is automatically scoped to "my_app"
+    docs = await db.my_collection.find({}).to_list(length=10)
+    return {"data": docs}
+
+@app.post("/data")
+async def create_data(db=Depends(get_scoped_db)):
+    # Writes are automatically scoped to "my_app"
+    result = await db.my_collection.insert_one({"name": "New Document"})
+    return {"inserted_id": str(result.inserted_id)}
+```
+
+### Legacy FastAPI Integration
+
+For apps not using `engine.create_app()`:
 
 ```python
 from fastapi import FastAPI, Depends
@@ -508,8 +631,8 @@ db1 = engine.get_scoped_db("app1")
 db2 = engine.get_scoped_db("app2")
 
 # Cross-app read (read from app1, write to app2)
-shared_db = ScopedMongoWrapper(
-    real_db=engine.mongo_db,
+shared_db = engine.get_scoped_db(
+    app_slug="shared",
     read_scopes=["app1", "app2"],
     write_scope="shared"
 )

mdb_engine/database/__init__.py

@@ -6,11 +6,20 @@ and MongoDB-style API for familiarity.
 """
 
 from .abstraction import AppDB, Collection, get_app_db
-from .connection import (close_shared_client, get_pool_metrics,
-                         get_shared_mongo_client, register_client_for_metrics,
-                         verify_shared_client)
-from .scoped_wrapper import (AsyncAtlasIndexManager, AutoIndexManager,
-                             ScopedCollectionWrapper, ScopedMongoWrapper)
+from .connection import (
+    close_shared_client,
+    get_pool_metrics,
+    register_client_for_metrics,
+    verify_shared_client,
+)
+from .query_validator import QueryValidator
+from .resource_limiter import ResourceLimiter
+from .scoped_wrapper import (
+    AsyncAtlasIndexManager,
+    AutoIndexManager,
+    ScopedCollectionWrapper,
+    ScopedMongoWrapper,
+)
 
 __all__ = [
     # Scoped wrappers
@@ -18,12 +27,14 @@ __all__ = [
     "ScopedCollectionWrapper",
     "AsyncAtlasIndexManager",
     "AutoIndexManager",
+    # Query security
+    "QueryValidator",
+    "ResourceLimiter",
     # Database abstraction
     "AppDB",
     "Collection",
     "get_app_db",
     # Connection pooling
-    "get_shared_mongo_client",
     "verify_shared_client",
     "get_pool_metrics",
     "register_client_for_metrics",

mdb_engine/database/abstraction.py

@@ -24,15 +24,20 @@ Usage:
 """
 
 import logging
-from typing import Any, Callable, Dict, List, Optional
+from collections.abc import Callable
+from typing import Any
 
 from ..exceptions import MongoDBEngineError
 from .scoped_wrapper import ScopedMongoWrapper
 
 try:
-    from pymongo.errors import (AutoReconnect, ConnectionFailure,
-                                InvalidOperation, OperationFailure,
-                                ServerSelectionTimeoutError)
+    from pymongo.errors import (
+        AutoReconnect,
+        ConnectionFailure,
+        InvalidOperation,
+        OperationFailure,
+        ServerSelectionTimeoutError,
+    )
 except ImportError:
     OperationFailure = Exception
     AutoReconnect = Exception
@@ -41,8 +46,12 @@ except ImportError:
 
 try:
     from motor.motor_asyncio import AsyncIOMotorCursor
-    from pymongo.results import (DeleteResult, InsertManyResult,
-                                 InsertOneResult, UpdateResult)
+    from pymongo.results import (
+        DeleteResult,
+        InsertManyResult,
+        InsertOneResult,
+        UpdateResult,
+    )
 except ImportError:
     AsyncIOMotorCursor = None
     InsertOneResult = None
@@ -80,8 +89,8 @@ class Collection:
         self._collection = scoped_collection
 
     async def find_one(
-        self, filter: Optional[Dict[str, Any]] = None, *args, **kwargs
-    ) -> Optional[Dict[str, Any]]:
+        self, filter: dict[str, Any] | None = None, *args, **kwargs
+    ) -> dict[str, Any] | None:
         """
         Find a single document matching the filter.
 
@@ -112,9 +121,7 @@ class Collection:
                 context={"operation": "find_one"},
             ) from e
 
-    def find(
-        self, filter: Optional[Dict[str, Any]] = None, *args, **kwargs
-    ) -> AsyncIOMotorCursor:
+    def find(self, filter: dict[str, Any] | None = None, *args, **kwargs) -> AsyncIOMotorCursor:
         """
         Find documents matching the filter.
 
@@ -140,9 +147,7 @@ class Collection:
         """
         return self._collection.find(filter or {}, *args, **kwargs)
 
-    async def insert_one(
-        self, document: Dict[str, Any], *args, **kwargs
-    ) -> InsertOneResult:
+    async def insert_one(self, document: dict[str, Any], *args, **kwargs) -> InsertOneResult:
         """
         Insert a single document.
 
@@ -174,7 +179,7 @@ class Collection:
             ) from e
 
     async def insert_many(
-        self, documents: List[Dict[str, Any]], *args, **kwargs
+        self, documents: list[dict[str, Any]], *args, **kwargs
     ) -> InsertManyResult:
         """
         Insert multiple documents at once.
@@ -209,7 +214,7 @@ class Collection:
             ) from e
 
     async def update_one(
-        self, filter: Dict[str, Any], update: Dict[str, Any], *args, **kwargs
+        self, filter: dict[str, Any], update: dict[str, Any], *args, **kwargs
     ) -> UpdateResult:
         """
         Update a single document matching the filter.
@@ -247,7 +252,7 @@ class Collection:
             ) from e
 
     async def update_many(
-        self, filter: Dict[str, Any], update: Dict[str, Any], *args, **kwargs
+        self, filter: dict[str, Any], update: dict[str, Any], *args, **kwargs
     ) -> UpdateResult:
         """
         Update multiple documents matching the filter.
@@ -284,7 +289,7 @@ class Collection:
             ) from e
 
     async def replace_one(
-        self, filter: Dict[str, Any], replacement: Dict[str, Any], *args, **kwargs
+        self, filter: dict[str, Any], replacement: dict[str, Any], *args, **kwargs
     ) -> UpdateResult:
         """
         Replace a single document matching the filter.
@@ -363,7 +368,7 @@ class Collection:
                 context={"operation": "replace_one"},
             ) from e
 
-    async def delete_one(self, filter: Dict[str, Any], *args, **kwargs) -> DeleteResult:
+    async def delete_one(self, filter: dict[str, Any], *args, **kwargs) -> DeleteResult:
         """
         Delete a single document matching the filter.
 
@@ -396,7 +401,7 @@ class Collection:
             ) from e
 
     async def delete_many(
-        self, filter: Optional[Dict[str, Any]] = None, *args, **kwargs
+        self, filter: dict[str, Any] | None = None, *args, **kwargs
     ) -> DeleteResult:
         """
         Delete multiple documents matching the filter.
@@ -428,9 +433,7 @@ class Collection:
                 context={"operation": "delete_many"},
             ) from e
 
-    async def count_documents(
-        self, filter: Optional[Dict[str, Any]] = None, *args, **kwargs
-    ) -> int:
+    async def count_documents(self, filter: dict[str, Any] | None = None, *args, **kwargs) -> int:
         """
         Count documents matching the filter.
 
@@ -460,9 +463,7 @@ class Collection:
                 context={"operation": "count_documents"},
             ) from e
 
-    def aggregate(
-        self, pipeline: List[Dict[str, Any]], *args, **kwargs
-    ) -> AsyncIOMotorCursor:
+    def aggregate(self, pipeline: list[dict[str, Any]], *args, **kwargs) -> AsyncIOMotorCursor:
         """
         Perform aggregation pipeline.
 
@@ -518,7 +519,7 @@ class AppDB:
             raise RuntimeError("ScopedMongoWrapper is not available. Check imports.")
 
         self._wrapper = scoped_wrapper
-        self._collection_cache: Dict[str, Collection] = {}
+        self._collection_cache: dict[str, Collection] = {}
 
     def collection(self, name: str) -> Collection:
         """
@@ -554,11 +555,17 @@ class AppDB:
         Example:
             db.users.get("user_123")  # Instead of db.collection("users").get("user_123")
         """
-        # Only proxy collection names, not internal attributes
-        if name.startswith("_"):
+        # Explicitly block access to 'database' property (removed for security)
+        if name == "database":
             raise AttributeError(
-                f"'{type(self).__name__}' object has no attribute '{name}'"
+                "'database' property has been removed for security. "
+                "Use collection.index_manager for index operations. "
+                "All data access must go through scoped collections."
             )
+
+        # Only proxy collection names, not internal attributes
+        if name.startswith("_"):
+            raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
         return self.collection(name)
 
     @property
@@ -576,26 +583,6 @@ class AppDB:
         """
         return self._wrapper
 
-    @property
-    def database(self):
-        """
-        Access the underlying AsyncIOMotorDatabase (unscoped).
-
-        This is useful for advanced operations that need direct access to the
-        real database without scoping, such as index management or administrative
-        operations.
-
-        Returns:
-            The underlying AsyncIOMotorDatabase instance
-
-        Example:
-            # Access underlying database for index management
-            real_db = db.database
-            collection = real_db["my_collection"]
-            index_manager = AsyncAtlasIndexManager(collection)
-        """
-        return self._wrapper.database
-
 
 # FastAPI dependency helper
 async def get_app_db(request, get_scoped_db_func: Callable) -> AppDB: