mdb-engine 0.1.6__py3-none-any.whl

mdb_engine/constants.py

@@ -0,0 +1,160 @@
+"""
+Constants for MDB_ENGINE.
+
+This module contains all shared constants used across the codebase to avoid
+magic numbers and improve maintainability.
+"""
+
+from typing import Final
+
+# ============================================================================
+# INDEX MANAGEMENT CONSTANTS
+# ============================================================================
+
+# Polling intervals (in seconds)
+DEFAULT_POLL_INTERVAL: Final[int] = 5
+"""Default polling interval for checking index status (seconds)."""
+
+# Timeouts (in seconds)
+DEFAULT_SEARCH_TIMEOUT: Final[int] = 600  # 10 minutes
+"""Default timeout for waiting for search indexes to become ready (seconds)."""
+
+DEFAULT_DROP_TIMEOUT: Final[int] = 300  # 5 minutes
+"""Default timeout for waiting for search indexes to be dropped (seconds)."""
+
+# Auto-indexing constants
+AUTO_INDEX_HINT_THRESHOLD: Final[int] = 3
+"""Number of times a query pattern must be seen before creating an index."""
+
+MAX_INDEX_FIELDS: Final[int] = 4
+"""Maximum number of fields in a compound index (MongoDB best practice)."""
+
+MAX_CACHE_SIZE: Final[int] = 1000
+"""Maximum size of authorization and validation caches before eviction."""
+
+# ============================================================================
+# DATABASE CONSTANTS
+# ============================================================================
+
+# Connection pool defaults
+DEFAULT_MAX_POOL_SIZE: Final[int] = 50
+"""Default maximum MongoDB connection pool size."""
+
+DEFAULT_MIN_POOL_SIZE: Final[int] = 10
+"""Default minimum MongoDB connection pool size."""
+
+DEFAULT_SERVER_SELECTION_TIMEOUT_MS: Final[int] = 5000
+"""Default server selection timeout in milliseconds."""
+
+DEFAULT_MAX_IDLE_TIME_MS: Final[int] = 45000
+"""Default maximum idle time before closing connections (milliseconds)."""
+
+# ============================================================================
+# VALIDATION CONSTANTS
+# ============================================================================
+
+# Collection name constraints
+MAX_COLLECTION_NAME_LENGTH: Final[int] = 255
+"""Maximum length for MongoDB collection names."""
+
+# App slug constraints
+MAX_APP_SLUG_LENGTH: Final[int] = 100
+"""Maximum length for app slugs."""
+
+# TTL index constraints
+MIN_TTL_SECONDS: Final[int] = 1
+"""Minimum TTL value in seconds."""
+
+MAX_TTL_SECONDS: Final[int] = 31536000  # 1 year
+"""Maximum recommended TTL value in seconds (1 year)."""
+
+# Vector index constraints
+MIN_VECTOR_DIMENSIONS: Final[int] = 1
+"""Minimum number of dimensions for vector indexes."""
+
+MAX_VECTOR_DIMENSIONS: Final[int] = 10000
+"""Maximum number of dimensions for vector indexes."""
+
+# ============================================================================
+# CACHING CONSTANTS
+# ============================================================================
+
+AUTHZ_CACHE_TTL: Final[int] = 300  # 5 minutes
+"""Authorization cache TTL in seconds."""
+
+# ============================================================================
+# TOKEN MANAGEMENT CONSTANTS
+# ============================================================================
+
+# Token lifetimes (in seconds)
+ACCESS_TOKEN_TTL: Final[int] = 900  # 15 minutes
+"""Default access token TTL in seconds."""
+
+REFRESH_TOKEN_TTL: Final[int] = 604800  # 7 days
+"""Default refresh token TTL in seconds."""
+
+# Token management settings
+TOKEN_ROTATION_ENABLED: Final[bool] = True
+"""Whether to rotate refresh tokens on each use."""
+
+MAX_SESSIONS_PER_USER: Final[int] = 10
+"""Maximum number of concurrent sessions per user."""
+
+SESSION_INACTIVITY_TIMEOUT: Final[int] = 1800  # 30 minutes
+"""Session inactivity timeout in seconds before automatic cleanup."""
+
+# Token versioning
+CURRENT_TOKEN_VERSION: Final[str] = "1.0"
+"""Current token schema version for migration support."""
+
+# ============================================================================
+# SCHEMA CONSTANTS
+# ============================================================================
+
+CURRENT_SCHEMA_VERSION: Final[str] = "2.0"
+"""Current manifest schema version."""
+
+DEFAULT_SCHEMA_VERSION: Final[str] = "1.0"
+"""Default schema version for manifests without version field."""
+
+# ============================================================================
+# INDEX TYPE CONSTANTS
+# ============================================================================
+
+INDEX_TYPE_REGULAR: Final[str] = "regular"
+INDEX_TYPE_VECTOR_SEARCH: Final[str] = "vectorSearch"
+INDEX_TYPE_SEARCH: Final[str] = "search"
+INDEX_TYPE_TEXT: Final[str] = "text"
+INDEX_TYPE_GEOSPATIAL: Final[str] = "geospatial"
+INDEX_TYPE_TTL: Final[str] = "ttl"
+INDEX_TYPE_PARTIAL: Final[str] = "partial"
+INDEX_TYPE_HYBRID: Final[str] = "hybrid"
+
+# Supported index types
+SUPPORTED_INDEX_TYPES: Final[tuple[str, ...]] = (
+    INDEX_TYPE_REGULAR,
+    INDEX_TYPE_VECTOR_SEARCH,
+    INDEX_TYPE_SEARCH,
+    INDEX_TYPE_TEXT,
+    INDEX_TYPE_GEOSPATIAL,
+    INDEX_TYPE_TTL,
+    INDEX_TYPE_PARTIAL,
+    INDEX_TYPE_HYBRID,
+)
+
+# ============================================================================
+# APP STATUS CONSTANTS
+# ============================================================================
+
+APP_STATUS_ACTIVE: Final[str] = "active"
+APP_STATUS_DRAFT: Final[str] = "draft"
+APP_STATUS_ARCHIVED: Final[str] = "archived"
+APP_STATUS_INACTIVE: Final[str] = "inactive"
+
+# Supported app statuses
+SUPPORTED_APP_STATUSES: Final[tuple[str, ...]] = (
+    APP_STATUS_ACTIVE,
+    APP_STATUS_DRAFT,
+    APP_STATUS_ARCHIVED,
+    APP_STATUS_INACTIVE,
+)

mdb_engine/core/README.md

@@ -0,0 +1,542 @@
+# Core MongoDB Engine Module
+
+The central orchestration engine for MDB_ENGINE that manages database connections, app registration, manifest validation, index management, and resource lifecycle.
+
+## Features
+
+- **MongoDBEngine**: Central orchestration for all engine components
+- **Manifest System**: JSON schema validation with versioning (v1.0, v2.0)
+- **App Registration**: Automatic app setup with isolation and indexing
+- **Health Checks**: Built-in health monitoring and status reporting
+- **Connection Management**: Efficient MongoDB connection pooling
+- **Schema Migration**: Automatic migration between manifest schema versions
+
+## Installation
+
+The core module is part of MDB_ENGINE. No additional installation required.
+
+## Quick Start
+
+### Basic Usage
+
+```python
+from mdb_engine import MongoDBEngine
+
+# Initialize engine
+engine = MongoDBEngine(
+    mongo_uri="mongodb://localhost:27017",
+    db_name="my_database"
+)
+
+# Initialize connection
+await engine.initialize()
+
+# Load and register app from manifest
+manifest = await engine.load_manifest("manifest.json")
+await engine.register_app(manifest)
+
+# Get scoped database for app
+db = engine.get_scoped_db("my_app")
+```
+
+## MongoDBEngine
+
+The `MongoDBEngine` class is the central component that orchestrates all engine functionality.
+
+### Initialization
+
+```python
+from mdb_engine import MongoDBEngine
+
+engine = MongoDBEngine(
+    mongo_uri="mongodb://localhost:27017",
+    db_name="my_database",
+    manifests_dir=Path("./manifests"),  # Optional
+    authz_provider=authz_provider,      # Optional
+    max_pool_size=10,                   # Optional
+    min_pool_size=1                     # Optional
+)
+
+await engine.initialize()
+```
+
+### Get Scoped Database
+
+Get a database wrapper that automatically isolates data by app:
+
+```python
+# Basic usage - all operations scoped to "my_app"
+db = engine.get_scoped_db("my_app")
+
+# Read from multiple apps, write to one
+db = engine.get_scoped_db(
+    app_slug="my_app",
+    read_scopes=["my_app", "shared_app"],  # Can read from multiple apps
+    write_scope="my_app"                    # Write to single app
+)
+
+# Disable automatic indexing
+db = engine.get_scoped_db("my_app", auto_index=False)
+```
+
+### App Registration
+
+Register an app with the MongoDB Engine:
+
+```python
+# Load manifest from file
+manifest = await engine.load_manifest("manifest.json")
+
+# Register app (creates indexes, sets up auth, etc.)
+await engine.register_app(manifest)
+```
+
+### Health Checks
+
+Check engine and MongoDB health:
+
+```python
+from mdb_engine.observability import check_engine_health, check_mongodb_health
+
+# Check MongoDB connection
+mongodb_status = await check_mongodb_health(engine.mongo_client)
+print(mongodb_status)
+
+# Check engine health
+engine_status = await check_engine_health(engine)
+print(engine_status)
+```
+
+### Get Registered Apps
+
+```python
+# Get all registered apps
+apps = engine.get_registered_apps()
+for slug, app_info in apps.items():
+    print(f"App: {slug}, Status: {app_info['status']}")
+```
+
+## Manifest System
+
+The manifest system provides JSON schema validation, versioning, and migration.
+
+### Manifest Schema Versions
+
+- **v1.0**: Initial schema (default for manifests without version field)
+- **v2.0**: Current schema with all features (auth.policy, auth.users, managed_indexes, etc.)
+
+### Manifest Structure
+
+A basic manifest.json:
+
+```json
+{
+  "schema_version": "2.0",
+  "slug": "my_app",
+  "name": "My Application",
+  "description": "Application description",
+  "status": "active",
+  "auth_required": true,
+  "collections": {
+    "users": {
+      "indexes": [
+        {
+          "name": "email_idx",
+          "type": "regular",
+          "keys": {"email": 1},
+          "unique": true
+        }
+      ]
+    }
+  }
+}
+```
+
+### Manifest Validation
+
+```python
+from mdb_engine.core import ManifestValidator, validate_manifest
+
+# Using MongoDBEngine
+is_valid, error, paths = await engine.validate_manifest(manifest)
+
+# Using validator directly
+validator = ManifestValidator()
+is_valid, error, paths = validator.validate(manifest)
+
+# Using convenience function
+is_valid, error, paths = validate_manifest(manifest)
+```
+
+### Manifest Parsing
+
+```python
+from mdb_engine.core import ManifestParser
+
+parser = ManifestParser()
+
+# Parse manifest (handles version detection and migration)
+parsed = parser.parse(manifest)
+
+# Get schema version
+version = parser.get_schema_version(manifest)
+
+# Migrate manifest to target version
+migrated = parser.migrate(manifest, target_version="2.0")
+```
+
+### Schema Migration
+
+Manifests are automatically migrated to the current schema version:
+
+```python
+from mdb_engine.core import migrate_manifest, get_schema_version
+
+# Get current schema version
+version = get_schema_version(manifest)  # Returns "1.0" or "2.0"
+
+# Migrate to latest version
+migrated = migrate_manifest(manifest)
+
+# Migrate to specific version
+migrated = migrate_manifest(manifest, target_version="2.0")
+```
+
+### Parallel Manifest Validation
+
+Validate multiple manifests in parallel:
+
+```python
+from mdb_engine.core import validate_manifests_parallel
+
+manifests = [
+    {"slug": "app1", ...},
+    {"slug": "app2", ...},
+    {"slug": "app3", ...}
+]
+
+results = await validate_manifests_parallel(manifests)
+for slug, (is_valid, error, paths) in results.items():
+    print(f"{slug}: {'valid' if is_valid else 'invalid'}")
+```
+
+### Validation Cache
+
+Validation results are cached for performance:
+
+```python
+from mdb_engine.core import clear_validation_cache
+
+# Clear validation cache (useful after schema updates)
+clear_validation_cache()
+```
+
+## ManifestParser
+
+The `ManifestParser` class handles manifest parsing, version detection, and migration.
+
+### Basic Usage
+
+```python
+from mdb_engine.core import ManifestParser
+
+parser = ManifestParser()
+
+# Parse manifest (auto-detects version and migrates if needed)
+parsed = parser.parse(manifest)
+
+# Get schema version
+version = parser.get_schema_version(manifest)
+
+# Check if migration is needed
+needs_migration = parser.needs_migration(manifest)
+```
+
+## ManifestValidator
+
+The `ManifestValidator` class provides schema validation with caching.
+
+### Basic Usage
+
+```python
+from mdb_engine.core import ManifestValidator
+
+validator = ManifestValidator()
+
+# Validate manifest
+is_valid, error_message, error_paths = validator.validate(manifest)
+
+if not is_valid:
+    print(f"Validation failed: {error_message}")
+    print(f"Error paths: {error_paths}")
+```
+
+### Validation with Database
+
+Validate manifest and check against database:
+
+```python
+from mdb_engine.core import validate_manifest_with_db
+
+is_valid, error, paths = await validate_manifest_with_db(
+    manifest=manifest,
+    db=db
+)
+```
+
+## API Reference
+
+### MongoDBEngine
+
+#### Methods
+
+- `initialize()` - Initialize MongoDB connection and engine
+- `get_scoped_db(app_slug, read_scopes=None, write_scope=None, auto_index=True)` - Get scoped database wrapper
+- `validate_manifest(manifest)` - Validate manifest against schema
+- `load_manifest(path)` - Load and validate manifest from file
+- `register_app(manifest)` - Register app with engine
+- `get_registered_apps()` - Get all registered apps
+- `get_app_info(app_slug)` - Get information about registered app
+- `unregister_app(app_slug)` - Unregister an app
+
+#### Properties
+
+- `mongo_client` - MongoDB client instance
+- `mongo_db` - MongoDB database instance
+- `initialized` - Whether engine is initialized
+
+### ManifestValidator
+
+#### Methods
+
+- `validate(manifest)` - Validate manifest against schema
+  - Returns: `(is_valid, error_message, error_paths)`
+
+### ManifestParser
+
+#### Methods
+
+- `parse(manifest)` - Parse manifest (auto-migrates if needed)
+- `get_schema_version(manifest)` - Get schema version of manifest
+- `needs_migration(manifest)` - Check if migration is needed
+- `migrate(manifest, target_version=None)` - Migrate manifest to target version
+
+### Functions
+
+- `validate_manifest(manifest)` - Convenience function for validation
+- `validate_manifest_with_db(manifest, db)` - Validate with database checks
+- `validate_managed_indexes(manifest)` - Validate managed index definitions
+- `validate_index_definition(index_def, index_name)` - Validate single index definition
+- `get_schema_version(manifest)` - Get schema version
+- `migrate_manifest(manifest, target_version=None)` - Migrate manifest
+- `get_schema_for_version(version)` - Get schema definition for version
+- `clear_validation_cache()` - Clear validation cache
+- `validate_manifests_parallel(manifests)` - Validate multiple manifests in parallel
+
+## Configuration
+
+### Environment Variables
+
+```bash
+# MongoDB connection pool settings
+export MONGO_ACTOR_MAX_POOL_SIZE=10
+export MONGO_ACTOR_MIN_POOL_SIZE=1
+
+# Server selection timeout
+export MONGO_SERVER_SELECTION_TIMEOUT_MS=5000
+```
+
+### MongoDBEngine Parameters
+
+- `mongo_uri` (required): MongoDB connection URI
+- `db_name` (required): Database name
+- `manifests_dir` (optional): Directory containing manifest files
+- `authz_provider` (optional): Authorization provider instance
+- `max_pool_size` (optional): Maximum connection pool size (default: 10)
+- `min_pool_size` (optional): Minimum connection pool size (default: 1)
+
+## Manifest Schema Features
+
+### Collections
+
+Define collections with indexes:
+
+```json
+{
+  "collections": {
+    "users": {
+      "indexes": [
+        {
+          "name": "email_idx",
+          "type": "regular",
+          "keys": {"email": 1},
+          "unique": true
+        }
+      ]
+    }
+  }
+}
+```
+
+### Authentication & Authorization
+
+Configure unified authentication and authorization:
+
+```json
+{
+  "auth_required": true,
+  "auth_policy": {
+    "provider": "casbin",
+    "required": true,
+    "allow_anonymous": false,
+    "authorization": {
+      "model": "rbac",
+      "policies_collection": "casbin_policies",
+      "link_users_roles": true,
+      "default_roles": ["user", "admin"]
+    }
+  }
+}
+```
+
+**Key Features:**
+- **Auto-created Provider:** Casbin provider automatically created from manifest (default)
+- **MongoDB-backed:** Policies stored in MongoDB collection
+- **App-Level User Management:** App-level users automatically get Casbin roles
+- **Extensible:** Supports custom providers, models, and manual setup
+
+**Extensibility:**
+- Use `"provider": "oso"` for OSO/Polar-based authorization
+- Use `"provider": "custom"` and set `app.state.authz_provider` manually
+- Provide custom Casbin model files via `authorization.model` path
+- Implement `AuthorizationProvider` protocol for fully custom authorization logic
+
+### App-Level User Management
+
+Configure app-level user management:
+
+```json
+{
+  "auth": {
+    "users": {
+      "strategy": "app_users",
+      "allow_registration": true,
+      "demo_users": [
+        {
+          "email": "demo@example.com",
+          "password": "demo123",
+          "role": "user"
+        }
+      ]
+    }
+  }
+}
+```
+
+### Indexes
+
+Define various index types:
+
+```json
+{
+  "collections": {
+    "documents": {
+      "indexes": [
+        {
+          "name": "text_search",
+          "type": "text",
+          "keys": {"title": "text", "content": "text"}
+        },
+        {
+          "name": "vector_search",
+          "type": "vector_search",
+          "definition": {
+            "fields": [{
+              "type": "vector",
+              "path": "embedding",
+              "numDimensions": 1536
+            }]
+          }
+        }
+      ]
+    }
+  }
+}
+```
+
+## Best Practices
+
+1. **Always validate manifests** - Use `validate_manifest()` before registration
+2. **Use schema versioning** - Specify `schema_version` in manifests
+3. **Initialize once** - Create one MongoDBEngine instance per application
+4. **Use scoped databases** - Always use `get_scoped_db()` for data isolation
+5. **Monitor health** - Regularly check engine and MongoDB health
+6. **Cache validation** - Validation results are cached automatically
+7. **Handle errors** - Wrap initialization in try/except blocks
+8. **Clean up** - Unregister apps when no longer needed
+
+## Error Handling
+
+```python
+from mdb_engine.exceptions import InitializationError
+
+try:
+    await engine.initialize()
+except InitializationError as e:
+    print(f"Initialization failed: {e}")
+    print(f"MongoDB URI: {e.mongo_uri}")
+except Exception as e:
+    print(f"Unexpected error: {e}")
+```
+
+## Integration Examples
+
+### FastAPI Integration
+
+```python
+from fastapi import FastAPI
+from mdb_engine import MongoDBEngine
+
+app = FastAPI()
+engine = None
+
+@app.on_event("startup")
+async def startup():
+    global engine
+    engine = MongoDBEngine(mongo_uri="...", db_name="...")
+    await engine.initialize()
+
+    manifest = await engine.load_manifest("manifest.json")
+    await engine.register_app(manifest)
+
+@app.get("/data")
+async def get_data():
+    db = engine.get_scoped_db("my_app")
+    docs = await db.collection.find({}).to_list(length=10)
+    return {"data": docs}
+```
+
+### Multiple Apps
+
+```python
+# Register multiple apps
+manifests = [
+    await engine.load_manifest("app1/manifest.json"),
+    await engine.load_manifest("app2/manifest.json"),
+    await engine.load_manifest("app3/manifest.json")
+]
+
+for manifest in manifests:
+    await engine.register_app(manifest)
+
+# Access different apps
+db1 = engine.get_scoped_db("app1")
+db2 = engine.get_scoped_db("app2")
+```
+
+## Related Modules
+
+- **`database/`** - Database abstraction and scoped wrappers
+- **`auth/`** - Authentication and authorization
+- **`indexes/`** - Index management
+- **`observability/`** - Health checks and metrics