mdb-engine 0.1.6__py3-none-any.whl

mdb_engine/core/seeding.py

@@ -0,0 +1,179 @@
+"""
+Initial Data Seeding
+
+Provides utilities for seeding initial data into collections based on manifest configuration.
+
+This module is part of MDB_ENGINE - MongoDB Engine.
+"""
+
+import logging
+from datetime import datetime
+from typing import Any, Dict, List
+
+from pymongo.errors import (ConnectionFailure, OperationFailure,
+                            ServerSelectionTimeoutError)
+
+logger = logging.getLogger(__name__)
+
+
+async def seed_initial_data(
+    db, app_slug: str, initial_data: Dict[str, List[Dict[str, Any]]]
+) -> Dict[str, int]:
+    """
+    Seed initial data into collections.
+
+    This function:
+    1. Checks if each collection is empty
+    2. Only seeds if collection is empty (idempotent)
+    3. Tracks seeded collections in metadata collection
+    4. Handles datetime conversion for seed data
+
+    Args:
+        db: Database wrapper (ScopedMongoWrapper or AppDB)
+        app_slug: App slug identifier
+        initial_data: Dictionary mapping collection names to arrays of documents
+
+    Returns:
+        Dictionary mapping collection names to number of documents inserted
+    """
+    results = {}
+
+    # Check metadata collection to see if we've already seeded
+    # Use a non-underscore name to avoid ScopedMongoWrapper attribute access restrictions
+    metadata_collection_name = "app_seeding_metadata"
+    # Use getattr to access collection (works with ScopedMongoWrapper and AppDB)
+    metadata_collection = getattr(db, metadata_collection_name)
+
+    # Get existing seeding metadata
+    seeding_metadata = await metadata_collection.find_one({"app_slug": app_slug})
+    seeded_collections = (
+        set(seeding_metadata.get("seeded_collections", []))
+        if seeding_metadata
+        else set()
+    )
+
+    for collection_name, documents in initial_data.items():
+        try:
+            # Check if already seeded
+            if collection_name in seeded_collections:
+                logger.debug(
+                    f"Collection '{collection_name}' already seeded for {app_slug}, skipping"
+                )
+                results[collection_name] = 0
+                continue
+
+            # Get collection
+            collection = getattr(db, collection_name)
+
+            # Check if collection is empty (idempotent check)
+            count = await collection.count_documents({})
+            if count > 0:
+                logger.info(
+                    f"Collection '{collection_name}' is not empty "
+                    f"({count} documents) for {app_slug}, "
+                    f"skipping seed to avoid duplicates"
+                )
+                # Mark as seeded even though we didn't insert (collection already has data)
+                seeded_collections.add(collection_name)
+                results[collection_name] = 0
+                continue
+
+            # Prepare documents for insertion
+            # Convert datetime strings to datetime objects if needed
+            prepared_docs = []
+            for doc in documents:
+                prepared_doc = doc.copy()
+
+                # Convert datetime strings to datetime objects
+                for key, value in prepared_doc.items():
+                    if isinstance(value, str):
+                        # Try to parse ISO format datetime strings
+                        try:
+                            # Check if it looks like a datetime string
+                            if "T" in value and (
+                                "Z" in value or "+" in value or "-" in value[-6:]
+                            ):
+                                # Try parsing as ISO format
+                                from dateutil.parser import parse as parse_date
+
+                                prepared_doc[key] = parse_date(value)
+                        except (ValueError, ImportError):
+                            # Not a datetime string or dateutil not available, keep as string
+                            pass
+                    elif isinstance(value, dict) and "$date" in value:
+                        # MongoDB extended JSON format
+                        try:
+                            from dateutil.parser import parse as parse_date
+
+                            prepared_doc[key] = parse_date(value["$date"])
+                        except (ValueError, ImportError):
+                            pass
+
+                # Add created_at if not present and document doesn't have timestamp fields
+                if (
+                    "created_at" not in prepared_doc
+                    and "date_created" not in prepared_doc
+                ):
+                    prepared_doc["created_at"] = datetime.utcnow()
+
+                prepared_docs.append(prepared_doc)
+
+            # Insert documents
+            if prepared_docs:
+                result = await collection.insert_many(prepared_docs)
+                inserted_count = len(result.inserted_ids)
+                results[collection_name] = inserted_count
+
+                # Mark as seeded
+                seeded_collections.add(collection_name)
+
+                logger.info(
+                    f"✅ Seeded {inserted_count} document(s) into collection "
+                    f"'{collection_name}' for {app_slug}"
+                )
+            else:
+                results[collection_name] = 0
+                logger.warning(
+                    f"No documents to seed for collection '{collection_name}' in {app_slug}"
+                )
+
+        except (
+            OperationFailure,
+            ConnectionFailure,
+            ServerSelectionTimeoutError,
+            Exception,
+        ) as e:
+            # Type 2: Recoverable - log error and continue with other collections
+            logger.exception(
+                f"Failed to seed collection '{collection_name}' for {app_slug}: {e}"
+            )
+            results[collection_name] = 0
+
+    # Update seeding metadata
+    try:
+        if seeding_metadata:
+            await metadata_collection.update_one(
+                {"app_slug": app_slug},
+                {"$set": {"seeded_collections": list(seeded_collections)}},
+            )
+        else:
+            await metadata_collection.insert_one(
+                {
+                    "app_slug": app_slug,
+                    "seeded_collections": list(seeded_collections),
+                    "created_at": datetime.utcnow(),
+                }
+            )
+    except (
+        OperationFailure,
+        ConnectionFailure,
+        ServerSelectionTimeoutError,
+        ValueError,
+        TypeError,
+        KeyError,
+    ) as e:
+        logger.warning(
+            f"Failed to update seeding metadata for {app_slug}: {e}", exc_info=True
+        )
+
+    return results

mdb_engine/core/service_initialization.py

@@ -0,0 +1,355 @@
+"""
+Service initialization for MongoDB Engine.
+
+This module handles initialization of optional services:
+- Memory service (Mem0)
+- WebSocket endpoints
+- Observability (health checks, metrics, logging)
+- Data seeding
+
+This module is part of MDB_ENGINE - MongoDB Engine.
+"""
+
+import logging
+from typing import Any, Callable, Dict, List, Optional
+
+from pymongo.errors import (ConnectionFailure, OperationFailure,
+                            ServerSelectionTimeoutError)
+
+from ..database import ScopedMongoWrapper
+from ..observability import get_logger as get_contextual_logger
+
+logger = logging.getLogger(__name__)
+contextual_logger = get_contextual_logger(__name__)
+
+
+class ServiceInitializer:
+    """
+    Manages initialization of optional services for apps.
+
+    Handles memory service, WebSocket endpoints, observability, and data seeding.
+    """
+
+    def __init__(
+        self,
+        mongo_uri: str,
+        db_name: str,
+        get_scoped_db_fn: Callable[[str], ScopedMongoWrapper],
+    ) -> None:
+        """
+        Initialize the service initializer.
+
+        Args:
+            mongo_uri: MongoDB connection URI
+            db_name: Database name
+            get_scoped_db_fn: Function to get scoped database wrapper
+        """
+        self.mongo_uri = mongo_uri
+        self.db_name = db_name
+        self.get_scoped_db_fn = get_scoped_db_fn
+        self._memory_services: Dict[str, Any] = {}
+        self._websocket_configs: Dict[str, Dict[str, Any]] = {}
+
+    async def initialize_memory_service(
+        self, slug: str, memory_config: Dict[str, Any]
+    ) -> None:
+        """
+        Initialize Mem0 memory service for an app.
+
+        Memory support is OPTIONAL - only processes if dependencies are available.
+        mem0 handles embeddings and LLM via environment variables (.env).
+
+        Args:
+            slug: App slug
+            memory_config: Memory configuration from manifest (already validated)
+        """
+        # Try to import Memory service (optional dependency)
+        try:
+            from ..memory import Mem0MemoryService, Mem0MemoryServiceError
+        except ImportError as e:
+            contextual_logger.warning(
+                f"Memory configuration found for app '{slug}' but "
+                f"dependencies are not available: {e}. "
+                f"Memory support will be disabled for this app. Install with: "
+                f"pip install mem0ai"
+            )
+            return
+
+        contextual_logger.info(
+            f"Initializing Mem0 memory service for app '{slug}'",
+            extra={
+                "app_slug": slug,
+                "collection_name": memory_config.get(
+                    "collection_name", f"{slug}_memories"
+                ),
+                "enable_graph": memory_config.get("enable_graph", False),
+                "embedding_model_dims": memory_config.get("embedding_model_dims", 1536),
+                "infer": memory_config.get("infer", True),
+            },
+        )
+
+        try:
+            # Extract memory config (exclude 'enabled')
+            service_config = {
+                k: v
+                for k, v in memory_config.items()
+                if k != "enabled"
+                and k
+                in [
+                    "collection_name",
+                    "embedding_model_dims",
+                    "enable_graph",
+                    "infer",
+                    "async_mode",
+                    "embedding_model",
+                    "chat_model",
+                    "temperature",
+                ]
+            }
+
+            # Set default collection name if not provided
+            if "collection_name" not in service_config:
+                service_config["collection_name"] = f"{slug}_memories"
+            else:
+                # Ensure collection name is prefixed with app slug
+                collection_name = service_config["collection_name"]
+                if not collection_name.startswith(f"{slug}_"):
+                    service_config["collection_name"] = f"{slug}_{collection_name}"
+                    contextual_logger.info(
+                        f"Prefixed memory collection name: "
+                        f"'{collection_name}' -> "
+                        f"'{service_config['collection_name']}'",
+                        extra={
+                            "app_slug": slug,
+                            "original": collection_name,
+                            "prefixed": service_config["collection_name"],
+                        },
+                    )
+
+            # Create Memory service with MongoDB integration
+            memory_service = Mem0MemoryService(
+                mongo_uri=self.mongo_uri,
+                db_name=self.db_name,
+                app_slug=slug,
+                config=service_config,
+            )
+            self._memory_services[slug] = memory_service
+
+            contextual_logger.info(
+                f"Mem0 memory service initialized for app '{slug}'",
+                extra={"app_slug": slug},
+            )
+        except Mem0MemoryServiceError as e:
+            contextual_logger.error(
+                f"Failed to initialize memory service for app '{slug}': {e}",
+                extra={"app_slug": slug, "error": str(e)},
+                exc_info=True,
+            )
+        except (ImportError, AttributeError, TypeError, ValueError) as e:
+            contextual_logger.error(
+                f"Error initializing memory service for app '{slug}': {e}",
+                extra={"app_slug": slug, "error": str(e)},
+                exc_info=True,
+            )
+
+    async def register_websockets(
+        self, slug: str, websockets_config: Dict[str, Any]
+    ) -> None:
+        """
+        Register WebSocket endpoints for an app.
+
+        WebSocket support is OPTIONAL - only processes if dependencies are available.
+
+        Args:
+            slug: App slug
+            websockets_config: WebSocket configuration from manifest
+        """
+        # Try to import WebSocket support (optional dependency)
+        try:
+            from ..routing.websockets import get_websocket_manager
+        except ImportError as e:
+            contextual_logger.warning(
+                f"WebSocket configuration found for app '{slug}' but "
+                f"dependencies are not available: {e}. "
+                f"WebSocket support will be disabled for this app. "
+                f"Install FastAPI with WebSocket support."
+            )
+            return
+
+        contextual_logger.info(
+            f"Registering WebSocket endpoints for app '{slug}'",
+            extra={"app_slug": slug, "endpoint_count": len(websockets_config)},
+        )
+
+        # Store WebSocket configuration for later route registration
+        self._websocket_configs[slug] = websockets_config
+
+        # Pre-initialize WebSocket managers
+        for endpoint_name, endpoint_config in websockets_config.items():
+            path = endpoint_config.get("path", f"/{endpoint_name}")
+            try:
+                await get_websocket_manager(slug)
+            except (ImportError, AttributeError, RuntimeError) as e:
+                contextual_logger.warning(
+                    f"Could not initialize WebSocket manager for {slug}: {e}"
+                )
+                continue
+            contextual_logger.debug(
+                f"Configured WebSocket endpoint '{endpoint_name}' at path '{path}'",
+                extra={"app_slug": slug, "endpoint": endpoint_name, "path": path},
+            )
+
+    async def seed_initial_data(
+        self, slug: str, initial_data: Dict[str, List[Dict[str, Any]]]
+    ) -> None:
+        """
+        Seed initial data into collections for an app.
+
+        Args:
+            slug: App slug
+            initial_data: Dictionary mapping collection names to arrays of documents
+        """
+        try:
+            from .seeding import seed_initial_data
+
+            db = self.get_scoped_db_fn(slug)
+            results = await seed_initial_data(db, slug, initial_data)
+
+            total_inserted = sum(results.values())
+            if total_inserted > 0:
+                contextual_logger.info(
+                    f"Seeded initial data for app '{slug}'",
+                    extra={
+                        "app_slug": slug,
+                        "collections_seeded": len(
+                            [c for c, count in results.items() if count > 0]
+                        ),
+                        "total_documents": total_inserted,
+                    },
+                )
+            else:
+                contextual_logger.debug(
+                    f"No initial data seeded for app '{slug}' "
+                    f"(collections already had data or were empty)",
+                    extra={"app_slug": slug},
+                )
+        except (
+            OperationFailure,
+            ConnectionFailure,
+            ServerSelectionTimeoutError,
+            ValueError,
+            TypeError,
+        ) as e:
+            contextual_logger.error(
+                f"Failed to seed initial data for app '{slug}': {e}",
+                extra={"app_slug": slug, "error": str(e)},
+                exc_info=True,
+            )
+
+    async def setup_observability(
+        self, slug: str, manifest: Dict[str, Any], observability_config: Dict[str, Any]
+    ) -> None:
+        """
+        Set up observability features (health checks, metrics, logging) from manifest.
+
+        Args:
+            slug: App slug
+            manifest: Full manifest dictionary
+            observability_config: Observability configuration from manifest
+        """
+        try:
+            # Set up health checks
+            health_config = observability_config.get("health_checks", {})
+            if health_config.get("enabled", True):
+                endpoint = health_config.get("endpoint", "/health")
+                contextual_logger.info(
+                    f"Health checks configured for {slug}",
+                    extra={
+                        "endpoint": endpoint,
+                        "interval_seconds": health_config.get("interval_seconds", 30),
+                    },
+                )
+
+            # Set up metrics
+            metrics_config = observability_config.get("metrics", {})
+            if metrics_config.get("enabled", True):
+                contextual_logger.info(
+                    f"Metrics collection configured for {slug}",
+                    extra={
+                        "operation_metrics": metrics_config.get(
+                            "collect_operation_metrics", True
+                        ),
+                        "performance_metrics": metrics_config.get(
+                            "collect_performance_metrics", True
+                        ),
+                        "custom_metrics": metrics_config.get("custom_metrics", []),
+                    },
+                )
+
+            # Set up logging
+            logging_config = observability_config.get("logging", {})
+            if logging_config:
+                log_level = logging_config.get("level", "INFO")
+                log_format = logging_config.get("format", "json")
+                contextual_logger.info(
+                    f"Logging configured for {slug}",
+                    extra={
+                        "level": log_level,
+                        "format": log_format,
+                        "include_request_id": logging_config.get(
+                            "include_request_id", True
+                        ),
+                    },
+                )
+
+        except (ImportError, AttributeError, TypeError, ValueError, KeyError) as e:
+            contextual_logger.warning(
+                f"Could not set up observability for {slug}: {e}", exc_info=True
+            )
+
+    def get_websocket_config(self, slug: str) -> Optional[Dict[str, Any]]:
+        """
+        Get WebSocket configuration for an app.
+
+        Args:
+            slug: App slug
+
+        Returns:
+            WebSocket configuration dict or None if not configured
+        """
+        return self._websocket_configs.get(slug)
+
+    def get_memory_service(self, slug: str) -> Optional[Any]:
+        """
+        Get Mem0 memory service for an app.
+
+        Args:
+            slug: App slug
+
+        Returns:
+            Mem0MemoryService instance if memory is enabled for this app, None otherwise
+        """
+        try:
+            service = self._memory_services.get(slug)
+            if service is not None:
+                # Quick health check - ensure it has the memory attribute
+                if not hasattr(service, "memory"):
+                    contextual_logger.warning(
+                        f"Memory service for '{slug}' is missing 'memory' "
+                        f"attribute, returning None",
+                        extra={"app_slug": slug},
+                    )
+                    return None
+            return service
+        except (KeyError, AttributeError, TypeError) as e:
+            contextual_logger.error(
+                f"Error retrieving memory service for '{slug}': {e}",
+                exc_info=True,
+                extra={"app_slug": slug, "error": str(e)},
+            )
+            return None
+
+    def clear_services(self) -> None:
+        """Clear all service state."""
+        self._memory_services.clear()
+        self._websocket_configs.clear()