kailash 0.5.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

kailash/gateway/resource_resolver.py

@@ -0,0 +1,217 @@
+"""Resource resolution system for gateway.
+
+This module provides resource reference resolution to handle non-serializable
+objects like database connections, HTTP clients, and caches through the API.
+"""
+
+import hashlib
+import json
+import logging
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, Optional
+
+from ..resources.factory import CacheFactory, DatabasePoolFactory, HttpClientFactory
+from ..resources.registry import ResourceFactory, ResourceRegistry
+from .security import SecretManager
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ResourceReference:
+    """Reference to a resource that can be resolved by the gateway."""
+
+    type: str  # database, http_client, cache, message_queue, etc.
+    config: Dict[str, Any]
+    credentials_ref: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to JSON-serializable dict."""
+        return {
+            "type": self.type,
+            "config": self.config,
+            "credentials_ref": self.credentials_ref,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ResourceReference":
+        """Create from dictionary."""
+        return cls(
+            type=data["type"],
+            config=data["config"],
+            credentials_ref=data.get("credentials_ref"),
+        )
+
+
+class ResourceResolver:
+    """Resolves resource references to actual resources."""
+
+    def __init__(
+        self, resource_registry: ResourceRegistry, secret_manager: SecretManager
+    ):
+        self.resource_registry = resource_registry
+        self.secret_manager = secret_manager
+        self._resolvers = {
+            "database": self._resolve_database,
+            "http_client": self._resolve_http_client,
+            "cache": self._resolve_cache,
+            "message_queue": self._resolve_message_queue,
+            "s3": self._resolve_s3_client,
+        }
+
+    async def resolve(self, reference: ResourceReference) -> Any:
+        """Resolve a resource reference."""
+        resolver = self._resolvers.get(reference.type)
+        if not resolver:
+            raise ValueError(f"Unknown resource type: {reference.type}")
+
+        # Get credentials if needed
+        credentials = None
+        if reference.credentials_ref:
+            credentials = await self.secret_manager.get_secret(
+                reference.credentials_ref
+            )
+
+        # Resolve resource
+        return await resolver(reference.config, credentials)
+
+    async def _resolve_database(
+        self, config: Dict[str, Any], credentials: Optional[Dict[str, Any]]
+    ) -> Any:
+        """Resolve database resource."""
+        # Merge config with credentials
+        connection_config = {**config}
+        if credentials:
+            # Only add known database credential fields
+            for key in ["user", "password", "username", "host", "port", "database"]:
+                if key in credentials:
+                    connection_config[key] = credentials[key]
+
+        # Create unique key for this configuration
+        config_str = json.dumps(connection_config, sort_keys=True)
+        pool_key = f"db_{hashlib.md5(config_str.encode()).hexdigest()[:8]}"
+
+        try:
+            # Try to get existing pool
+            return await self.resource_registry.get_resource(pool_key)
+        except:
+            # Register and create new pool
+            factory = DatabasePoolFactory(**connection_config)
+
+            # Health check
+            async def health_check(pool):
+                try:
+                    async with pool.acquire() as conn:
+                        await conn.fetchval("SELECT 1")
+                    return True
+                except Exception:
+                    return False
+
+            # Cleanup
+            async def cleanup(pool):
+                await pool.close()
+
+            self.resource_registry.register_factory(
+                pool_key, factory, health_check=health_check, cleanup_handler=cleanup
+            )
+            return await self.resource_registry.get_resource(pool_key)
+
+    async def _resolve_http_client(
+        self, config: Dict[str, Any], credentials: Optional[Dict[str, Any]]
+    ) -> Any:
+        """Resolve HTTP client resource."""
+        # Apply credentials as headers if provided
+        if credentials and "headers" not in config:
+            config["headers"] = {}
+
+        if credentials:
+            if "api_key" in credentials:
+                config["headers"]["Authorization"] = f"Bearer {credentials['api_key']}"
+            elif "token" in credentials:
+                config["headers"]["Authorization"] = f"Bearer {credentials['token']}"
+            elif "headers" in credentials:
+                config["headers"].update(credentials["headers"])
+
+        # Create unique key
+        config_str = json.dumps(config, sort_keys=True)
+        client_key = f"http_{hashlib.md5(config_str.encode()).hexdigest()[:8]}"
+
+        try:
+            return await self.resource_registry.get_resource(client_key)
+        except:
+            factory = HttpClientFactory(**config)
+
+            async def cleanup(session):
+                await session.close()
+
+            self.resource_registry.register_factory(
+                client_key, factory, cleanup_handler=cleanup
+            )
+            return await self.resource_registry.get_resource(client_key)
+
+    async def _resolve_cache(
+        self, config: Dict[str, Any], credentials: Optional[Dict[str, Any]]
+    ) -> Any:
+        """Resolve cache resource."""
+        if credentials and "password" in credentials:
+            config["password"] = credentials["password"]
+
+        # Create unique key
+        cache_key = (
+            f"cache_{config.get('host', 'localhost')}_{config.get('port', 6379)}"
+        )
+
+        try:
+            return await self.resource_registry.get_resource(cache_key)
+        except:
+            factory = CacheFactory(**config)
+
+            async def health_check(cache):
+                try:
+                    await cache.ping()
+                    return True
+                except Exception:
+                    return False
+
+            async def cleanup(cache):
+                await cache.aclose()
+
+            self.resource_registry.register_factory(
+                cache_key, factory, health_check=health_check, cleanup_handler=cleanup
+            )
+            return await self.resource_registry.get_resource(cache_key)
+
+    async def _resolve_message_queue(
+        self, config: Dict[str, Any], credentials: Optional[Dict[str, Any]]
+    ) -> Any:
+        """Resolve message queue resource."""
+        # Implementation depends on the message queue system
+        # This is a placeholder
+        queue_type = config.get("type", "rabbitmq")
+
+        if queue_type == "rabbitmq":
+            # Would implement RabbitMQ connection
+            pass
+        elif queue_type == "kafka":
+            # Would implement Kafka connection
+            pass
+
+        raise NotImplementedError(f"Message queue type {queue_type} not implemented")
+
+    async def _resolve_s3_client(
+        self, config: Dict[str, Any], credentials: Optional[Dict[str, Any]]
+    ) -> Any:
+        """Resolve S3 client resource."""
+        # Implementation would depend on boto3 or aioboto3
+        # This is a placeholder
+        if credentials:
+            if "access_key" in credentials:
+                config["aws_access_key_id"] = credentials["access_key"]
+            if "secret_key" in credentials:
+                config["aws_secret_access_key"] = credentials["secret_key"]
+
+        # Create unique key
+        s3_key = f"s3_{config.get('region', 'us-east-1')}"
+
+        # Would implement S3 client creation here
+        raise NotImplementedError("S3 client resolution not yet implemented")

kailash/gateway/security.py

@@ -0,0 +1,227 @@
+"""Security and secret management for gateway.
+
+This module provides secure credential management with encryption
+and multiple backend options for storing secrets.
+"""
+
+import asyncio
+import base64
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+from datetime import UTC, datetime, timedelta
+from typing import Any, Dict, Optional, Tuple
+
+from cryptography.fernet import Fernet
+
+logger = logging.getLogger(__name__)
+
+
+class SecretNotFoundError(Exception):
+    """Raised when secret is not found."""
+
+    pass
+
+
+class SecretBackend(ABC):
+    """Abstract backend for secret storage."""
+
+    @abstractmethod
+    async def get_secret(self, reference: str) -> Dict[str, Any]:
+        """Get secret by reference."""
+        pass
+
+    @abstractmethod
+    async def store_secret(self, reference: str, secret: Dict[str, Any]) -> None:
+        """Store a secret."""
+        pass
+
+    @abstractmethod
+    async def delete_secret(self, reference: str) -> None:
+        """Delete a secret."""
+        pass
+
+
+class SecretManager:
+    """Manages secrets for resource credentials."""
+
+    def __init__(
+        self,
+        backend: Optional[SecretBackend] = None,
+        encryption_key: Optional[str] = None,
+        cache_ttl: int = 300,  # 5 minutes
+    ):
+        self.backend = backend or EnvironmentSecretBackend()
+        self._cache: Dict[str, Tuple[Any, datetime]] = {}
+        self._ttl = timedelta(seconds=cache_ttl)
+        self._lock = asyncio.Lock()
+
+        # Set up encryption
+        if encryption_key:
+            self._cipher = Fernet(encryption_key.encode())
+        else:
+            # Generate key from environment or use default
+            key = os.environ.get("KAILASH_ENCRYPTION_KEY")
+            if not key:
+                # Warning: This is not secure for production!
+                logger.warning(
+                    "Using default encryption key - not secure for production!"
+                )
+                # Generate a proper Fernet key
+                key = Fernet.generate_key()
+            elif isinstance(key, str):
+                key = key.encode()
+            self._cipher = Fernet(key)
+
+    async def get_secret(self, reference: str) -> Dict[str, Any]:
+        """Get secret by reference."""
+        async with self._lock:
+            # Check cache
+            if reference in self._cache:
+                value, timestamp = self._cache[reference]
+                if datetime.now(UTC) - timestamp < self._ttl:
+                    return value
+                else:
+                    # Expired, remove from cache
+                    del self._cache[reference]
+
+        # Fetch from backend
+        encrypted_secret = await self.backend.get_secret(reference)
+
+        # Decrypt if needed
+        if isinstance(encrypted_secret, str) and encrypted_secret.startswith(
+            "encrypted:"
+        ):
+            decrypted = self._cipher.decrypt(encrypted_secret[10:].encode()).decode()
+            secret = json.loads(decrypted)
+        elif isinstance(encrypted_secret, dict) and "value" in encrypted_secret:
+            # Handle case where backend returns {"value": "encrypted:..."}
+            value = encrypted_secret["value"]
+            if isinstance(value, str) and value.startswith("encrypted:"):
+                decrypted = self._cipher.decrypt(value[10:].encode()).decode()
+                secret = json.loads(decrypted)
+            else:
+                secret = encrypted_secret
+        else:
+            secret = encrypted_secret
+
+        # Cache it
+        async with self._lock:
+            self._cache[reference] = (secret, datetime.now(UTC))
+
+        return secret
+
+    async def store_secret(
+        self, reference: str, secret: Dict[str, Any], encrypt: bool = True
+    ) -> None:
+        """Store a secret."""
+        if encrypt:
+            # Encrypt the secret
+            secret_json = json.dumps(secret)
+            encrypted = self._cipher.encrypt(secret_json.encode())
+            encrypted_value = f"encrypted:{encrypted.decode()}"
+            await self.backend.store_secret(reference, encrypted_value)
+        else:
+            await self.backend.store_secret(reference, secret)
+
+        # Clear from cache
+        async with self._lock:
+            if reference in self._cache:
+                del self._cache[reference]
+
+    async def delete_secret(self, reference: str) -> None:
+        """Delete a secret."""
+        await self.backend.delete_secret(reference)
+
+        # Clear from cache
+        async with self._lock:
+            if reference in self._cache:
+                del self._cache[reference]
+
+    async def clear_cache(self):
+        """Clear the secret cache."""
+        async with self._lock:
+            self._cache.clear()
+
+
+class EnvironmentSecretBackend(SecretBackend):
+    """Secret backend using environment variables."""
+
+    def __init__(self, prefix: str = "KAILASH_SECRET_"):
+        self.prefix = prefix
+
+    async def get_secret(self, reference: str) -> Dict[str, Any]:
+        """Get secret from environment."""
+        # Convert reference to env var name
+        env_var = f"{self.prefix}{reference.upper()}"
+
+        value = os.environ.get(env_var)
+        if not value:
+            raise SecretNotFoundError(f"Secret {reference} not found")
+
+        # Try to parse as JSON
+        try:
+            return json.loads(value)
+        except json.JSONDecodeError:
+            # Return as simple key-value
+            return {"value": value}
+
+    async def store_secret(self, reference: str, secret: Any) -> None:
+        """Store secret in environment (not recommended for production)."""
+        env_var = f"{self.prefix}{reference.upper()}"
+
+        if isinstance(secret, dict):
+            os.environ[env_var] = json.dumps(secret)
+        else:
+            os.environ[env_var] = str(secret)
+
+    async def delete_secret(self, reference: str) -> None:
+        """Delete secret from environment."""
+        env_var = f"{self.prefix}{reference.upper()}"
+        if env_var in os.environ:
+            del os.environ[env_var]
+
+
+class FileSecretBackend(SecretBackend):
+    """Secret backend using encrypted file storage."""
+
+    def __init__(self, secrets_dir: str = "/etc/kailash/secrets"):
+        self.secrets_dir = secrets_dir
+        os.makedirs(secrets_dir, exist_ok=True)
+
+    async def get_secret(self, reference: str) -> Dict[str, Any]:
+        """Get secret from file."""
+        file_path = os.path.join(self.secrets_dir, f"{reference}.json")
+
+        if not os.path.exists(file_path):
+            raise SecretNotFoundError(f"Secret {reference} not found")
+
+        with open(file_path, "r") as f:
+            return json.load(f)
+
+    async def store_secret(self, reference: str, secret: Any) -> None:
+        """Store secret in file."""
+        file_path = os.path.join(self.secrets_dir, f"{reference}.json")
+
+        with open(file_path, "w") as f:
+            if isinstance(secret, str):
+                f.write(secret)
+            else:
+                json.dump(secret, f)
+
+        # Set restrictive permissions
+        os.chmod(file_path, 0o600)
+
+    async def delete_secret(self, reference: str) -> None:
+        """Delete secret file."""
+        file_path = os.path.join(self.secrets_dir, f"{reference}.json")
+        if os.path.exists(file_path):
+            os.remove(file_path)
+
+
+# For production, you would implement:
+# - VaultSecretBackend for HashiCorp Vault
+# - AWSSecretsManagerBackend for AWS Secrets Manager
+# - AzureKeyVaultBackend for Azure Key Vault
+# - GCPSecretManagerBackend for Google Cloud Secret Manager

kailash/middleware/auth/access_control.py

@@ -72,7 +72,7 @@ class MiddlewareAccessControlManager:
         """Check if user can access a specific session."""
 
         # Use Kailash permission check node
-        result = self.permission_check_node.process(
+        result = self.permission_check_node.execute(
             {
                 "user_context": user_context,
                 "resource_type": "session",
@@ -114,7 +114,7 @@ class MiddlewareAccessControlManager:
 
         # Audit logging using Kailash audit node
         if self.enable_audit and self.audit_node:
-            self.audit_node.process(
+            self.audit_node.execute(
                 {
                     "event_type": "workflow_access_check",
                     "user_id": user_context.user_id,
@@ -192,7 +192,7 @@ class MiddlewareAccessControlManager:
     ) -> Dict[str, Any]:
         """Assign role to user using Kailash role management node."""
 
-        result = self.role_mgmt_node.process(
+        result = self.role_mgmt_node.execute(
             {
                 "action": "assign_role",
                 "user_id": user_id,
@@ -239,7 +239,7 @@ class MiddlewareAccessControlManager:
 
         # Audit the rule creation
         if self.enable_audit and self.audit_node:
-            self.audit_node.process(
+            self.audit_node.execute(
                 {
                     "event_type": "permission_rule_created",
                     "rule_data": rule_data,
@@ -366,7 +366,7 @@ class MiddlewareAuthenticationMiddleware:
         try:
             # This would typically validate JWT token
             # For now, simulating with credential manager
-            cred_result = self.credential_manager.process(
+            cred_result = self.credential_manager.execute(
                 {"action": "validate_token", "token": token}
             )
 
@@ -388,7 +388,7 @@ class MiddlewareAuthenticationMiddleware:
 
         except Exception as e:
             # Log security event using Kailash security event node
-            self.access_manager.security_event_node.process(
+            self.access_manager.security_event_node.execute(
                 {
                     "event_type": "authentication_failure",
                     "error": str(e),

kailash/middleware/auth/models.py

@@ -6,7 +6,7 @@ These models can be imported anywhere in the codebase safely.
 """
 
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import List, Optional
 
 
@@ -102,7 +102,7 @@ class RefreshTokenData:
 
     def __post_init__(self):
         if self.created_at is None:
-            self.created_at = datetime.utcnow()
+            self.created_at = datetime.now(UTC)
 
 
 @dataclass

kailash/middleware/communication/ai_chat.py

@@ -371,7 +371,7 @@ EXPLANATION:
 
         try:
             result = await asyncio.to_thread(
-                self.llm_node.process, messages=[{"role": "user", "content": prompt}]
+                self.llm_node.execute, messages=[{"role": "user", "content": prompt}]
             )
 
             # Extract content from response
@@ -847,10 +847,10 @@ What would you like to work on? Just describe what you want to accomplish and I'
         """Store chat message with embedding in vector database."""
         try:
             # Generate embedding
-            embedding_result = await self.embedding_node.process({"text": content})
+            embedding_result = self.embedding_node.execute(text=content)
 
             # Store in database (simplified for now)
-            await self.vector_db.process(
+            self.vector_db.execute(
                 {
                     "query": "INSERT INTO chat_messages (id, session_id, user_id, content, role, timestamp) VALUES (?, ?, ?, ?, ?, ?)",
                     "parameters": [
@@ -875,10 +875,10 @@ What would you like to work on? Just describe what you want to accomplish and I'
         """Find similar past conversations using vector search."""
         try:
             # Generate query embedding
-            query_embedding = await self.embedding_node.process({"text": query})
+            query_embedding = self.embedding_node.execute(text=query)
 
             # Search for similar messages (simplified for now)
-            search_result = await self.vector_db.process(
+            search_result = self.vector_db.execute(
                 {
                     "query": "SELECT * FROM chat_messages WHERE role = 'user' ORDER BY timestamp DESC LIMIT ?",
                     "parameters": [limit * 2],
@@ -930,7 +930,7 @@ What would you like to work on? Just describe what you want to accomplish and I'
 
         try:
             # Generate query embedding
-            query_embedding = await self.embedding_node.process({"text": query})
+            query_embedding = self.embedding_node.execute(text=query)
 
             # Prepare filters
             filters = {}
@@ -948,7 +948,7 @@ What would you like to work on? Just describe what you want to accomplish and I'
             query_parts.append("ORDER BY timestamp DESC LIMIT ?")
             params.append(limit)
 
-            search_result = await self.vector_db.process(
+            search_result = self.vector_db.execute(
                 {"query": " ".join(query_parts), "parameters": params}
             )
 

kailash/middleware/communication/api_gateway.py

@@ -225,10 +225,8 @@ class APIGateway:
         # Data transformer for request/response formatting
         self.data_transformer = DataTransformer(
             name="gateway_transformer",
-            transformations=[
-                {"type": "validate", "schema": "api_response"},
-                {"type": "add_field", "field": "timestamp", "value": "now()"},
-            ],
+            # Transformations will be provided at runtime
+            transformations=[],
         )
 
         # Credential manager for gateway security
@@ -362,17 +360,9 @@ class APIGateway:
                     "active": session.active,
                 }
 
-                transformed = await self.data_transformer.process(
-                    {
-                        "data": response_data,
-                        "transformations": [
-                            {
-                                "type": "add_field",
-                                "field": "api_version",
-                                "value": self.version,
-                            }
-                        ],
-                    }
+                transformed = self.data_transformer.execute(
+                    data=response_data,
+                    transformations=[f"{{**data, 'api_version': '{self.version}'}}"],
                 )
 
                 return SessionResponse(**transformed["result"])

kailash/middleware/database/base_models.py

@@ -8,13 +8,7 @@ import uuid
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional
 
-from sqlalchemy import (
-    JSON,
-    Boolean,
-    CheckConstraint,
-    Column,
-    DateTime,
-)
+from sqlalchemy import JSON, Boolean, CheckConstraint, Column, DateTime
 from sqlalchemy import Enum as SQLEnum
 from sqlalchemy import (
     Float,

kailash/middleware/gateway/__init__.py

@@ -0,0 +1,22 @@
+"""Durable gateway implementation for production-grade request handling.
+
+This module provides:
+- Request durability with checkpointing
+- Automatic deduplication
+- Event sourcing for full auditability
+- Long-running request support
+"""
+
+from .checkpoint_manager import CheckpointManager
+from .deduplicator import RequestDeduplicator
+from .durable_request import DurableRequest, RequestState
+from .event_store import EventStore, RequestEvent
+
+__all__ = [
+    "DurableRequest",
+    "RequestState",
+    "CheckpointManager",
+    "RequestDeduplicator",
+    "EventStore",
+    "RequestEvent",
+]