kailash 0.7.0__py3-none-any.whl → 0.8.1__py3-none-any.whl

kailash/middleware/gateway/storage_backends.py

@@ -0,0 +1,393 @@
+"""Storage backend implementations for middleware components."""
+
+import asyncio
+import json
+from abc import ABC, abstractmethod
+from datetime import UTC, datetime
+from typing import Any, Dict, List, Optional
+
+try:
+    import redis.asyncio as redis
+except ImportError:
+    try:
+        import aioredis as redis
+    except ImportError:
+        redis = None
+import asyncpg
+
+
+class StorageBackend(ABC):
+    """Abstract base class for storage backends."""
+
+    @abstractmethod
+    async def save(self, key: str, data: bytes) -> None:
+        """Save data to storage."""
+        pass
+
+    @abstractmethod
+    async def load(self, key: str) -> Optional[bytes]:
+        """Load data from storage."""
+        pass
+
+    @abstractmethod
+    async def delete(self, key: str) -> None:
+        """Delete data from storage."""
+        pass
+
+    @abstractmethod
+    async def list_keys(self, prefix: str = "") -> List[str]:
+        """List keys with optional prefix."""
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Close storage connection."""
+        pass
+
+
+class RedisStorage(StorageBackend):
+    """Redis-based storage backend."""
+
+    def __init__(
+        self,
+        host: str = "localhost",
+        port: int = 6379,
+        db: int = 0,
+        password: Optional[str] = None,
+        key_prefix: str = "kailash:",
+    ):
+        self.host = host
+        self.port = port
+        self.db = db
+        self.password = password
+        self.key_prefix = key_prefix
+        self._redis: Optional[redis.Redis] = None
+
+    async def _get_redis(self) -> redis.Redis:
+        """Get Redis connection."""
+        if self._redis is None:
+            self._redis = await redis.from_url(
+                f"redis://{self.host}:{self.port}",
+                db=self.db,
+                password=self.password,
+                decode_responses=False,
+            )
+        return self._redis
+
+    def _make_key(self, key: str) -> str:
+        """Create prefixed key."""
+        return f"{self.key_prefix}{key}"
+
+    async def save(self, key: str, data: bytes) -> None:
+        """Save data to Redis."""
+        redis = await self._get_redis()
+        await redis.set(self._make_key(key), data)
+
+    async def load(self, key: str) -> Optional[bytes]:
+        """Load data from Redis."""
+        redis = await self._get_redis()
+        return await redis.get(self._make_key(key))
+
+    async def delete(self, key: str) -> None:
+        """Delete data from Redis."""
+        redis = await self._get_redis()
+        await redis.delete(self._make_key(key))
+
+    async def list_keys(self, prefix: str = "") -> List[str]:
+        """List keys with prefix."""
+        redis = await self._get_redis()
+        pattern = self._make_key(f"{prefix}*")
+        keys = await redis.keys(pattern)
+
+        # Remove prefix from keys
+        prefix_len = len(self.key_prefix)
+        return [key.decode()[prefix_len:] for key in keys]
+
+    async def append(self, key: str, data: List[Dict[str, Any]]) -> None:
+        """Append data to a Redis list."""
+        redis = await self._get_redis()
+        serialized_data = [json.dumps(item) for item in data]
+        await redis.lpush(self._make_key(key), *serialized_data)
+
+    async def get(self, key: str) -> List[Dict[str, Any]]:
+        """Get data from Redis list."""
+        redis = await self._get_redis()
+        data = await redis.lrange(self._make_key(key), 0, -1)
+        return [json.loads(item) for item in reversed(data)]
+
+    async def close(self) -> None:
+        """Close Redis connection."""
+        if self._redis:
+            await self._redis.close()
+            self._redis = None
+
+
+class PostgreSQLStorage(StorageBackend):
+    """PostgreSQL-based storage backend."""
+
+    def __init__(
+        self,
+        host: str = "localhost",
+        port: int = 5432,
+        database: str = "kailash",
+        username: str = "postgres",
+        password: str = "",
+        table_name: str = "storage",
+    ):
+        self.host = host
+        self.port = port
+        self.database = database
+        self.username = username
+        self.password = password
+        self.table_name = table_name
+        self._pool: Optional[asyncpg.Pool] = None
+
+    async def _get_pool(self) -> asyncpg.Pool:
+        """Get PostgreSQL connection pool."""
+        if self._pool is None:
+            self._pool = await asyncpg.create_pool(
+                host=self.host,
+                port=self.port,
+                database=self.database,
+                user=self.username,
+                password=self.password,
+            )
+            await self._ensure_table()
+        return self._pool
+
+    async def _ensure_table(self) -> None:
+        """Ensure storage table exists."""
+        pool = self._pool
+        async with pool.acquire() as conn:
+            await conn.execute(
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.table_name} (
+                    key VARCHAR PRIMARY KEY,
+                    data BYTEA NOT NULL,
+                    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+                    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+                )
+            """
+            )
+
+    async def save(self, key: str, data: bytes) -> None:
+        """Save data to PostgreSQL."""
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute(
+                f"""
+                INSERT INTO {self.table_name} (key, data, updated_at)
+                VALUES ($1, $2, NOW())
+                ON CONFLICT (key) DO UPDATE SET
+                    data = EXCLUDED.data,
+                    updated_at = NOW()
+                """,
+                key,
+                data,
+            )
+
+    async def load(self, key: str) -> Optional[bytes]:
+        """Load data from PostgreSQL."""
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            row = await conn.fetchrow(
+                f"SELECT data FROM {self.table_name} WHERE key = $1", key
+            )
+            return row["data"] if row else None
+
+    async def delete(self, key: str) -> None:
+        """Delete data from PostgreSQL."""
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute(f"DELETE FROM {self.table_name} WHERE key = $1", key)
+
+    async def list_keys(self, prefix: str = "") -> List[str]:
+        """List keys with prefix."""
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            rows = await conn.fetch(
+                f"SELECT key FROM {self.table_name} WHERE key LIKE $1",
+                f"{prefix}%",
+            )
+            return [row["key"] for row in rows]
+
+    async def close(self) -> None:
+        """Close PostgreSQL connection pool."""
+        if self._pool:
+            await self._pool.close()
+            self._pool = None
+
+
+class RedisEventStorage:
+    """Redis-based event storage for EventStore."""
+
+    def __init__(
+        self,
+        host: str = "localhost",
+        port: int = 6379,
+        db: int = 0,
+        password: Optional[str] = None,
+        key_prefix: str = "events:",
+    ):
+        self.host = host
+        self.port = port
+        self.db = db
+        self.password = password
+        self.key_prefix = key_prefix
+        self._redis: Optional[redis.Redis] = None
+
+    async def _get_redis(self) -> redis.Redis:
+        """Get Redis connection."""
+        if self._redis is None:
+            self._redis = await redis.from_url(
+                f"redis://{self.host}:{self.port}",
+                db=self.db,
+                password=self.password,
+                decode_responses=False,
+            )
+        return self._redis
+
+    def _make_key(self, key: str) -> str:
+        """Create prefixed key."""
+        return f"{self.key_prefix}{key}"
+
+    async def append(self, key: str, events: List[Dict[str, Any]]) -> None:
+        """Append events to Redis list."""
+        redis = await self._get_redis()
+        serialized_events = [json.dumps(event) for event in events]
+        await redis.lpush(self._make_key(key), *serialized_events)
+
+    async def get(self, key: str) -> List[Dict[str, Any]]:
+        """Get events from Redis list."""
+        redis = await self._get_redis()
+        data = await redis.lrange(self._make_key(key), 0, -1)
+        return [json.loads(item) for item in reversed(data)]
+
+    async def close(self) -> None:
+        """Close Redis connection."""
+        if self._redis:
+            await self._redis.close()
+            self._redis = None
+
+
+class PostgreSQLEventStorage:
+    """PostgreSQL-based event storage for EventStore."""
+
+    def __init__(
+        self,
+        host: str = "localhost",
+        port: int = 5432,
+        database: str = "kailash",
+        username: str = "postgres",
+        password: str = "",
+        table_name: str = "events",
+    ):
+        self.host = host
+        self.port = port
+        self.database = database
+        self.username = username
+        self.password = password
+        self.table_name = table_name
+        self._pool: Optional[asyncpg.Pool] = None
+
+    async def _get_pool(self) -> asyncpg.Pool:
+        """Get PostgreSQL connection pool."""
+        if self._pool is None:
+            self._pool = await asyncpg.create_pool(
+                host=self.host,
+                port=self.port,
+                database=self.database,
+                user=self.username,
+                password=self.password,
+            )
+            await self._ensure_table()
+        return self._pool
+
+    async def _ensure_table(self) -> None:
+        """Ensure events table exists."""
+        pool = self._pool
+        async with pool.acquire() as conn:
+            await conn.execute(
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.table_name} (
+                    id SERIAL PRIMARY KEY,
+                    request_id VARCHAR NOT NULL,
+                    event_id VARCHAR NOT NULL,
+                    event_type VARCHAR NOT NULL,
+                    sequence_number INTEGER NOT NULL,
+                    timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
+                    data JSONB,
+                    metadata JSONB,
+                    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+                )
+            """
+            )
+
+            await conn.execute(
+                f"""
+                CREATE INDEX IF NOT EXISTS idx_{self.table_name}_request_id
+                ON {self.table_name} (request_id)
+            """
+            )
+
+    async def append(self, key: str, events: List[Dict[str, Any]]) -> None:
+        """Append events to PostgreSQL."""
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            for event in events:
+                await conn.execute(
+                    f"""
+                    INSERT INTO {self.table_name}
+                    (request_id, event_id, event_type, sequence_number, timestamp, data, metadata)
+                    VALUES ($1, $2, $3, $4, $5, $6, $7)
+                    """,
+                    event["request_id"],
+                    event["event_id"],
+                    event["event_type"],
+                    event["sequence_number"],
+                    datetime.fromisoformat(event["timestamp"]),
+                    json.dumps(event["data"]),
+                    json.dumps(event["metadata"]),
+                )
+
+    async def get(self, key: str) -> List[Dict[str, Any]]:
+        """Get events from PostgreSQL."""
+        # Extract request_id from key (format: "events:request_id")
+        request_id = key.split(":", 1)[1] if ":" in key else key
+
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            rows = await conn.fetch(
+                f"""
+                SELECT event_id, event_type, request_id, sequence_number,
+                       timestamp, data, metadata
+                FROM {self.table_name}
+                WHERE request_id = $1
+                ORDER BY sequence_number
+                """,
+                request_id,
+            )
+
+            events = []
+            for row in rows:
+                events.append(
+                    {
+                        "event_id": row["event_id"],
+                        "event_type": row["event_type"],
+                        "request_id": row["request_id"],
+                        "sequence_number": row["sequence_number"],
+                        "timestamp": row["timestamp"].isoformat(),
+                        "data": json.loads(row["data"]) if row["data"] else {},
+                        "metadata": (
+                            json.loads(row["metadata"]) if row["metadata"] else {}
+                        ),
+                    }
+                )
+
+            return events
+
+    async def close(self) -> None:
+        """Close PostgreSQL connection pool."""
+        if self._pool:
+            await self._pool.close()
+            self._pool = None

kailash/nexus/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Nexus CLI module for command-line workflow interaction."""
+
+from .main import main
+
+__all__ = ["main"]

kailash/nexus/cli/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for Nexus CLI when run as a module."""
+
+from .main import main
+
+if __name__ == "__main__":
+    main()

kailash/nexus/cli/main.py

@@ -0,0 +1,176 @@
+"""Main CLI interface for Nexus workflow orchestration.
+
+This module provides command-line access to Nexus workflows running on a server.
+It connects to a running Nexus instance and allows listing and executing workflows.
+"""
+
+import argparse
+import json
+import sys
+from typing import Any, Dict, Optional
+
+import requests
+
+
+class NexusCLI:
+    """Command-line interface for Nexus workflows."""
+
+    def __init__(self, base_url: str = "http://localhost:8000"):
+        """Initialize CLI with Nexus server URL.
+
+        Args:
+            base_url: Base URL of the Nexus server
+        """
+        self.base_url = base_url.rstrip("/")
+
+    def list_workflows(self) -> None:
+        """List all available workflows."""
+        try:
+            response = requests.get(f"{self.base_url}/workflows", timeout=5)
+            response.raise_for_status()
+
+            workflows = response.json()
+
+            if not workflows:
+                print("No workflows available.")
+                return
+
+            print("Available workflows:")
+            for workflow_name in sorted(workflows.keys()):
+                print(f"  - {workflow_name}")
+
+        except requests.RequestException as e:
+            print(f"Error connecting to Nexus server: {e}", file=sys.stderr)
+            sys.exit(1)
+        except json.JSONDecodeError as e:
+            print(f"Error parsing server response: {e}", file=sys.stderr)
+            sys.exit(1)
+
+    def run_workflow(
+        self, workflow_name: str, parameters: Optional[Dict[str, Any]] = None
+    ) -> None:
+        """Execute a workflow with optional parameters.
+
+        Args:
+            workflow_name: Name of the workflow to execute
+            parameters: Optional parameters for the workflow
+        """
+        try:
+            payload = {"parameters": parameters or {}}
+
+            response = requests.post(
+                f"{self.base_url}/workflows/{workflow_name}", json=payload, timeout=30
+            )
+            response.raise_for_status()
+
+            result = response.json()
+
+            # Handle enterprise workflow execution format
+            if "outputs" in result:
+                # Extract results from each node
+                for node_name, node_result in result["outputs"].items():
+                    if "result" in node_result:
+                        node_output = node_result["result"]
+                        # Print meaningful output
+                        for key, value in node_output.items():
+                            print(f"{key}: {value}")
+            else:
+                # Handle direct result format
+                print(json.dumps(result, indent=2))
+
+        except requests.RequestException as e:
+            print(f"Error executing workflow: {e}", file=sys.stderr)
+            sys.exit(1)
+        except json.JSONDecodeError as e:
+            print(f"Error parsing execution result: {e}", file=sys.stderr)
+            sys.exit(1)
+
+    def parse_parameters(self, param_strings: list) -> Dict[str, Any]:
+        """Parse parameter strings in key=value format.
+
+        Args:
+            param_strings: List of parameter strings in "key=value" format
+
+        Returns:
+            Dictionary of parsed parameters
+        """
+        parameters = {}
+
+        for param_str in param_strings:
+            if "=" not in param_str:
+                print(
+                    f"Invalid parameter format: {param_str}. Use key=value format.",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+
+            key, value = param_str.split("=", 1)
+
+            # Try to parse as JSON for complex values, otherwise use as string
+            try:
+                parameters[key] = json.loads(value)
+            except json.JSONDecodeError:
+                parameters[key] = value
+
+        return parameters
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Nexus CLI - Command-line interface for workflow orchestration",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  python -m kailash.nexus.cli list
+  python -m kailash.nexus.cli run my-workflow
+  python -m kailash.nexus.cli run my-workflow --param name=value --param count=5
+
+  # Connect to different server:
+  python -m kailash.nexus.cli --url http://localhost:8001 list
+        """,
+    )
+
+    parser.add_argument(
+        "--url",
+        default="http://localhost:8000",
+        help="Base URL of the Nexus server (default: http://localhost:8000)",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # List command
+    list_parser = subparsers.add_parser("list", help="List available workflows")
+
+    # Run command
+    run_parser = subparsers.add_parser("run", help="Execute a workflow")
+    run_parser.add_argument("workflow", help="Name of the workflow to execute")
+    run_parser.add_argument(
+        "--param",
+        action="append",
+        default=[],
+        help="Workflow parameters in key=value format (can be used multiple times)",
+    )
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    # Initialize CLI client
+    cli = NexusCLI(base_url=args.url)
+
+    # Execute command
+    if args.command == "list":
+        cli.list_workflows()
+    elif args.command == "run":
+        parameters = cli.parse_parameters(args.param)
+        cli.run_workflow(args.workflow, parameters)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

kailash/nodes/__init__.py

@@ -1,7 +1,11 @@
 """Node system for the Kailash SDK."""
 
-# Import all node modules to ensure registration
-from kailash.nodes import (
+# Import all node modules to ensure registration - fixed circular import
+from kailash.nodes.base import Node, NodeParameter, NodeRegistry, register_node
+from kailash.nodes.base_cycle_aware import CycleAwareNode
+from kailash.nodes.code import PythonCodeNode
+
+from . import (
     ai,
     alerts,
     api,
@@ -19,9 +23,6 @@ from kailash.nodes import (
     transaction,
     transform,
 )
-from kailash.nodes.base import Node, NodeParameter, NodeRegistry, register_node
-from kailash.nodes.base_cycle_aware import CycleAwareNode
-from kailash.nodes.code import PythonCodeNode
 
 # Compatibility alias - AsyncNode is now just Node
 AsyncNode = Node

kailash/nodes/base.py

@@ -1427,11 +1427,35 @@ class NodeRegistry:
             - Factory methods: Dynamic node creation
         """
         if node_name not in cls._nodes:
-            available_nodes = ", ".join(sorted(cls._nodes.keys()))
-            raise NodeConfigurationError(
-                f"Node '{node_name}' not found in registry. "
-                f"Available nodes: {available_nodes}"
-            )
+            available_nodes = sorted(cls._nodes.keys())
+
+            # Provide more helpful error message based on registry state
+            if not available_nodes:
+                # Registry is empty - likely a test isolation or import issue
+                raise NodeConfigurationError(
+                    f"Node '{node_name}' not found in registry. "
+                    f"The node registry is empty. This usually means:\n"
+                    f"  1. Nodes haven't been imported yet (try: import kailash.nodes)\n"
+                    f"  2. Test isolation cleared the registry without re-importing\n"
+                    f"  3. The node module containing '{node_name}' wasn't imported\n"
+                    f"Common nodes: PythonCodeNode, CSVReaderNode, SQLDatabaseNode, HTTPRequestNode"
+                )
+            else:
+                # Registry has nodes but not the requested one
+                nodes_list = ", ".join(available_nodes)
+
+                # Try to suggest similar node names
+                similar = [
+                    n
+                    for n in available_nodes
+                    if node_name.lower() in n.lower() or n.lower() in node_name.lower()
+                ]
+                suggestion = f"\nDid you mean: {', '.join(similar)}?" if similar else ""
+
+                raise NodeConfigurationError(
+                    f"Node '{node_name}' not found in registry. "
+                    f"Available nodes: {nodes_list}{suggestion}"
+                )
         return cls._nodes[node_name]
 
     @classmethod