kailash 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

kailash/api/custom_nodes_secure.py

@@ -350,7 +350,7 @@ async def _execute_api_node(
 ) -> Dict[str, Any]:
     """Execute an API-based custom node"""
 
-    from kailash.nodes.api.http import HTTPClientNode
+    from kailash.nodes.api.http import HTTPRequestNode
     from kailash.security import TenantContext
 
     # Execute in tenant context
@@ -359,7 +359,7 @@ async def _execute_api_node(
         api_config = node.implementation.get("api", {})
 
         # Create HTTP client node
-        http_node = HTTPClientNode(
+        http_node = HTTPRequestNode(
             url=api_config.get("url", ""),
             method=api_config.get("method", "GET"),
             headers=api_config.get("headers", {}),

kailash/api/studio_secure.py

@@ -369,7 +369,7 @@ class WorkflowStudioAPI:
                                 ),
                             }
                         )
-                    except:
+                    except Exception:
                         outputs.append({"name": "output", "type": "any"})
                 else:
                     # Default output for all nodes

kailash/mcp/client_new.py

@@ -292,7 +292,7 @@ class MCPClient:
 
 # Convenience functions for LLM agents
 async def discover_and_prepare_tools(
-    mcp_servers: List[Union[str, Dict[str, Any]]]
+    mcp_servers: List[Union[str, Dict[str, Any]]],
 ) -> List[Dict[str, Any]]:
     """
     Discover tools from multiple MCP servers and prepare them for LLM use.

kailash/nodes/ai/a2a.py

@@ -832,7 +832,7 @@ Focus on actionable intelligence rather than just listing what each agent said."
                     summary = result.get("response", {}).get("content", "")
                     if summary:
                         return f"Shared Context Summary:\n{summary}"
-        except:
+        except Exception:
             pass
 
         # Fallback to simple summary

kailash/nodes/api/__init__.py

@@ -18,6 +18,8 @@ Design philosophy:
 - Enable both synchronous and asynchronous operation
 """
 
+import warnings
+
 from .auth import APIKeyNode, BasicAuthNode, OAuth2Node
 from .graphql import AsyncGraphQLClientNode, GraphQLClientNode
 from .http import AsyncHTTPRequestNode, HTTPRequestNode
@@ -32,6 +34,23 @@ from .rate_limiting import (
 )
 from .rest import AsyncRESTClientNode, RESTClientNode
 
+# Backwards compatibility aliases
+HTTPClientNode = HTTPRequestNode  # Deprecated: Use HTTPRequestNode instead
+
+
+def __getattr__(name):
+    """Provide deprecation warnings for backwards compatibility."""
+    if name == "HTTPClientNode":
+        warnings.warn(
+            "HTTPClientNode is deprecated and will be removed in v0.3.0. "
+            "Use HTTPRequestNode instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return HTTPRequestNode
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
 __all__ = [
     "HTTPRequestNode",
     "AsyncHTTPRequestNode",
@@ -49,4 +68,6 @@ __all__ = [
     "RateLimitedAPINode",
     "AsyncRateLimitedAPINode",
     "create_rate_limiter",
+    # Backwards compatibility
+    "HTTPClientNode",  # Deprecated alias
 ]

kailash/nodes/code/python.py

@@ -93,6 +93,12 @@ ALLOWED_MODULES = {
     "matplotlib",
     "seaborn",
     "plotly",
+    # File processing modules
+    "csv",  # For CSV file processing
+    "mimetypes",  # For MIME type detection
+    "pathlib",  # For modern path operations
+    "glob",  # For file pattern matching
+    "xml",  # For XML processing
 }
 
 

kailash/nodes/data/__init__.py

@@ -80,6 +80,7 @@ Example Workflows:
     workflow.connect('process', 'publish')
 """
 
+from kailash.nodes.data.directory import DirectoryReaderNode
 from kailash.nodes.data.readers import CSVReaderNode, JSONReaderNode, TextReaderNode
 from kailash.nodes.data.retrieval import RelevanceScorerNode
 from kailash.nodes.data.sharepoint_graph import (
@@ -102,6 +103,8 @@ from kailash.nodes.data.vector_db import (
 from kailash.nodes.data.writers import CSVWriterNode, JSONWriterNode, TextWriterNode
 
 __all__ = [
+    # Directory
+    "DirectoryReaderNode",
     # Readers
     "CSVReaderNode",
     "JSONReaderNode",

kailash/nodes/data/directory.py

@@ -0,0 +1,278 @@
+"""Directory processing nodes for file discovery and batch operations."""
+
+import mimetypes
+import os
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from kailash.nodes.base import Node, NodeParameter, register_node
+from kailash.security import validate_file_path
+
+
+@register_node()
+class DirectoryReaderNode(Node):
+    """
+    Discovers and catalogs files in a directory with metadata extraction.
+
+    This node provides comprehensive directory scanning capabilities, handling
+    file discovery, metadata extraction, and filtering. It's designed for
+    batch file processing workflows and dynamic data source discovery.
+
+    Design Philosophy:
+        The DirectoryReaderNode embodies the principle of "dynamic data discovery."
+        Instead of hardcoding file paths, workflows can dynamically discover
+        available data sources at runtime. This makes workflows more flexible
+        and adaptable to changing data environments.
+
+    Features:
+        - Recursive directory scanning
+        - File type detection and filtering
+        - Metadata extraction (size, timestamps, MIME types)
+        - Pattern-based filtering
+        - Security-validated path operations
+
+    Use Cases:
+        - Batch file processing workflows
+        - Dynamic data pipeline creation
+        - File monitoring and cataloging
+        - Multi-format document processing
+        - Data lake exploration
+
+    Output Format:
+        Returns a structured catalog of discovered files with:
+        - File paths and names
+        - File types and MIME types
+        - File sizes and timestamps
+        - Directory structure information
+    """
+
+    def get_parameters(self) -> Dict[str, NodeParameter]:
+        """Define input parameters for directory scanning."""
+        return {
+            "directory_path": NodeParameter(
+                name="directory_path",
+                type=str,
+                required=True,
+                description="Path to the directory to scan",
+            ),
+            "recursive": NodeParameter(
+                name="recursive",
+                type=bool,
+                required=False,
+                default=False,
+                description="Whether to scan subdirectories recursively",
+            ),
+            "file_patterns": NodeParameter(
+                name="file_patterns",
+                type=list,
+                required=False,
+                default=[],
+                description="List of file patterns to include (e.g., ['*.csv', '*.json'])",
+            ),
+            "exclude_patterns": NodeParameter(
+                name="exclude_patterns",
+                type=list,
+                required=False,
+                default=[],
+                description="List of file patterns to exclude",
+            ),
+            "include_hidden": NodeParameter(
+                name="include_hidden",
+                type=bool,
+                required=False,
+                default=False,
+                description="Whether to include hidden files (starting with .)",
+            ),
+        }
+
+    def run(self, **kwargs) -> Dict[str, Any]:
+        """Execute directory scanning operation.
+
+        Returns:
+            Dictionary containing:
+            - discovered_files: List of file information dictionaries
+            - files_by_type: Files grouped by type
+            - directory_stats: Summary statistics
+        """
+        directory_path = kwargs.get("directory_path")
+        recursive = kwargs.get("recursive", False)
+        file_patterns = kwargs.get("file_patterns", [])
+        exclude_patterns = kwargs.get("exclude_patterns", [])
+        include_hidden = kwargs.get("include_hidden", False)
+
+        # Validate directory path for security
+        validated_path = validate_file_path(directory_path, operation="directory scan")
+
+        if not os.path.isdir(validated_path):
+            raise FileNotFoundError(f"Directory not found: {directory_path}")
+
+        discovered_files = []
+
+        try:
+            if recursive:
+                # Recursive scan
+                for root, dirs, files in os.walk(validated_path):
+                    for filename in files:
+                        file_path = os.path.join(root, filename)
+                        file_info = self._extract_file_info(
+                            file_path,
+                            filename,
+                            include_hidden,
+                            file_patterns,
+                            exclude_patterns,
+                        )
+                        if file_info:
+                            discovered_files.append(file_info)
+            else:
+                # Single directory scan
+                for filename in os.listdir(validated_path):
+                    file_path = os.path.join(validated_path, filename)
+
+                    # Skip directories in non-recursive mode
+                    if os.path.isdir(file_path):
+                        continue
+
+                    file_info = self._extract_file_info(
+                        file_path,
+                        filename,
+                        include_hidden,
+                        file_patterns,
+                        exclude_patterns,
+                    )
+                    if file_info:
+                        discovered_files.append(file_info)
+
+        except PermissionError as e:
+            raise PermissionError(f"Permission denied accessing directory: {e}")
+        except Exception as e:
+            raise RuntimeError(f"Error scanning directory: {e}")
+
+        # Group files by type
+        files_by_type = {}
+        for file_info in discovered_files:
+            file_type = file_info["file_type"]
+            if file_type not in files_by_type:
+                files_by_type[file_type] = []
+            files_by_type[file_type].append(file_info)
+
+        # Generate directory statistics
+        directory_stats = {
+            "total_files": len(discovered_files),
+            "file_types": list(files_by_type.keys()),
+            "files_by_type_count": {
+                file_type: len(files) for file_type, files in files_by_type.items()
+            },
+            "total_size": sum(f["file_size"] for f in discovered_files),
+            "scan_time": datetime.now().isoformat(),
+            "directory_path": directory_path,
+            "recursive": recursive,
+        }
+
+        return {
+            "discovered_files": discovered_files,
+            "files_by_type": files_by_type,
+            "directory_stats": directory_stats,
+        }
+
+    def _extract_file_info(
+        self,
+        file_path: str,
+        filename: str,
+        include_hidden: bool,
+        file_patterns: List[str],
+        exclude_patterns: List[str],
+    ) -> Optional[Dict[str, Any]]:
+        """Extract metadata from a single file.
+
+        Args:
+            file_path: Full path to the file
+            filename: Name of the file
+            include_hidden: Whether to include hidden files
+            file_patterns: Patterns to include
+            exclude_patterns: Patterns to exclude
+
+        Returns:
+            File information dictionary or None if file should be excluded
+        """
+        # Skip hidden files if not included
+        if not include_hidden and filename.startswith("."):
+            return None
+
+        # Check exclude patterns
+        for pattern in exclude_patterns:
+            if self._matches_pattern(filename, pattern):
+                return None
+
+        # Check include patterns (if specified)
+        if file_patterns:
+            included = any(
+                self._matches_pattern(filename, pattern) for pattern in file_patterns
+            )
+            if not included:
+                return None
+
+        try:
+            # Get file statistics
+            file_stat = os.stat(file_path)
+            file_ext = os.path.splitext(filename)[1].lower()
+
+            # Map extensions to types
+            ext_to_type = {
+                ".csv": "csv",
+                ".json": "json",
+                ".txt": "txt",
+                ".xml": "xml",
+                ".md": "markdown",
+                ".py": "python",
+                ".js": "javascript",
+                ".html": "html",
+                ".css": "css",
+                ".pdf": "pdf",
+                ".doc": "word",
+                ".docx": "word",
+                ".xls": "excel",
+                ".xlsx": "excel",
+                ".png": "image",
+                ".jpg": "image",
+                ".jpeg": "image",
+                ".gif": "image",
+                ".svg": "image",
+            }
+
+            file_type = ext_to_type.get(file_ext, "unknown")
+
+            # Get MIME type
+            mime_type, _ = mimetypes.guess_type(file_path)
+            if not mime_type:
+                mime_type = "application/octet-stream"
+
+            return {
+                "file_path": file_path,
+                "file_name": filename,
+                "file_type": file_type,
+                "file_extension": file_ext,
+                "file_size": file_stat.st_size,
+                "mime_type": mime_type,
+                "created_time": datetime.fromtimestamp(file_stat.st_ctime).isoformat(),
+                "modified_time": datetime.fromtimestamp(file_stat.st_mtime).isoformat(),
+                "discovered_at": datetime.now().isoformat(),
+            }
+
+        except (OSError, PermissionError) as e:
+            # Log error but continue with other files
+            self.logger.warning(f"Could not process file {file_path}: {e}")
+            return None
+
+    def _matches_pattern(self, filename: str, pattern: str) -> bool:
+        """Check if filename matches a glob-style pattern.
+
+        Args:
+            filename: Name of the file to check
+            pattern: Glob pattern (e.g., '*.csv', 'data*', 'file?.txt')
+
+        Returns:
+            True if filename matches pattern
+        """
+        import fnmatch
+
+        return fnmatch.fnmatch(filename, pattern)

kailash/nodes/data/sql.py

@@ -166,10 +166,10 @@ class SQLDatabaseNode(Node):
     Example:
         >>> # Initialize with project configuration
         >>> SQLDatabaseNode.initialize('kailash_project.yaml')
-        >>> 
+        >>>
         >>> # Create node with database connection configuration
         >>> sql_node = SQLDatabaseNode(connection='customer_db')
-        >>> 
+        >>>
         >>> # Execute multiple queries with the same node
         >>> result1 = sql_node.run(
         ...     query='SELECT * FROM customers WHERE active = ?',

kailash/nodes/transform/processors.py

@@ -332,12 +332,35 @@ class DataTransformer(Node):
             },  # Support for up to 5 additional arguments
         }
 
+    def validate_inputs(self, **kwargs) -> Dict[str, Any]:
+        """Override validate_inputs to accept arbitrary parameters for transformations.
+
+        DataTransformer needs to accept any input parameters that might be mapped
+        from other nodes, not just the predefined parameters in get_parameters().
+        This enables flexible data flow in workflows.
+        """
+        # First, do the standard validation for defined parameters
+        validated = super().validate_inputs(**kwargs)
+
+        # Then, add any extra parameters that aren't in the schema
+        # These will be passed to the transformation context
+        defined_params = set(self.get_parameters().keys())
+        for key, value in kwargs.items():
+            if key not in defined_params:
+                validated[key] = value  # Accept arbitrary additional parameters
+
+        return validated
+
     def run(self, **kwargs) -> Dict[str, Any]:
         # Extract the transformation functions
         transformations = kwargs.get("transformations", [])
         if not transformations:
             return {"result": kwargs.get("data", [])}
 
+        # Debug: Check what kwargs we received
+        print(f"DATATRANSFORMER RUN DEBUG: kwargs keys = {list(kwargs.keys())}")
+        print(f"DATATRANSFORMER RUN DEBUG: kwargs = {kwargs}")
+
         # Get all input data
         input_data = {}
         for key, value in kwargs.items():
@@ -371,6 +394,14 @@ class DataTransformer(Node):
                     local_vars = input_data.copy()
                     local_vars["result"] = result
 
+                    # Debug: Print available variables
+                    print(
+                        f"DataTransformer DEBUG - Available variables: {list(local_vars.keys())}"
+                    )
+                    print(
+                        f"DataTransformer DEBUG - Input data keys: {list(input_data.keys())}"
+                    )
+
                     # Execute the code block
                     exec(transform_str, safe_globals, local_vars)  # noqa: S102
 

kailash/runtime/local.py

@@ -399,8 +399,13 @@ class LocalRuntime:
             source_node_id = edge[0]
             mapping = edge[2].get("mapping", {})
 
+            print(f"LOCAL RUNTIME DEBUG: Processing edge {source_node_id} -> {node_id}")
+            print(f"  Edge data: {edge[2]}")
+            print(f"  Mapping: {mapping}")
+
             if source_node_id in node_outputs:
                 source_outputs = node_outputs[source_node_id]
+                print(f"  Source outputs: {list(source_outputs.keys())}")
 
                 # Check if the source node failed
                 if isinstance(source_outputs, dict) and source_outputs.get("failed"):
@@ -411,11 +416,19 @@ class LocalRuntime:
                 for source_key, target_key in mapping.items():
                     if source_key in source_outputs:
                         inputs[target_key] = source_outputs[source_key]
+                        print(
+                            f"  MAPPED: {source_key} -> {target_key} (type: {type(source_outputs[source_key])})"
+                        )
                     else:
+                        print(
+                            f"  MISSING: {source_key} not in {list(source_outputs.keys())}"
+                        )
                         self.logger.warning(
                             f"Source output '{source_key}' not found in node '{source_node_id}'. "
                             f"Available outputs: {list(source_outputs.keys())}"
                         )
+            else:
+                print(f"  No outputs found for source node {source_node_id}")
 
         # Apply parameter overrides
         inputs.update(parameters)

kailash/workflow/convergence.py

@@ -229,7 +229,7 @@ class AdaptiveCondition(ConvergenceCondition):
 
 
 def create_convergence_condition(
-    spec: Union[str, int, Callable, Dict]
+    spec: Union[str, int, Callable, Dict],
 ) -> ConvergenceCondition:
     """Factory function to create convergence conditions from various specs.
 

kailash/workflow/graph.py

@@ -928,6 +928,14 @@ class Workflow:
                     to_input = edge_data.get("to_input")
                     mapping = edge_data.get("mapping", {})
 
+                    print(f"CONNECTION DEBUG: {source_node_id} -> {node_id}")
+                    print(f"  Edge data: {edge_data}")
+                    print(f"  from_output: {from_output}, to_input: {to_input}")
+                    print(f"  mapping: {mapping}")
+                    print(
+                        f"  source_results keys: {list(results.get(source_node_id, {}).keys())}"
+                    )
+
                     source_results = results.get(source_node_id, {})
 
                     # Handle backward compatibility - from_output/to_input can be string or list
@@ -951,6 +959,13 @@ class Workflow:
                     for source_key, target_key in mapping.items():
                         if source_key in source_results:
                             node_inputs[target_key] = source_results[source_key]
+                            print(
+                                f"MAPPING DEBUG: {source_key} -> {target_key}, value type: {type(source_results[source_key])}"
+                            )
+                        else:
+                            print(
+                                f"MAPPING DEBUG: Source key '{source_key}' not found in source results: {list(source_results.keys())}"
+                            )
 
                 # Apply overrides
                 node_overrides = inputs.get(node_id, {})

kailash-0.2.2.dist-info/METADATA

@@ -0,0 +1,121 @@
+Metadata-Version: 2.4
+Name: kailash
+Version: 0.2.2
+Summary: Python SDK for the Kailash container-node architecture
+Home-page: https://github.com/integrum/kailash-python-sdk
+Author: Integrum
+Author-email: Integrum <info@integrum.com>
+Project-URL: Homepage, https://github.com/integrum/kailash-python-sdk
+Project-URL: Bug Tracker, https://github.com/integrum/kailash-python-sdk/issues
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: networkx>=2.7
+Requires-Dist: pydantic>=1.9
+Requires-Dist: matplotlib>=3.5
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: click>=8.0
+Requires-Dist: pytest>=8.3.5
+Requires-Dist: mcp[cli]>=1.9.2
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: numpy>=2.2.5
+Requires-Dist: scipy>=1.15.3
+Requires-Dist: scikit-learn>=1.6.1
+Requires-Dist: requests>=2.32.3
+Requires-Dist: pytest-cov>=6.1.1
+Requires-Dist: isort>=6.0.1
+Requires-Dist: aiohttp>=3.12.4
+Requires-Dist: ruff>=0.11.12
+Requires-Dist: msal>=1.32.3
+Requires-Dist: sphinx>=8.2.3
+Requires-Dist: sphinx-rtd-theme>=3.0.2
+Requires-Dist: sphinx-copybutton>=0.5.2
+Requires-Dist: sphinxcontrib-mermaid>=1.0.0
+Requires-Dist: sphinx-autobuild>=2024.10.3
+Requires-Dist: autodoc>=0.5.0
+Requires-Dist: myst-parser>=4.0.1
+Requires-Dist: black>=25.1.0
+Requires-Dist: psutil>=7.0.0
+Requires-Dist: fastapi>=0.115.12
+Requires-Dist: uvicorn[standard]>=0.31.0
+Requires-Dist: pytest-asyncio>=1.0.0
+Requires-Dist: pre-commit>=4.2.0
+Requires-Dist: twine>=6.1.0
+Requires-Dist: ollama>=0.5.1
+Requires-Dist: sqlalchemy>=2.0.0
+Requires-Dist: psycopg2-binary>=2.9.0
+Requires-Dist: pymysql>=1.1.0
+Requires-Dist: aiosqlite>=0.19.0
+Requires-Dist: websockets>=12.0
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: python-jose>=3.5.0
+Requires-Dist: pytest-xdist>=3.6.0
+Requires-Dist: pytest-timeout>=2.3.0
+Requires-Dist: pytest-split>=0.9.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=3.0; extra == "dev"
+Requires-Dist: black>=22.0; extra == "dev"
+Requires-Dist: isort>=5.10; extra == "dev"
+Requires-Dist: mypy>=0.9; extra == "dev"
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# Kailash Python SDK - Internal Development Guide
+
+This directory contains internal documentation for developers and contributors to the Kailash Python SDK. These documents are not included in the PyPI distribution and are only accessible to those with access to this private repository.
+
+## Directory Structure
+
+### Core Directories (with README.md):
+- **`adr/`** - Architecture Decision Records (35+ design decisions)
+- **`features/`** - In-depth feature implementation guides
+- **`reference/`** - LLM-optimized API references and patterns
+- **`instructions/`** - Detailed coding and documentation standards
+- **`frontend/`** - Frontend development guide
+- **`workflows/`** - Development workflows and task checklists
+
+### Additional Directories:
+- **`development/`** - SDK development guides and tools
+  - **`custom-nodes/`** - Comprehensive custom node development guide (parameter types, examples, troubleshooting)
+  - **`pre-commit-hooks.md`** - Development workflow automation
+- **`infrastructure/`** - CI/CD and runner configuration
+- **`mistakes/`** - Documented mistakes and lessons learned (73+ issues, including critical v0.2.1 base node fixes)
+- **`prd/`** - Product Requirements Documents
+- **`todos/`** - Active task tracking system
+- **`SECURITY.md`** - Comprehensive security documentation
+
+### In project root:
+- **`CLAUDE.md`** - Compact LLM quick reference (optimized navigation)
+
+## Important Notes
+
+1. **Private Documentation**: All content in this directory is considered internal and should not be shared publicly.
+
+2. **Not Distributed**: These files are explicitly excluded from PyPI packages via `MANIFEST.in`.
+
+3. **Development Reference**: Use these documents to understand design decisions, development patterns, and project history.
+
+## For Contributors
+
+When contributing to the project:
+1. Review `CLAUDE.md` for coding standards and conventions
+2. Check ADRs for architectural decisions
+3. Consult PRDs for product requirements
+4. Learn from documented mistakes to avoid common pitfalls
+5. Track tasks using the todos system
+6. **Creating custom nodes?** See `development/custom-nodes/` for critical parameter type constraints
+
+## Accessing Documentation
+
+These documents are only available when:
+- Cloning the repository directly from GitHub
+- Having access to the private repository
+- Working on development (not from PyPI installation)