kailash 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

kailash/nodes/data/readers.py

@@ -33,63 +33,117 @@ import json
 from typing import Any, Dict
 
 from kailash.nodes.base import Node, NodeParameter, register_node
+from kailash.security import safe_open, validate_file_path
 
 
 @register_node()
 class CSVReaderNode(Node):
-    """Reads data from a CSV file.
-
-    This node provides robust CSV file reading capabilities with support for
-    various delimiters, header detection, and encoding options. It's designed
-    to handle common CSV formats and edge cases.
-
-    Design Features:
-    1. Automatic header detection
-    2. Configurable delimiters
-    3. Memory-efficient line-by-line reading
-    4. Consistent dictionary output format
-    5. Unicode support through encoding parameter
-
-    Data Flow:
-    - Input: File path and configuration parameters
-    - Processing: Reads CSV line by line, converting to dictionaries
-    - Output: List of dictionaries (with headers) or list of lists
-
-    Common Usage Patterns:
-    1. Reading data exports from databases
-    2. Processing spreadsheet data
-    3. Loading configuration from CSV
-    4. Ingesting sensor data logs
-
-    Upstream Sources:
-    - File system paths from user input
-    - Output paths from previous nodes
-    - Configuration management systems
+    """
+    Reads data from CSV files with automatic header detection and type inference.
+
+    This node provides comprehensive CSV file reading capabilities, handling various
+    formats, encodings, and edge cases. It automatically detects headers, infers data
+    types, and provides consistent structured output for downstream processing in
+    Kailash workflows.
+
+    Design Philosophy:
+        The CSVReaderNode embodies the principle of "data accessibility without
+        complexity." It abstracts the intricacies of CSV parsing while providing
+        flexibility for various formats. The design prioritizes memory efficiency,
+        automatic format detection, and consistent output structure, making it easy
+        to integrate diverse CSV data sources into workflows.
+
+    Upstream Dependencies:
+        - File system providing CSV files
+        - Workflow orchestrators specifying file paths
+        - Configuration systems providing parsing options
+        - Previous nodes generating CSV file paths
+        - User inputs defining data sources
 
     Downstream Consumers:
-    - DataTransformer: Processes tabular data
-    - Aggregator: Summarizes data
-    - CSVWriter: Reformats and saves
-    - Visualizer: Creates charts from data
+        - DataTransformNode: Processes tabular data
+        - FilterNode: Applies row/column filtering
+        - AggregatorNode: Summarizes data
+        - PythonCodeNode: Custom data processing
+        - WriterNodes: Exports to other formats
+        - Visualization nodes: Creates charts
+        - ML nodes: Uses as training data
+
+    Configuration:
+        The node supports extensive CSV parsing options:
+        - Delimiter detection (comma, tab, pipe, etc.)
+        - Header row identification
+        - Encoding specification (UTF-8, Latin-1, etc.)
+        - Quote character handling
+        - Skip rows/comments functionality
+        - Column type inference
+        - Missing value handling
+
+    Implementation Details:
+        - Uses Python's csv module for robust parsing
+        - Implements streaming for large files
+        - Automatic delimiter detection when not specified
+        - Header detection based on first row analysis
+        - Type inference for numeric/date columns
+        - Memory-efficient processing with generators
+        - Unicode normalization for consistent encoding
 
     Error Handling:
-    - FileNotFoundError: Invalid file path
-    - PermissionError: Insufficient read permissions
-    - UnicodeDecodeError: Encoding mismatch
-    - csv.Error: Malformed CSV data
-
-    Example:
-        >>> # Read customer data with headers
-        >>> reader = CSVReaderNode(
-        ...     file_path='customers.csv',
-        ...     headers=True,
-        ...     delimiter=','
+        - FileNotFoundError: Clear message with path
+        - PermissionError: Access rights guidance
+        - UnicodeDecodeError: Encoding detection hints
+        - csv.Error: Malformed data diagnostics
+        - EmptyFileError: Handles zero-byte files
+        - Partial read recovery for corrupted files
+
+    Side Effects:
+        - Reads from file system
+        - May consume significant memory for large files
+        - Creates file handles (properly closed)
+        - Updates internal read statistics
+
+    Examples:
+        >>> # Basic CSV reading with headers
+        >>> reader = CSVReaderNode()
+        >>> result = reader.run(
+        ...     file_path="customers.csv",
+        ...     headers=True
         ... )
-        >>> result = reader.execute()
-        >>> # result['data'] = [
-        >>> #     {'id': '1', 'name': 'John', 'age': '30'},
-        >>> #     {'id': '2', 'name': 'Jane', 'age': '25'}
+        >>> assert isinstance(result["data"], list)
+        >>> assert all(isinstance(row, dict) for row in result["data"])
+        >>> # Example output:
+        >>> # result["data"] = [
+        >>> #     {"id": "1", "name": "John Doe", "age": "30"},
+        >>> #     {"id": "2", "name": "Jane Smith", "age": "25"}
         >>> # ]
+        >>>
+        >>> # Reading with custom delimiter
+        >>> result = reader.run(
+        ...     file_path="data.tsv",
+        ...     delimiter="\\t",
+        ...     headers=True
+        ... )
+        >>>
+        >>> # Reading without headers (returns list of lists)
+        >>> result = reader.run(
+        ...     file_path="data.csv",
+        ...     headers=False
+        ... )
+        >>> assert all(isinstance(row, list) for row in result["data"])
+        >>>
+        >>> # Reading with specific encoding
+        >>> result = reader.run(
+        ...     file_path="european_data.csv",
+        ...     encoding="iso-8859-1",
+        ...     headers=True
+        ... )
+        >>>
+        >>> # Handling quoted fields
+        >>> result = reader.run(
+        ...     file_path="complex.csv",
+        ...     headers=True,
+        ...     quotechar='"'
+        ... )
     """
 
     def get_parameters(self) -> Dict[str, NodeParameter]:
@@ -192,7 +246,7 @@ class CSVReaderNode(Node):
             - Analyzers can process row-by-row
             - data_indexed is useful for lookups and joins
         """
-        file_path = kwargs["file_path"]
+        file_path = kwargs.get("file_path") or self.config.get("file_path")
         headers = kwargs.get("headers", True)
         delimiter = kwargs.get("delimiter", ",")
         index_column = kwargs.get("index_column")
@@ -200,7 +254,10 @@ class CSVReaderNode(Node):
         data = []
         data_indexed = {}
 
-        with open(file_path, "r", encoding="utf-8") as f:
+        # Validate file path for security
+        validated_path = validate_file_path(file_path, operation="CSV read")
+
+        with safe_open(validated_path, "r", encoding="utf-8") as f:
             reader = csv.reader(f, delimiter=delimiter)
 
             if headers:
@@ -349,9 +406,12 @@ class JSONReaderNode(Node):
             - Compatible with JSONWriter for round-trip
             - Transform nodes can process nested data
         """
-        file_path = kwargs["file_path"]
+        file_path = kwargs.get("file_path") or self.config.get("file_path")
 
-        with open(file_path, "r", encoding="utf-8") as f:
+        # Validate file path for security
+        validated_path = validate_file_path(file_path, operation="JSON read")
+
+        with safe_open(validated_path, "r", encoding="utf-8") as f:
             data = json.load(f)
 
         return {"data": data}
@@ -487,10 +547,13 @@ class TextReaderNode(Node):
             - Pattern nodes can search content
             - Writers can save processed text
         """
-        file_path = kwargs["file_path"]
+        file_path = kwargs.get("file_path") or self.config.get("file_path")
         encoding = kwargs.get("encoding", "utf-8")
 
-        with open(file_path, "r", encoding=encoding) as f:
+        # Validate file path for security
+        validated_path = validate_file_path(file_path, operation="text read")
+
+        with safe_open(validated_path, "r", encoding=encoding) as f:
             text = f.read()
 
         return {"text": text}

kailash/nodes/data/writers.py

@@ -34,6 +34,7 @@ import json
 from typing import Any, Dict
 
 from kailash.nodes.base import Node, NodeParameter, register_node
+from kailash.security import safe_open, validate_file_path
 
 
 @register_node()
@@ -190,7 +191,7 @@ class CSVWriterNode(Node):
             - External tools can process output
             - Metrics available for monitoring
         """
-        file_path = kwargs["file_path"]
+        file_path = kwargs.get("file_path") or self.config.get("file_path")
         data = kwargs["data"]
         headers = kwargs.get("headers")
         delimiter = kwargs.get("delimiter", ",")
@@ -198,7 +199,10 @@ class CSVWriterNode(Node):
         if not data:
             return {"rows_written": 0}
 
-        with open(file_path, "w", newline="", encoding="utf-8") as f:
+        # Validate file path for security
+        validated_path = validate_file_path(file_path, operation="CSV write")
+
+        with safe_open(validated_path, "w", newline="", encoding="utf-8") as f:
             if isinstance(data[0], dict):
                 # Writing dictionaries
                 if not headers:
@@ -357,11 +361,14 @@ class JSONWriterNode(Node):
             - Version control can track
             - APIs can import data
         """
-        file_path = kwargs["file_path"]
+        file_path = kwargs.get("file_path") or self.config.get("file_path")
         data = kwargs["data"]
         indent = kwargs.get("indent", 2)
 
-        with open(file_path, "w", encoding="utf-8") as f:
+        # Validate file path for security
+        validated_path = validate_file_path(file_path, operation="JSON write")
+
+        with safe_open(validated_path, "w", encoding="utf-8") as f:
             json.dump(data, f, indent=indent, ensure_ascii=False)
 
         return {"file_path": file_path}
@@ -517,13 +524,16 @@ class TextWriterNode(Node):
             - Log analyzers can process
             - Metrics available for monitoring
         """
-        file_path = kwargs["file_path"]
+        file_path = kwargs.get("file_path") or self.config.get("file_path")
         text = kwargs["text"]
         encoding = kwargs.get("encoding", "utf-8")
         append = kwargs.get("append", False)
 
         mode = "a" if append else "w"
-        with open(file_path, mode, encoding=encoding) as f:
+        # Validate file path for security
+        validated_path = validate_file_path(file_path, operation="text write")
+
+        with safe_open(validated_path, mode, encoding=encoding) as f:
             f.write(text)
 
         return {"file_path": file_path, "bytes_written": len(text.encode(encoding))}

kailash/nodes/logic/__init__.py

@@ -1,6 +1,11 @@
 """Logic operation nodes for the Kailash SDK."""
 
 from kailash.nodes.logic.async_operations import AsyncMergeNode, AsyncSwitchNode
+from kailash.nodes.logic.convergence import (
+    ConvergenceCheckerNode,
+    MultiCriteriaConvergenceNode,
+)
+from kailash.nodes.logic.loop import LoopNode
 from kailash.nodes.logic.operations import MergeNode, SwitchNode
 from kailash.nodes.logic.workflow import WorkflowNode
 
@@ -10,4 +15,7 @@ __all__ = [
     "AsyncSwitchNode",
     "AsyncMergeNode",
     "WorkflowNode",
+    "LoopNode",
+    "ConvergenceCheckerNode",
+    "MultiCriteriaConvergenceNode",
 ]

kailash/nodes/logic/async_operations.py

@@ -32,15 +32,25 @@ class AsyncMergeNode(AsyncNode):
     concat (list concatenation), zip (parallel iteration), and merge_dict
     (dictionary merging with optional key-based joining).
 
-    Usage example:
-        # Create an AsyncMergeNode in a workflow
-        async_merge = AsyncMergeNode(merge_type="merge_dict", key="id")
-        workflow.add_node("data_combine", async_merge)
-
-        # Connect multiple data sources
-        workflow.connect("api_results", "data_combine", {"output": "data1"})
-        workflow.connect("database_query", "data_combine", {"results": "data2"})
-        workflow.connect("file_processor", "data_combine", {"processed_data": "data3"})
+    Example usage:
+        >>> # Create an AsyncMergeNode
+        >>> async_merge = AsyncMergeNode(merge_type="merge_dict", key="id")
+        >>> async_merge.metadata.name
+        'AsyncMergeNode'
+
+        >>> # Using in a workflow
+        >>> from kailash.workflow.graph import Workflow
+        >>> workflow = Workflow("wf-001", "async_example")
+        >>> workflow.add_node("data_combine", async_merge)
+        >>> "data_combine" in workflow.nodes
+        True
+
+        >>> # Async execution with concat
+        >>> import asyncio
+        >>> async_merge = AsyncMergeNode(merge_type="concat")
+        >>> result = asyncio.run(async_merge.execute_async(data1=[1, 2], data2=[3, 4]))
+        >>> result['merged_data']
+        [1, 2, 3, 4]
     """
 
     def get_parameters(self) -> Dict[str, NodeParameter]:
@@ -364,6 +374,35 @@ class AsyncSwitchNode(AsyncNode):
 
     The basic functionality is the same as the synchronous SwitchNode but optimized
     for asynchronous execution.
+
+    Example usage:
+        >>> # Boolean condition routing
+        >>> import asyncio
+        >>> async_switch = AsyncSwitchNode(
+        ...     condition_field="status",
+        ...     operator="==",
+        ...     value="active"
+        ... )
+        >>> result = asyncio.run(async_switch.execute_async(
+        ...     input_data={"status": "active", "data": "test"}
+        ... ))
+        >>> result['true_output']
+        {'status': 'active', 'data': 'test'}
+        >>> result['false_output'] is None
+        True
+
+        >>> # Multi-case switching
+        >>> async_switch = AsyncSwitchNode(
+        ...     condition_field="priority",
+        ...     cases=["high", "medium", "low"]
+        ... )
+        >>> result = asyncio.run(async_switch.execute_async(
+        ...     input_data={"priority": "high", "task": "urgent"}
+        ... ))
+        >>> result['case_high']
+        {'priority': 'high', 'task': 'urgent'}
+        >>> result['default']
+        {'priority': 'high', 'task': 'urgent'}
     """
 
     def get_parameters(self) -> Dict[str, NodeParameter]: