kailash 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

kailash/runtime/docker.py

@@ -174,7 +174,7 @@ def main():
     logger.info(f"Loaded configuration for {node_data['class']} node")
 
     # Load runtime inputs if available
-    input_path = Path("/examples/data/input/inputs.json")
+    input_path = Path("/data/inputs/json/inputs.json")
     runtime_inputs = {}
     if input_path.exists():
         logger.info(f"Loading inputs from {input_path}")
@@ -206,7 +206,7 @@ def main():
     except Exception as e:
         logger.error(f"Node execution failed: {e}")
         # Save error information
-        with open("/examples/data/output/error.json", 'w') as f:
+        with open("/data/outputs/json/error.json", 'w') as f:
             json.dump({
                 "error": str(e),
                 "type": e.__class__.__name__
@@ -216,7 +216,7 @@ def main():
     # Save results
     logger.info("Saving execution results")
     try:
-        result_path = Path("/examples/data/output/result.json")
+        result_path = Path("/data/outputs/json/result.json")
         with open(result_path, 'w') as f:
             # Handle non-serializable objects with basic conversion
             try:
@@ -590,7 +590,7 @@ class DockerRuntime:
 
         try:
             # Validate workflow
-            workflow.validate()
+            workflow.validate(runtime_parameters=inputs)
 
             # Get execution order
             execution_order = workflow.get_execution_order()

kailash/runtime/local.py

@@ -113,8 +113,8 @@ class LocalRuntime:
         run_id = None
 
         try:
-            # Validate workflow
-            workflow.validate()
+            # Validate workflow with runtime parameters (Session 061)
+            workflow.validate(runtime_parameters=parameters)
 
             # Initialize tracking
             if task_manager:
@@ -296,6 +296,10 @@ class LocalRuntime:
                     parameters=parameters.get(node_id, {}),
                 )
 
+                # Update node config with parameters (Session 061: direct config update)
+                {**node_instance.config, **parameters.get(node_id, {})}
+                node_instance.config.update(parameters.get(node_id, {}))
+
                 if self.debug:
                     self.logger.debug(f"Node {node_id} inputs: {inputs}")
 
@@ -391,16 +395,36 @@ class LocalRuntime:
         """
         inputs = {}
 
-        # Start with node configuration
-        inputs.update(node_instance.config)
+        # NOTE: Node configuration is handled separately in configure() call
+        # Only add runtime inputs and data from connected nodes here
+
+        # Add runtime parameters (those not used for node configuration)
+        # Map specific runtime parameters for known node types
+        if "consumer_timeout_ms" in parameters:
+            inputs["timeout_ms"] = parameters["consumer_timeout_ms"]
+
+        # Add other potential runtime parameters that are not configuration
+        runtime_param_names = {"max_messages", "timeout_ms", "limit", "offset"}
+        for param_name, param_value in parameters.items():
+            if param_name in runtime_param_names:
+                inputs[param_name] = param_value
 
         # Add connected inputs from other nodes
         for edge in workflow.graph.in_edges(node_id, data=True):
             source_node_id = edge[0]
             mapping = edge[2].get("mapping", {})
 
+            if self.debug:
+                self.logger.debug(f"Processing edge {source_node_id} -> {node_id}")
+                self.logger.debug(f"  Edge data: {edge[2]}")
+                self.logger.debug(f"  Mapping: {mapping}")
+
             if source_node_id in node_outputs:
                 source_outputs = node_outputs[source_node_id]
+                if self.debug:
+                    self.logger.debug(
+                        f"  Source outputs: {list(source_outputs.keys())}"
+                    )
 
                 # Check if the source node failed
                 if isinstance(source_outputs, dict) and source_outputs.get("failed"):
@@ -411,11 +435,24 @@ class LocalRuntime:
                 for source_key, target_key in mapping.items():
                     if source_key in source_outputs:
                         inputs[target_key] = source_outputs[source_key]
+                        if self.debug:
+                            self.logger.debug(
+                                f"  MAPPED: {source_key} -> {target_key} (type: {type(source_outputs[source_key])})"
+                            )
                     else:
+                        if self.debug:
+                            self.logger.debug(
+                                f"  MISSING: {source_key} not in {list(source_outputs.keys())}"
+                            )
                         self.logger.warning(
                             f"Source output '{source_key}' not found in node '{source_node_id}'. "
                             f"Available outputs: {list(source_outputs.keys())}"
                         )
+            else:
+                if self.debug:
+                    self.logger.debug(
+                        f"  No outputs found for source node {source_node_id}"
+                    )
 
         # Apply parameter overrides
         inputs.update(parameters)

kailash/runtime/parallel.py

@@ -91,7 +91,7 @@ class ParallelRuntime:
 
         try:
             # Validate workflow
-            workflow.validate()
+            workflow.validate(runtime_parameters=parameters)
 
             # Initialize semaphore for concurrent execution control
             self.semaphore = asyncio.Semaphore(self.max_workers)
@@ -398,7 +398,7 @@ class ParallelRuntime:
                     async def execute_with_metrics():
                         with collector.collect(node_id=node_id) as context:
                             result = await loop.run_in_executor(
-                                None, lambda: node_instance.execute(**inputs)
+                                None, lambda: node_instance.run(**inputs)
                             )
                             return result, context.result()
 

kailash/runtime/parallel_cyclic.py

@@ -80,7 +80,7 @@ class ParallelCyclicRuntime:
 
         try:
             # Validate workflow
-            workflow.validate()
+            workflow.validate(runtime_parameters=parameters)
 
             # Check for cycles first
             if self.enable_cycles and workflow.has_cycles():
@@ -380,7 +380,7 @@ class ParallelCyclicRuntime:
             # Execute node with metrics collection
             collector = MetricsCollector()
             with collector.collect(node_id=node_id) as metrics_context:
-                outputs = node_instance.execute(**inputs)
+                outputs = node_instance.run(**inputs)
 
             # Get performance metrics
             performance_metrics = metrics_context.result()

kailash/runtime/testing.py

@@ -229,12 +229,12 @@ class NodeTestHelper:
         """Test node execution with given inputs."""
         if should_fail:
             try:
-                result = node.execute(**inputs)
+                result = node.run(**inputs)
                 assert False, "Node execution should have failed but didn't"
             except (NodeValidationError, WorkflowExecutionError):
                 return {}
         else:
-            result = node.execute(**inputs)
+            result = node.run(**inputs)
 
             # Check expected output keys
             for key in expected_keys:

kailash/utils/templates.py

@@ -327,7 +327,7 @@ A Kailash workflow project.
 
 - `workflows/`: Workflow definitions
 - `nodes/`: Custom node implementations
-- `examples/data/`: Input data files
+- `data/inputs/`: Input data files
 - `outputs/`: Output files
 
 ## Usage
@@ -363,7 +363,7 @@ workflow = Workflow(
 )
 
 # Add nodes
-workflow.add_node("reader", CSVReaderNode(), file_path="examples/examples/data/input.csv")
+workflow.add_node("reader", CSVReaderNode(), file_path="data/inputs/csv/input.csv")
 workflow.add_node("filter", Filter(), field="value", operator=">", value=100)
 workflow.add_node("sort", Sort(), field="value", reverse=True)
 workflow.add_node("aggregate", Aggregator(), group_by="category", operation="sum")
@@ -482,8 +482,8 @@ workflow = Workflow(
 )
 
 # Data ingestion
-workflow.add_node("csv_reader", CSVReaderNode(), file_path="examples/examples/data/sales_data.csv")
-workflow.add_node("json_reader", JSONReaderNode(), file_path="examples/examples/data/product_data.json")
+workflow.add_node("csv_reader", CSVReaderNode(), file_path="data/inputs/csv/sales_data.csv")
+workflow.add_node("json_reader", JSONReaderNode(), file_path="data/inputs/json/product_data.json")
 
 # Transform data
 workflow.add_node("filter_sales", Filter(), field="amount", operator=">", value=1000)
@@ -553,7 +553,7 @@ workflow = Workflow(
 )
 
 # Data ingestion
-workflow.add_node("read_data", CSVReaderNode(), file_path="examples/examples/data/text_data.csv")
+workflow.add_node("read_data", CSVReaderNode(), file_path="data/inputs/csv/text_data.csv")
 
 # Preprocessing
 workflow.add_node("extract_text", Map(), field="content")
@@ -616,7 +616,7 @@ workflow = Workflow(
 )
 
 # Read configuration
-workflow.add_node("read_config", JSONReaderNode(), file_path="examples/examples/data/api_config.json")
+workflow.add_node("read_config", JSONReaderNode(), file_path="data/inputs/json/api_config.json")
 
 # Process with AI agent
 workflow.add_node("chat_agent", ChatAgent(),

kailash/visualization/performance.py

@@ -63,8 +63,12 @@ class PerformanceVisualizer:
             Dictionary mapping chart names to file paths
         """
         if output_dir is None:
-            # Use relative path that works from project root or create in current directory
-            output_dir = Path.cwd() / "outputs" / "performance"
+            # Use centralized output directory
+            # Get project root and use data/outputs/visualizations/performance
+            project_root = Path(__file__).parent.parent.parent.parent
+            output_dir = (
+                project_root / "data" / "outputs" / "visualizations" / "performance"
+            )
         output_dir.mkdir(parents=True, exist_ok=True)
 
         # Get run data
@@ -719,7 +723,16 @@ class PerformanceVisualizer:
     ) -> Path:
         """Compare performance across multiple runs."""
         if output_path is None:
-            output_path = Path.cwd() / "outputs" / "performance" / "comparison.png"
+            # Use centralized output directory
+            project_root = Path(__file__).parent.parent.parent.parent
+            output_path = (
+                project_root
+                / "data"
+                / "outputs"
+                / "visualizations"
+                / "performance"
+                / "comparison.png"
+            )
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
         fig, axes = plt.subplots(2, 2, figsize=(15, 12))

kailash/visualization/reports.py

@@ -168,9 +168,13 @@ class WorkflowPerformanceReporter:
         """
         if output_path is None:
             timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            # Use centralized output directory
+            project_root = Path(__file__).parent.parent.parent.parent
             output_path = (
-                Path.cwd()
+                project_root
+                / "data"
                 / "outputs"
+                / "reports"
                 / f"workflow_report_{run_id[:8]}_{timestamp}.{format.value}"
             )
 

kailash/workflow/convergence.py

@@ -229,7 +229,7 @@ class AdaptiveCondition(ConvergenceCondition):
 
 
 def create_convergence_condition(
-    spec: Union[str, int, Callable, Dict]
+    spec: Union[str, int, Callable, Dict],
 ) -> ConvergenceCondition:
     """Factory function to create convergence conditions from various specs.
 

kailash/workflow/cycle_analyzer.py

@@ -84,7 +84,14 @@ class CycleAnalyzer:
         self.analysis_level = analysis_level
         self.enable_profiling = enable_profiling
         self.enable_debugging = enable_debugging
-        self.output_directory = Path(output_directory) if output_directory else None
+
+        # Set output directory - use centralized location if not specified
+        if output_directory:
+            self.output_directory = Path(output_directory)
+        else:
+            # Use centralized output directory by default
+            project_root = Path(__file__).parent.parent.parent.parent
+            self.output_directory = project_root / "data" / "outputs" / "cycle_analysis"
 
         # Initialize components based on configuration
         debug_level = {

kailash/workflow/cyclic_runner.py

@@ -168,7 +168,7 @@ class CyclicWorkflowExecutor:
             WorkflowValidationError: If workflow is invalid
         """
         # Validate workflow (including cycles)
-        workflow.validate()
+        workflow.validate(runtime_parameters=parameters)
 
         # Generate run ID if not provided
         if not run_id:

kailash/workflow/graph.py

@@ -191,10 +191,12 @@ class Workflow:
 
         # Store node instance and metadata
         try:
+            # Use the node instance's actual config, which includes both original config and any updates
+            actual_config = node_instance.config.copy()
             node_instance_data = NodeInstance(
                 node_id=node_id,
                 node_type=node_type,
-                config=config,
+                config=actual_config,
                 position=(len(self.nodes) * 150, 100),
             )
             self.nodes[node_id] = node_instance_data
@@ -203,8 +205,10 @@ class Workflow:
 
         self._node_instances[node_id] = node_instance
 
-        # Add to graph
-        self.graph.add_node(node_id, node=node_instance, type=node_type, config=config)
+        # Add to graph with actual config
+        self.graph.add_node(
+            node_id, node=node_instance, type=node_type, config=actual_config
+        )
         logger.info(f"Added node '{node_id}' of type '{node_type}'")
 
     def _add_node_internal(
@@ -707,9 +711,12 @@ class Workflow:
                 # This shouldn't happen, but handle gracefully
                 raise WorkflowValidationError("Unable to determine execution order")
 
-    def validate(self) -> None:
+    def validate(self, runtime_parameters: Optional[Dict[str, Any]] = None) -> None:
         """Validate the workflow structure.
 
+        Args:
+            runtime_parameters: Parameters that will be provided at runtime (Session 061)
+
         Raises:
             WorkflowValidationError: If workflow is invalid
         """
@@ -758,14 +765,19 @@ class Workflow:
                         # Check nested config
                         found_in_config = param_name in node_instance.config["config"]
 
-                    if not found_in_config:
+                    # Session 061: Check if parameter will be provided at runtime
+                    found_in_runtime = False
+                    if runtime_parameters and node_id in runtime_parameters:
+                        found_in_runtime = param_name in runtime_parameters[node_id]
+
+                    if not found_in_config and not found_in_runtime:
                         if param_def.default is None:
                             missing_inputs.append(param_name)
 
             if missing_inputs:
                 raise WorkflowValidationError(
                     f"Node '{node_id}' missing required inputs: {missing_inputs}. "
-                    f"Provide these inputs via connections or node configuration"
+                    f"Provide these inputs via connections, node configuration, or runtime parameters"
                 )
 
         logger.info(f"Workflow '{self.name}' validated successfully")
@@ -928,6 +940,14 @@ class Workflow:
                     to_input = edge_data.get("to_input")
                     mapping = edge_data.get("mapping", {})
 
+                    print(f"CONNECTION DEBUG: {source_node_id} -> {node_id}")
+                    print(f"  Edge data: {edge_data}")
+                    print(f"  from_output: {from_output}, to_input: {to_input}")
+                    print(f"  mapping: {mapping}")
+                    print(
+                        f"  source_results keys: {list(results.get(source_node_id, {}).keys())}"
+                    )
+
                     source_results = results.get(source_node_id, {})
 
                     # Handle backward compatibility - from_output/to_input can be string or list
@@ -951,6 +971,13 @@ class Workflow:
                     for source_key, target_key in mapping.items():
                         if source_key in source_results:
                             node_inputs[target_key] = source_results[source_key]
+                            print(
+                                f"MAPPING DEBUG: {source_key} -> {target_key}, value type: {type(source_results[source_key])}"
+                            )
+                        else:
+                            print(
+                                f"MAPPING DEBUG: Source key '{source_key}' not found in source results: {list(source_results.keys())}"
+                            )
 
                 # Apply overrides
                 node_overrides = inputs.get(node_id, {})

kailash/workflow/visualization.py

@@ -399,8 +399,16 @@ class WorkflowVisualizer:
 
         # Determine output path
         if output_path is None:
-            # Create default directory if it doesn't exist
-            output_dir = Path.cwd() / "outputs" / "workflow_executions"
+            # Use centralized output directory
+            # Get project root and use data/outputs/visualizations
+            project_root = Path(__file__).parent.parent.parent.parent
+            output_dir = (
+                project_root
+                / "data"
+                / "outputs"
+                / "visualizations"
+                / "workflow_executions"
+            )
             output_dir.mkdir(parents=True, exist_ok=True)
             output_path = output_dir / f"execution_{run_id}.md"
         else: