hexdag 0.5.0.dev1__py3-none-any.whl

hexdag_plugins/hexdag_etl/examples/03_file_io_pipeline.py

@@ -0,0 +1,109 @@
+"""Example: File-based ETL pipeline with CSV input and output.
+
+This example demonstrates:
+1. Reading CSV files with FileReaderNode
+2. Writing results to CSV with FileWriterNode
+
+For a complete example with transforms, use YAML pipelines where
+template expressions ({{ }}) are properly resolved.
+"""
+
+import asyncio
+import tempfile
+from pathlib import Path
+
+import pandas as pd
+
+
+async def main() -> None:
+    """Run a simple file-based ETL pipeline (read -> write)."""
+    # Import after ensuring the module path is set
+    import sys
+
+    sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+
+    from hexdag.core.domain.dag import DirectedGraph
+    from hexdag.core.orchestration.orchestrator import Orchestrator
+
+    # Import ETL nodes
+    from hexdag_plugins.hexdag_etl.hexdag_etl.nodes.file_io import (
+        FileReaderNode,
+        FileWriterNode,
+    )
+
+    # Create temporary directory for test files
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir_path = Path(tmpdir)
+
+        # Create sample input CSV
+        input_csv = tmpdir_path / "sales_data.csv"
+        sample_data = pd.DataFrame(
+            {
+                "product": ["Widget A", "Widget B", "Widget C", "Widget A", "Widget B"],
+                "quantity": [10, 5, 8, 15, 3],
+                "price": [29.99, 49.99, 19.99, 29.99, 49.99],
+                "region": ["North", "South", "North", "East", "West"],
+            }
+        )
+        sample_data.to_csv(input_csv, index=False)
+        print(f"Created input CSV: {input_csv}")
+        print(f"Input data:\n{sample_data}\n")
+
+        # Create output path (using CSV for portability - no pyarrow required)
+        output_csv = tmpdir_path / "sales_copy.csv"
+
+        # Build the pipeline graph
+        graph = DirectedGraph()
+
+        # Step 1: Read CSV file
+        reader = FileReaderNode()
+        read_spec = reader(
+            name="read_sales",
+            file_path=str(input_csv),
+            format="csv",
+        )
+        graph.add(read_spec)
+
+        # Step 2: Write to CSV (directly from reader output)
+        writer = FileWriterNode()
+        write_spec = writer(
+            name="write_results",
+            file_path=str(output_csv),
+            format="csv",
+            input_key="data",
+            deps=["read_sales"],
+        )
+        graph.add(write_spec)
+
+        # Execute the pipeline
+        print("Executing ETL pipeline...")
+        orchestrator = Orchestrator()
+        results = await orchestrator.run(graph, {})
+
+        # Display results
+        print("\n=== Pipeline Results ===")
+        for node_name, result in results.items():
+            print(f"\n{node_name}:")
+            if isinstance(result, dict):
+                for key, value in result.items():
+                    if key == "data" and isinstance(value, pd.DataFrame):
+                        print(f"  {key}: DataFrame with {len(value)} rows")
+                    elif key == "columns":
+                        print(f"  {key}: {value}")
+                    else:
+                        print(f"  {key}: {value}")
+            else:
+                print(f"  {result}")
+
+        # Verify output file
+        if output_csv.exists():
+            output_df = pd.read_csv(output_csv)
+            print("\n=== Output CSV Contents ===")
+            print(output_df)
+            print(f"\nRows written: {len(output_df)}")
+        else:
+            print(f"\nWarning: Output file not created: {output_csv}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

hexdag_plugins/hexdag_etl/examples/test_pandas_transform.py

@@ -0,0 +1,84 @@
+"""Test pandas transform node directly without plugin registration."""
+
+import sys
+from pathlib import Path
+
+# Add parent directories to path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from hexdag.core.pipeline_builder.yaml_builder import YamlPipelineBuilder
+
+
+def test_node_registration():
+    """Test that node is discoverable."""
+    print("Testing node registration...")
+
+    # List all registered nodes
+    from hexdag.core.registry import registry
+
+    try:
+        # Try to get pandas_transform
+        pandas_factory = registry.get("pandas_transform", namespace="etl")
+        print(f"✓ PandasTransformNode factory: {pandas_factory}")
+        print(f"✓ PandasTransformNode class: {pandas_factory.__class__}")
+        return True
+    except Exception as e:
+        print(f"❌ Error getting pandas_transform: {e}")
+        print("  This is expected if nodes aren't auto-registered yet")
+        return False
+
+
+def test_yaml_parsing():
+    """Test YAML pipeline parsing."""
+    print("\nTesting YAML pipeline...")
+
+    pipeline_yaml = """
+apiVersion: hexdag/v1
+kind: Pipeline
+metadata:
+  name: test-pandas-transform
+spec:
+  nodes:
+    - kind: etl:pandas_transform
+      metadata:
+        name: test_transform
+      spec:
+        operations:
+          - type: transform
+            method: pandas.DataFrame.head
+            kwargs:
+              n: 5
+"""
+
+    try:
+        builder = YamlPipelineBuilder()
+        graph, config = builder.build_from_string(pipeline_yaml)
+        print("\n✓ Pipeline built successfully!")
+        print(f"✓ Nodes: {len(graph._graph.nodes())}")
+        return True
+    except Exception as e:
+        print(f"\n❌ Error building pipeline: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return False
+
+
+if __name__ == "__main__":
+    print("=" * 80)
+    print("Testing Pandas Transform Node")
+    print("=" * 80)
+
+    reg_ok = test_node_registration()
+    yaml_ok = test_yaml_parsing()
+
+    if reg_ok and yaml_ok:
+        print("\n" + "=" * 80)
+        print("✓ All tests passed!")
+        print("=" * 80)
+    else:
+        print("\n" + "=" * 80)
+        print("❌ Some tests failed")
+        print("=" * 80)
+        sys.exit(1)

hexdag_plugins/hexdag_etl/hexdag.toml

@@ -0,0 +1,25 @@
+# hexDAG ETL Plugin Configuration
+# This file configures how the plugin loads its components
+
+[tool.hexdag]
+# Core modules to load - include the plugin's nodes
+modules = [
+    "hexdag.core.ports",               # Core port definitions
+    "hexdag.builtin.nodes",            # Core node factories
+    "hexdag.builtin.tools.builtin_tools",
+    "hexdag.builtin.policies.execution_policies",
+    "hexdag.builtin.macros",
+    "hexdag_etl.nodes",                # ETL plugin nodes (pandas_transform, etc.)
+]
+
+# No additional plugins needed for ETL plugin
+plugins = []
+
+# Development mode enabled
+dev_mode = true
+
+[tool.hexdag.logging]
+level = "DEBUG"  # More verbose for plugin development
+format = "structured"
+use_color = true
+include_timestamp = true

hexdag_plugins/hexdag_etl/hexdag_etl/__init__.py

@@ -0,0 +1,48 @@
+"""hexdag-etl: ETL infrastructure for hexDAG pipelines.
+
+Provides file I/O and multi-operation pandas transform nodes for data transformation pipelines.
+
+This plugin extends hexDAG with ETL capabilities:
+- FileReaderNode: Read CSV, Parquet, JSON, Excel files
+- FileWriterNode: Write data to various file formats
+- PandasTransformNode: Chain pandas operations
+
+Example Pipeline:
+    - kind: etl:file_reader_node
+      metadata:
+        name: load_data
+      spec:
+        file_path: data/input.csv
+        format: csv
+
+    - kind: etl:pandas_transform_node
+      metadata:
+        name: transform
+      spec:
+        operations:
+          - type: filter
+            condition: "{{ df['value'] > 0 }}"
+      dependencies: [load_data]
+
+    - kind: etl:file_writer_node
+      metadata:
+        name: save_results
+      spec:
+        file_path: output/results.parquet
+        format: parquet
+      dependencies: [transform]
+"""
+
+from .nodes.file_io import FileReaderNode, FileWriterNode
+from .nodes.outlook import OutlookReaderNode, OutlookSenderNode
+from .nodes.pandas_transform import PandasTransformNode
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "FileReaderNode",
+    "FileWriterNode",
+    "OutlookReaderNode",
+    "OutlookSenderNode",
+    "PandasTransformNode",
+]

hexdag_plugins/hexdag_etl/hexdag_etl/nodes/__init__.py

@@ -0,0 +1,13 @@
+"""ETL nodes for data extraction, transformation, and loading."""
+
+from .file_io import FileReaderNode, FileWriterNode
+from .outlook import OutlookReaderNode, OutlookSenderNode
+from .pandas_transform import PandasTransformNode
+
+__all__ = [
+    "FileReaderNode",
+    "FileWriterNode",
+    "OutlookReaderNode",
+    "OutlookSenderNode",
+    "PandasTransformNode",
+]

hexdag_plugins/hexdag_etl/hexdag_etl/nodes/api_extract.py

@@ -0,0 +1,230 @@
+"""API extraction node for HTTP/REST API data extraction."""
+
+from typing import Any
+
+from hexdag.core.domain.dag import NodeSpec
+from hexdag.core.registry import node
+from hexdag.core.registry.models import NodeSubtype
+
+from .base_node_factory import BaseNodeFactory
+
+
+@node(name="api_extract", subtype=NodeSubtype.TOOL, namespace="etl")
+class APIExtractNode(BaseNodeFactory):
+    """Extract data from REST APIs with pagination, authentication, and error handling.
+
+    Supports:
+    - GET/POST requests
+    - Bearer token, API key, OAuth authentication
+    - Pagination (cursor, offset, page)
+    - Rate limiting
+    - Retry logic
+    - Structured error handling
+
+    Examples
+    --------
+    YAML pipeline::
+
+        - kind: api_extract_node
+          metadata:
+            name: fetch_customers
+          spec:
+            endpoint: https://api.example.com/v1/customers
+            method: GET
+            params:
+              limit: 100
+              status: active
+            pagination:
+              type: cursor
+              cursor_param: after
+              cursor_path: meta.next_cursor
+              has_more_path: meta.has_more
+            auth:
+              type: bearer
+              token: ${API_TOKEN}
+            output_artifact:
+              slot: raw_customers
+              key: customers_2024_01_15
+              format: json
+    """
+
+    def __call__(
+        self,
+        name: str,
+        endpoint: str,
+        method: str = "GET",
+        params: dict[str, Any] | None = None,
+        headers: dict[str, str] | None = None,
+        auth: dict[str, Any] | None = None,
+        pagination: dict[str, Any] | None = None,
+        output_artifact: dict[str, Any] | None = None,
+        timeout: int = 30,
+        max_retries: int = 3,
+        backoff_factor: float = 0.3,
+        rate_limit: dict[str, Any] | None = None,
+        deps: list[str] | None = None,
+        **kwargs: Any,
+    ) -> NodeSpec:
+        """Create API extraction node specification.
+
+        Parameters
+        ----------
+        name : str
+            Node name
+        endpoint : str
+            API endpoint URL
+        method : str
+            HTTP method: "GET", "POST", "PUT", "DELETE"
+        params : dict, optional
+            Query parameters or request body
+        headers : dict, optional
+            Additional HTTP headers
+        auth : dict, optional
+            Authentication configuration
+        pagination : dict, optional
+            Pagination configuration
+        output_artifact : dict, optional
+            Output artifact configuration
+        timeout : int
+            Request timeout in seconds
+        max_retries : int
+            Maximum retry attempts
+        backoff_factor : float
+            Backoff multiplier for retries
+        rate_limit : dict, optional
+            Rate limiting configuration
+        deps : list[str], optional
+            Dependency node names
+        **kwargs : Any
+            Additional parameters
+
+        Returns
+        -------
+        NodeSpec
+            Node specification ready for execution
+        """
+        # Create wrapped function
+        wrapped_fn = self._create_api_function(
+            name,
+            endpoint,
+            method,
+            params,
+            headers,
+            auth,
+            pagination,
+            output_artifact,
+            timeout,
+            max_retries,
+            backoff_factor,
+            rate_limit,
+        )
+
+        # Define schemas
+        input_schema = {"input_data": dict, "**ports": dict}
+        output_schema = {"output": dict, "metadata": dict}
+
+        input_model = self.create_pydantic_model(f"{name}Input", input_schema)
+        output_model = self.create_pydantic_model(f"{name}Output", output_schema)
+
+        # Store parameters
+        node_params = {
+            "endpoint": endpoint,
+            "method": method,
+            "params": params,
+            "headers": headers,
+            "auth": auth,
+            "pagination": pagination,
+            "output_artifact": output_artifact,
+            "timeout": timeout,
+            "max_retries": max_retries,
+            "backoff_factor": backoff_factor,
+            "rate_limit": rate_limit,
+            **kwargs,
+        }
+
+        return NodeSpec(
+            name=name,
+            fn=wrapped_fn,
+            in_model=input_model,
+            out_model=output_model,
+            deps=frozenset(deps or []),
+            params=node_params,
+        )
+
+    def _create_api_function(
+        self,
+        name: str,
+        endpoint: str,
+        method: str,
+        params: dict[str, Any] | None,
+        headers: dict[str, str] | None,
+        auth: dict[str, Any] | None,
+        pagination: dict[str, Any] | None,
+        output_artifact: dict[str, Any] | None,
+        timeout: int,
+        max_retries: int,
+        backoff_factor: float,
+        rate_limit: dict[str, Any] | None,
+    ) -> Any:
+        """Create the wrapped API extraction function.
+
+        Implementation details omitted for brevity - similar to the original
+        but simplified for demo purposes.
+
+        Parameters
+        ----------
+        name : str
+            Node name
+        endpoint : str
+            API endpoint
+        method : str
+            HTTP method
+        params : dict, optional
+            Request parameters
+        headers : dict, optional
+            Request headers
+        auth : dict, optional
+            Authentication config
+        pagination : dict, optional
+            Pagination config
+        output_artifact : dict, optional
+            Output artifact config
+        timeout : int
+            Request timeout
+        max_retries : int
+            Maximum retries
+        backoff_factor : float
+            Backoff factor
+        rate_limit : dict, optional
+            Rate limiting config
+
+        Returns
+        -------
+        Callable
+            Async function that performs API extraction
+        """
+
+        # Implementation would go here - simplified for the example
+        async def wrapped_fn(input_data: Any, **ports: Any) -> dict[str, Any]:
+            """Placeholder implementation."""
+            # In a real implementation, this would:
+            # 1. Make HTTP requests with aiohttp
+            # 2. Handle pagination
+            # 3. Apply authentication
+            # 4. Store results in artifact store
+            # 5. Return structured results
+
+            return {
+                "data": [],
+                "metadata": {
+                    "endpoint": endpoint,
+                    "method": method,
+                    "records_extracted": 0,
+                    "status": "placeholder_implementation",
+                },
+            }
+
+        wrapped_fn.__name__ = f"api_extract_{name}"
+        wrapped_fn.__doc__ = f"API extraction: {endpoint}"
+
+        return wrapped_fn

hexdag_plugins/hexdag_etl/hexdag_etl/nodes/base_node_factory.py

@@ -0,0 +1,181 @@
+"""Simplified BaseNodeFactory for creating nodes with Pydantic models."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, cast
+
+from hexdag.core.domain.dag import NodeSpec
+from hexdag.core.orchestration.prompt.template import PromptTemplate
+from hexdag.core.protocols import is_schema_type
+from pydantic import BaseModel, create_model
+
+
+class BaseNodeFactory(ABC):
+    """Minimal base class for node factories with Pydantic models."""
+
+    # Note: Event emission has been removed as it's now handled by the orchestrator
+    # The new event system uses ObserverManager at the orchestrator level
+
+    def create_pydantic_model(
+        self, name: str, schema: dict[str, Any] | type[BaseModel] | type[Any] | None
+    ) -> type[BaseModel] | None:
+        """Create a Pydantic model from a schema.
+
+        Raises
+        ------
+        ValueError
+            If schema type is not supported
+        """
+        if schema is None:
+            return None
+
+        if is_schema_type(schema):
+            return schema  # type: ignore[return-value]  # is_schema_type checks for BaseModel subclass
+
+        if isinstance(schema, dict):
+            # String type names mapping (for when field_type is a string)
+            type_map = {
+                "str": str,
+                "int": int,
+                "float": float,
+                "bool": bool,
+                "list": list,
+                "dict": dict,
+                "Any": Any,
+            }
+
+            field_definitions: dict[str, Any] = {}
+            for field_name, field_type in schema.items():
+                # Dispatch based on field_type's type using match pattern
+                match field_type:
+                    case str():
+                        # String type names - convert to actual types
+                        actual_type = type_map.get(field_type, Any)
+                        field_definitions[field_name] = (actual_type, ...)
+                    case type():
+                        # Already a type
+                        field_definitions[field_name] = (field_type, ...)
+                    case tuple():
+                        # Already in the correct format (type, default)
+                        field_definitions[field_name] = field_type
+                    case _:
+                        # Unknown type specification - use Any
+                        field_definitions[field_name] = (Any, ...)
+
+            return create_model(name, **field_definitions)
+
+        # At this point, schema should be a type
+        try:
+            return cast("type[Any] | None", create_model(name, value=(schema, ...)))
+        except (TypeError, AttributeError) as e:
+            # If we get here, schema is an unexpected type
+            raise ValueError(f"Schema must be a dict, type, or Pydantic model, got {type(schema).__name__}") from e
+
+    @staticmethod
+    def infer_input_schema_from_template(
+        template: str | PromptTemplate,
+        special_params: set[str] | None = None,
+    ) -> dict[str, Any]:
+        """Infer input schema from template variables with configurable filtering.
+
+        This method extracts variable names from a prompt template and creates
+        a schema dictionary mapping those variables to string types. It supports
+        filtering out special parameters that are not user inputs.
+
+        Parameters
+        ----------
+        template : str | PromptTemplate
+            The prompt template to analyze. Can be a string or PromptTemplate instance.
+        special_params : set[str] | None, optional
+            Set of parameter names to exclude from the schema (e.g., "context_history").
+            If None, no filtering is applied.
+
+        Returns
+        -------
+        dict[str, Any]
+            Schema dictionary mapping variable names to str type.
+            Returns {"input": str} if no variables found.
+
+        Examples
+        --------
+        >>> BaseNodeFactory.infer_input_schema_from_template("Hello {{name}}")
+        {'name': <class 'str'>}
+
+        >>> BaseNodeFactory.infer_input_schema_from_template(
+        ...     "Process {{user}} with {{context_history}}",
+        ...     special_params={"context_history"}
+        ... )
+        {'user': <class 'str'>}
+
+        >>> BaseNodeFactory.infer_input_schema_from_template("No variables")
+        {'input': <class 'str'>}
+        """
+
+        if isinstance(template, str):
+            template = PromptTemplate(template)
+
+        variables = getattr(template, "input_vars", [])
+
+        if special_params:
+            variables = [v for v in variables if v not in special_params]
+
+        if not variables:
+            return {"input": str}
+
+        schema: dict[str, Any] = {}
+        for var in variables:
+            base_var = var.split(".")[0]
+            # Double-check against special params for nested variables
+            if not special_params or base_var not in special_params:
+                schema[base_var] = str
+
+        return schema
+
+    def _copy_required_ports_to_wrapper(self, wrapper_fn: Any) -> None:
+        """Copy required_ports metadata from factory class to wrapper function.
+
+        This ensures port requirements are preserved when creating node functions.
+        """
+        if hasattr(self.__class__, "_hexdag_required_ports"):
+            # _hexdag_required_ports is added dynamically by @node decorator
+            wrapper_fn._hexdag_required_ports = self.__class__._hexdag_required_ports  # pyright: ignore[reportAttributeAccessIssue]  # noqa: B010
+
+    def create_node_with_mapping(
+        self,
+        name: str,
+        wrapped_fn: Any,
+        input_schema: dict[str, Any] | None,
+        output_schema: dict[str, Any] | type[BaseModel] | None,
+        deps: list[str] | None = None,
+        **kwargs: Any,
+    ) -> NodeSpec:
+        """Universal NodeSpec creation."""
+        # Copy required_ports metadata to wrapper
+        self._copy_required_ports_to_wrapper(wrapped_fn)
+
+        input_model = self.create_pydantic_model(f"{name}Input", input_schema)
+        output_model = self.create_pydantic_model(f"{name}Output", output_schema)
+
+        return NodeSpec(
+            name=name,
+            fn=wrapped_fn,
+            in_model=input_model,
+            out_model=output_model,
+            deps=frozenset(deps or []),
+            params=kwargs,
+        )
+
+    @abstractmethod
+    def __call__(self, name: str, *args: Any, **kwargs: Any) -> NodeSpec:  # noqa: ARG002
+        """Create a NodeSpec.
+
+        Must be implemented by subclasses.
+
+        Args:
+            name: Name of the node
+            *args: Additional positional arguments (unused, for subclass flexibility)
+            **kwargs: Additional keyword arguments
+        """
+        _ = args  # Marked as intentionally unused for subclass API flexibility
+        raise NotImplementedError