hexdag 0.5.0.dev1__py3-none-any.whl

hexdag/core/orchestration/orchestrator_factory.py

@@ -0,0 +1,275 @@
+"""Orchestrator Factory - Creates orchestrator instances from pipeline configuration.
+
+This factory bridges the gap between declarative YAML configuration and the
+runtime orchestrator, instantiating adapters and policies from their specs.
+"""
+
+from typing import Any
+
+from hexdag.core.logging import get_logger
+from hexdag.core.orchestration.orchestrator import Orchestrator
+from hexdag.core.pipeline_builder.component_instantiator import ComponentInstantiator
+from hexdag.core.pipeline_builder.pipeline_config import PipelineConfig
+from hexdag.core.ports_builder import PortsBuilder
+
+logger = get_logger(__name__)
+
+
+class OrchestratorFactory:
+    """Factory for creating orchestrator instances from pipeline configuration.
+
+    This factory handles the complex task of:
+    1. Instantiating adapter instances from port specs
+    2. Instantiating policy instances from policy specs
+    3. Wiring everything together into a configured orchestrator
+
+    Examples
+    --------
+    ```python
+    from hexdag.core.pipeline_builder.yaml_builder import YamlPipelineBuilder
+    from hexdag.core.orchestration.orchestrator_factory import OrchestratorFactory
+
+    # Parse YAML pipeline
+    builder = YamlPipelineBuilder()
+    graph, pipeline_config = builder.build_from_yaml_file("pipeline.yaml")
+
+    factory = OrchestratorFactory()
+    orchestrator = factory.create_orchestrator(pipeline_config)
+
+    # Execute the pipeline
+    result = await orchestrator.aexecute_dag(graph, initial_input={"query": "..."})
+    ```
+    """
+
+    def __init__(self) -> None:
+        """Initialize the orchestrator factory."""
+        self.component_instantiator = ComponentInstantiator()
+
+    def create_orchestrator(
+        self,
+        pipeline_config: PipelineConfig,
+        max_concurrent_nodes: int = 10,
+        strict_validation: bool = False,
+        default_node_timeout: float | None = None,
+        additional_ports: dict[str, Any] | None = None,
+    ) -> Orchestrator:
+        """Create an orchestrator instance from pipeline configuration.
+
+        Parameters
+        ----------
+        pipeline_config : PipelineConfig
+            Pipeline configuration with ports, policies, and metadata
+        max_concurrent_nodes : int, optional
+            Maximum number of nodes to execute concurrently, by default 10
+        strict_validation : bool, optional
+            If True, raise errors on validation failure, by default False
+        default_node_timeout : float | None, optional
+            Default timeout in seconds for each node, by default None
+        additional_ports : dict[str, Any] | None, optional
+            Additional ports to merge with configured ports, by default None
+
+        Returns
+        -------
+        Orchestrator
+            Configured orchestrator ready to execute pipelines
+
+        Examples
+        --------
+        Basic usage with global ports::
+
+            factory = OrchestratorFactory()
+
+            orchestrator = factory.create_orchestrator(
+                pipeline_config=config,
+                max_concurrent_nodes=5,
+                strict_validation=True,
+                default_node_timeout=60.0,
+            )
+
+        Notes
+        -----
+        **Known Limitations (Phase 4)**:
+
+        - **Type-specific ports** (``type_ports`` in YAML): Parsed and validated
+          but not yet used during execution. All nodes currently receive global ports.
+
+        - **Node-level port overrides**: Not yet implemented. Requires orchestrator
+          changes to resolve ports per-node at execution time.
+
+        **Workaround for Advanced Port Configuration**:
+
+        For type-specific or node-level port customization, use PortsBuilder
+        programmatically::
+
+            from hexdag.core.ports_builder import PortsBuilder
+
+            builder = (
+                PortsBuilder()
+                .with_llm(MockLLM())  # Global default
+                .for_type("agent", llm=OpenAIAdapter(model="gpt-4"))
+                .for_node("researcher", llm=AnthropicAdapter(model="claude-3"))
+            )
+
+            # Pass as additional_ports
+            orchestrator = factory.create_orchestrator(
+                pipeline_config,
+                additional_ports=builder.build()
+            )
+
+        These features are planned for future implementation when orchestrator
+        supports per-node port resolution.
+        """
+        logger.info(
+            "Creating orchestrator from pipeline config: {} ports, {} type_ports, {} policies",
+            len(pipeline_config.ports),
+            len(pipeline_config.type_ports),
+            len(pipeline_config.policies),
+        )
+
+        # Step 1: Build PortsConfiguration if type_ports or node_ports are configured
+        use_ports_config = bool(pipeline_config.type_ports)
+
+        if use_ports_config:
+            ports_config = self._build_ports_configuration(pipeline_config, additional_ports)
+            # Note: PortsConfiguration converts dicts to tuples for immutability
+            global_ports = {k: v.port for k, v in (ports_config.global_ports or ())}
+        else:
+            # Simple case: just global ports
+            ports_config = None
+            global_ports = self._instantiate_ports(pipeline_config.ports)
+            if additional_ports:
+                global_ports.update(additional_ports)
+
+        # Step 2: Create orchestrator with configured ports
+        orchestrator = Orchestrator(
+            max_concurrent_nodes=max_concurrent_nodes,
+            ports=ports_config if ports_config else global_ports,
+            strict_validation=strict_validation,
+            default_node_timeout=default_node_timeout,
+        )
+
+        logger.info(
+            "✅ Orchestrator created with {} ports",
+            len(global_ports),
+        )
+
+        return orchestrator
+
+    def _instantiate_ports(self, port_specs: dict[str, dict[str, Any]]) -> dict[str, Any]:
+        """Instantiate adapter instances from port specifications.
+
+        Parameters
+        ----------
+        port_specs : dict[str, dict[str, Any]]
+            Map of port_name -> adapter dict spec
+
+        Returns
+        -------
+        dict[str, Any]
+            Map of port_name -> adapter_instance
+        """
+        ports = {}
+
+        for port_name, port_spec in port_specs.items():
+            try:
+                logger.debug("Instantiating port: {} = {}", port_name, port_spec)
+                adapter = self.component_instantiator.instantiate_adapter(port_spec, port_name)
+                ports[port_name] = adapter
+                logger.debug("✅ Port instantiated: {}", port_name)
+            except Exception as e:
+                logger.error(
+                    "Failed to instantiate port '{}' from spec '{}': {}",
+                    port_name,
+                    port_spec,
+                    e,
+                )
+                raise
+
+        return ports
+
+    def _build_ports_configuration(
+        self, pipeline_config: PipelineConfig, additional_ports: dict[str, Any] | None
+    ) -> Any:
+        """Build a PortsConfiguration from pipeline config.
+
+        Parameters
+        ----------
+        pipeline_config : PipelineConfig
+            Pipeline configuration with ports, type_ports
+        additional_ports : dict[str, Any] | None
+            Additional ports to merge
+
+        Returns
+        -------
+        PortsConfiguration
+            Hierarchical ports configuration with type-specific support
+        """
+        from hexdag.core.orchestration.models import PortConfig, PortsConfiguration
+
+        # Step 1: Instantiate global ports
+        global_ports_dict = self._instantiate_ports(pipeline_config.ports)
+        if additional_ports:
+            global_ports_dict.update(additional_ports)
+
+        # Wrap in PortConfig
+        global_ports = {k: PortConfig(port=v) for k, v in global_ports_dict.items()}
+
+        # Step 2: Instantiate type-specific ports
+        type_ports = None
+        if pipeline_config.type_ports:
+            type_ports = {}
+            for node_type, type_port_specs in pipeline_config.type_ports.items():
+                type_port_instances = self._instantiate_ports(type_port_specs)
+                type_ports[node_type] = {
+                    k: PortConfig(port=v) for k, v in type_port_instances.items()
+                }
+                logger.debug(
+                    "Configured {} ports for node type '{}'",
+                    len(type_port_instances),
+                    node_type,
+                )
+
+        # Step 3: TODO: Add node-level port support when available in YAML builder
+
+        return PortsConfiguration(
+            global_ports=global_ports,
+            type_ports=type_ports,
+            node_ports=None,  # Not yet implemented
+        )
+
+    def create_ports_builder(self, pipeline_config: PipelineConfig) -> PortsBuilder:
+        """Create a PortsBuilder from pipeline configuration (convenience method).
+
+        This method provides an alternative workflow using PortsBuilder for
+        advanced use cases that need type-specific or node-specific port overrides.
+
+        Parameters
+        ----------
+        pipeline_config : PipelineConfig
+            Pipeline configuration with ports
+
+        Returns
+        -------
+        PortsBuilder
+            Builder with ports configured from pipeline config
+
+        Examples
+        --------
+        ```python
+        factory = OrchestratorFactory()
+
+        builder = factory.create_ports_builder(pipeline_config)
+
+        # Optionally add node-specific overrides
+        builder.for_node("researcher", llm=custom_llm)
+
+        ports = builder.build()
+        ```
+        """
+        ports = self._instantiate_ports(pipeline_config.ports)
+
+        builder = PortsBuilder()
+        for port_name, port_instance in ports.items():
+            builder.with_custom(port_name, port_instance)
+
+        return builder

hexdag/core/orchestration/port_wrappers.py

@@ -0,0 +1,327 @@
+"""Port wrappers that provide observability and control.
+
+These wrappers intercept port method calls to:
+1. **Observability** - Emit events (LLMPromptSent, LLMResponseReceived, etc.)
+2. **Control** - Enable policy-based control (rate limiting, caching, retry, fallback)
+3. **Metrics** - Track duration and performance
+4. **Extensibility** - Provide hooks for custom behavior
+
+This allows centralized control over infrastructure interactions without
+polluting business logic in nodes and macros.
+"""
+
+import time
+from typing import Any
+
+from hexdag.core.context import get_current_node_name, get_observer_manager
+from hexdag.core.logging import get_logger
+from hexdag.core.orchestration.events import (
+    LLMPromptSent,
+    LLMResponseReceived,
+    ToolCalled,
+    ToolCompleted,
+)
+from hexdag.core.ports.llm import LLM, LLMResponse, MessageList, SupportsGeneration
+
+logger = get_logger(__name__)
+
+
+class ObservableLLMWrapper:
+    """Wraps an LLM port for observability and policy-based control.
+
+    This wrapper provides:
+    - Automatic event emission for all LLM calls
+    - Policy evaluation before/after calls (future: rate limiting, caching)
+    - Duration tracking and performance metrics
+    - Transparent forwarding to underlying LLM
+    """
+
+    def __init__(self, llm: LLM):
+        """Initialize wrapper with underlying LLM port.
+
+        Parameters
+        ----------
+        llm : LLM
+            The underlying LLM port to wrap
+        """
+        self._llm = llm
+
+    async def aresponse(self, messages: MessageList, **kwargs: Any) -> str | None:
+        """Call LLM with observability and policy control.
+
+        This method:
+        1. Emits LLMPromptSent event (for observability)
+        2. Calls underlying LLM
+        3. Emits LLMResponseReceived event with duration
+        4. (Future: Policy evaluation for rate limiting, caching, retry)
+
+        Parameters
+        ----------
+        messages : MessageList
+            Messages to send to LLM
+        **kwargs : Any
+            Additional LLM parameters
+
+        Returns
+        -------
+        str | None
+            LLM response text
+        """
+        node_name = get_current_node_name() or "unknown"
+
+        # Emit prompt sent event (OBSERVABILITY)
+        if observer_mgr := get_observer_manager():
+            await observer_mgr.notify(
+                LLMPromptSent(
+                    node_name=node_name,
+                    messages=[{"role": m.role, "content": m.content} for m in messages],
+                )
+            )
+
+        # TODO(hexdag-team): Add policy evaluation here (CONTROL) #noqa: TD003
+        # if policy_mgr := get_policy_manager():
+        #     policy_response = await policy_mgr.evaluate(LLMCallPolicy(...))
+        #     if policy_response.signal == PolicySignal.SKIP:
+        #         return policy_response.data  # Cached response
+        #     elif policy_response.signal == PolicySignal.FAIL:
+        #         raise RateLimitError("LLM rate limit exceeded")
+
+        # Call underlying LLM
+        start_time = time.perf_counter()
+
+        # Check if the adapter supports generation
+        if isinstance(self._llm, SupportsGeneration):
+            response = await self._llm.aresponse(messages, **kwargs)
+        else:
+            raise NotImplementedError(
+                f"LLM adapter {type(self._llm).__name__} does not support text generation. "
+                "It must implement SupportsGeneration protocol."
+            )
+
+        duration_ms = (time.perf_counter() - start_time) * 1000
+
+        # Emit response received event (OBSERVABILITY)
+        if observer_mgr := get_observer_manager():
+            await observer_mgr.notify(
+                LLMResponseReceived(
+                    node_name=node_name, response=response or "", duration_ms=duration_ms
+                )
+            )
+
+        return response
+
+    async def aresponse_with_tools(
+        self,
+        messages: MessageList,
+        tools: list[dict[str, Any]],
+        tool_choice: str | dict[str, Any] = "auto",
+        **kwargs: Any,
+    ) -> LLMResponse:
+        """Call LLM with tools and emit events.
+
+        Parameters
+        ----------
+        messages : MessageList
+            Messages to send to LLM
+        tools : list[dict[str, Any]]
+            Tool definitions
+        tool_choice : str | dict[str, Any]
+            Tool choice strategy
+        **kwargs : Any
+            Additional LLM parameters
+
+        Returns
+        -------
+        LLMResponse
+            Structured response with tool calls
+        """
+        node_name = get_current_node_name() or "unknown"
+
+        # Emit prompt sent event
+        if observer_mgr := get_observer_manager():
+            await observer_mgr.notify(
+                LLMPromptSent(
+                    node_name=node_name,
+                    messages=[{"role": m.role, "content": m.content} for m in messages],
+                )
+            )
+
+        # Call underlying LLM
+        start_time = time.perf_counter()
+        response = await self._llm.aresponse_with_tools(messages, tools, tool_choice, **kwargs)
+        duration_ms = (time.perf_counter() - start_time) * 1000
+
+        # Emit response received event
+        if observer_mgr := get_observer_manager():
+            await observer_mgr.notify(
+                LLMResponseReceived(
+                    node_name=node_name, response=response.content or "", duration_ms=duration_ms
+                )
+            )
+
+        return response
+
+    def __getattr__(self, name: str) -> Any:
+        """Forward all other attribute access to underlying LLM."""
+        return getattr(self._llm, name)
+
+
+class ObservableToolRouterWrapper:
+    """Wraps a ToolRouter port for observability and policy-based control.
+
+    This wrapper provides:
+    - Automatic event emission for all tool calls (ToolCalled, ToolCompleted)
+    - Policy evaluation before/after calls (future: auth, rate limiting)
+    - Duration tracking and performance metrics
+    - Error handling and logging
+    """
+
+    def __init__(self, tool_router: Any):
+        """Initialize wrapper with underlying tool router.
+
+        Parameters
+        ----------
+        tool_router : Any
+            The underlying tool router to wrap
+        """
+        self._tool_router = tool_router
+
+    async def acall_tool(self, tool_name: str, params: dict[str, Any]) -> Any:
+        """Call tool with observability and policy control.
+
+        This method:
+        1. Emits ToolCalled event (for observability)
+        2. Calls underlying tool
+        3. Emits ToolCompleted event with duration and result
+        4. (Future: Policy evaluation for auth, rate limiting)
+
+        Parameters
+        ----------
+        tool_name : str
+            Name of the tool to call
+        params : dict[str, Any]
+            Tool parameters
+
+        Returns
+        -------
+        Any
+            Tool execution result
+        """
+        node_name = get_current_node_name() or "unknown"
+
+        # Emit tool called event (OBSERVABILITY)
+        if observer_mgr := get_observer_manager():
+            await observer_mgr.notify(
+                ToolCalled(node_name=node_name, tool_name=tool_name, params=params)
+            )
+
+        # TODO(hexdag-team): Add policy evaluation here (CONTROL) #noqa: TD003
+        # if policy_mgr := get_policy_manager():
+        #     policy_response = await policy_mgr.evaluate(ToolCallPolicy(...))
+        #     if policy_response.signal == PolicySignal.FAIL:
+        #         raise AuthorizationError(f"Tool '{tool_name}' not authorized")
+
+        # Call underlying tool
+        start_time = time.perf_counter()
+        try:
+            result = await self._tool_router.acall_tool(tool_name, params)
+            duration_ms = (time.perf_counter() - start_time) * 1000
+
+            # Emit tool completed event (OBSERVABILITY)
+            if observer_mgr := get_observer_manager():
+                await observer_mgr.notify(
+                    ToolCompleted(
+                        node_name=node_name,
+                        tool_name=tool_name,
+                        result=result,
+                        duration_ms=duration_ms,
+                    )
+                )
+
+            return result
+
+        except Exception as e:
+            duration_ms = (time.perf_counter() - start_time) * 1000
+            logger.error(f"Tool '{tool_name}' failed in {duration_ms:.2f}ms: {e}")
+
+            # Still emit completed event with error info
+            if observer_mgr := get_observer_manager():
+                await observer_mgr.notify(
+                    ToolCompleted(
+                        node_name=node_name,
+                        tool_name=tool_name,
+                        result={"error": str(e)},
+                        duration_ms=duration_ms,
+                    )
+                )
+
+            raise
+
+    def __getattr__(self, name: str) -> Any:
+        """Forward all other attribute access to underlying tool router."""
+        return getattr(self._tool_router, name)
+
+
+def wrap_llm_port(llm: Any) -> Any:
+    """Wrap LLM port with event emission if it implements the LLM protocol.
+
+    Parameters
+    ----------
+    llm : Any
+        The LLM port to potentially wrap
+
+    Returns
+    -------
+    Any
+        Wrapped LLM if it has aresponse method, otherwise original object
+    """
+    if hasattr(llm, "aresponse"):
+        return ObservableLLMWrapper(llm)
+    return llm
+
+
+def wrap_tool_router_port(tool_router: Any) -> Any:
+    """Wrap ToolRouter port with event emission.
+
+    Parameters
+    ----------
+    tool_router : Any
+        The tool router to potentially wrap
+
+    Returns
+    -------
+    Any
+        Wrapped tool router if it has acall_tool method, otherwise original object
+    """
+    if hasattr(tool_router, "acall_tool"):
+        return ObservableToolRouterWrapper(tool_router)
+    return tool_router
+
+
+def wrap_ports_with_observability(ports: dict[str, Any]) -> dict[str, Any]:
+    """Wrap all ports with event-emitting wrappers.
+
+    This wraps:
+    - LLM ports: Emit LLMPromptSent/LLMResponseReceived events
+    - ToolRouter ports: Emit ToolCalled/ToolCompleted events
+    - (Future: Database, Memory, etc.)
+
+    Parameters
+    ----------
+    ports : dict[str, Any]
+        Dictionary of port name to port adapter
+
+    Returns
+    -------
+    dict[str, Any]
+        Dictionary with wrapped ports
+    """
+    wrapped = {}
+    for name, port in ports.items():
+        if name == "llm":
+            wrapped[name] = wrap_llm_port(port)
+        elif name == "tool_router":
+            wrapped[name] = wrap_tool_router_port(port)
+        else:
+            wrapped[name] = port
+    return wrapped

hexdag/core/orchestration/prompt/__init__.py

@@ -0,0 +1,32 @@
+"""Prompt template utilities for the application layer.
+
+This module contains prompt template classes used by application nodes for rendering templated
+prompts with variable substitution.
+
+Note: ChatPromptTemplate, FewShotPromptTemplate, and ChatFewShotTemplate have been moved to
+hexdag.builtin.prompts.base but are re-exported here for backward compatibility.
+"""
+
+# Core template (minimal)
+# Advanced templates (from builtin, re-exported for backward compatibility)
+from hexdag.builtin.prompts.base import (
+    ChatFewShotTemplate,
+    ChatPromptTemplate,
+    FewShotPromptTemplate,
+)
+from hexdag.core.orchestration.prompt.template import PromptTemplate
+
+# Type aliases for template types (used across the framework)
+PromptInput = (
+    str | PromptTemplate | ChatPromptTemplate | ChatFewShotTemplate | FewShotPromptTemplate
+)
+TemplateType = PromptTemplate | ChatPromptTemplate | ChatFewShotTemplate | FewShotPromptTemplate
+
+__all__ = [
+    "PromptTemplate",
+    "FewShotPromptTemplate",
+    "ChatPromptTemplate",
+    "ChatFewShotTemplate",
+    "PromptInput",
+    "TemplateType",
+]