hexdag 0.5.0.dev1__py3-none-any.whl

hexdag/core/configurable.py

@@ -0,0 +1,294 @@
+"""Configurable interface and base class for plugins and adapters.
+
+This module provides:
+1. ConfigurableComponent - Protocol for components with configuration
+2. ConfigurableAdapter - Base class that implements the protocol and eliminates boilerplate
+3. ConfigurableNode - Base class for node factories with configuration
+4. ConfigurablePolicy - Base class for policies with configuration
+5. ConfigurableMacro - Base class for macros with configuration
+6. SecretField - Helper for declaring secret configuration fields
+"""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+from pydantic import BaseModel, ConfigDict, Field  # noqa: TC002
+
+from hexdag.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def _extract_config_fields(kwargs: dict[str, Any], config_class: type[BaseModel]) -> dict[str, Any]:
+    """Extract config fields from kwargs that match the config class schema.
+
+    Parameters
+    ----------
+    kwargs : dict[str, Any]
+        Keyword arguments containing config values
+    config_class : type[BaseModel]
+        Pydantic model class defining the config schema
+
+    Returns
+    -------
+    dict[str, Any]
+        Dictionary containing only the fields defined in config_class
+    """
+    return {
+        field_name: kwargs[field_name]
+        for field_name in config_class.model_fields
+        if field_name in kwargs
+    }
+
+
+# Legacy Config base classes removed - use simplified decorator pattern instead
+# See CLAUDE.md and SIMPLIFIED_PATTERN.md for migration guide
+
+
+class MacroConfig(BaseModel):
+    """Base configuration class for all macros.
+
+    Macros should define a nested Config class inheriting from this.
+    This enables:
+    - Type-safe configuration
+    - YAML schema generation
+    - Runtime validation
+    - Expansion strategy configuration
+
+    Examples
+    --------
+    Define a macro configuration class::
+
+        class ResearchMacroConfig(MacroConfig):
+            depth: int = 3
+            enable_synthesis: bool = True
+
+        config = ResearchMacroConfig(depth=5)
+        assert config.depth == 5
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+
+def SecretField(
+    env_var: str,
+    memory_key: str | None = None,
+    default: Any = None,
+    description: str | None = None,
+    **field_kwargs: Any,
+) -> Any:
+    """Create a secret field with auto-resolution from Memory or environment.
+
+    This helper marks a Pydantic field as a secret, which enables:
+    1. Auto-hiding in logs/repr (uses Pydantic SecretStr)
+    2. Auto-resolution from Memory (secret:KEY) via orchestrator
+    3. Fallback to environment variable if not in Memory
+    4. Clear documentation of secret requirements
+
+    Parameters
+    ----------
+    env_var : str
+        Environment variable name to read from (e.g., "OPENAI_API_KEY")
+    memory_key : str | None, optional
+        Key to use in Memory port (defaults to env_var).
+        Will be prefixed with "secret:" automatically.
+    default : Any, optional
+        Default value if not found (default: None)
+    description : str | None, optional
+        Field description for schema documentation
+    **field_kwargs : Any
+        Additional Pydantic Field() parameters
+
+    Returns
+    -------
+    Any
+        Pydantic Field with secret metadata (typed as Any for use in assignments)
+
+    Examples
+    --------
+    >>> from pydantic import BaseModel
+    >>> class MyAdapterConfig(BaseModel):
+    ...     api_key: SecretStr | None = SecretField(
+    ...         env_var="OPENAI_API_KEY",
+    ...         description="OpenAI API key"
+    ...     )
+    ...     timeout: float = 30.0
+    """
+    return Field(
+        default=default,
+        description=description,
+        json_schema_extra={
+            "secret": True,
+            "env_var": env_var,
+            "memory_key": memory_key or env_var,
+        },
+        **field_kwargs,
+    )
+
+
+@runtime_checkable
+class ConfigurableComponent(Protocol):
+    """Protocol for components that support configuration.
+
+    Any adapter or plugin that implements this protocol will be
+    automatically discoverable by the CLI config generation system.
+    """
+
+    @classmethod
+    def get_config_class(cls) -> type[BaseModel]:
+        """Return the Pydantic model class that defines configuration schema.
+
+        Returns
+        -------
+        type[BaseModel]
+            A Pydantic model class with field definitions, defaults, and validation
+        """
+        ...
+
+
+class ConfigurableMacro:
+    """Base class for macros with configuration support.
+
+    Macros are pipeline templates that expand into DirectedGraph subgraphs.
+    They are first-class registry components like nodes, adapters, and policies.
+
+    Key Concepts
+    ------------
+    - Macros live at a higher abstraction level than nodes
+    - Nodes are atomic operations (single LLM call, single function)
+    - Macros are compositions (multi-step workflows)
+    - Macros expand to graphs of nodes at build time or runtime
+
+    Architecture
+    ------------
+    ConfigurableMacro provides:
+    1. Type-safe configuration via MacroConfig subclasses
+    2. Consistent expansion interface via expand() method
+    3. Registry integration via @macro decorator
+    4. Support for both static and dynamic expansion strategies
+
+    Subclasses must:
+    - Define a nested Config class inheriting from MacroConfig
+    - Implement expand() method that returns DirectedGraph
+
+    Examples
+    --------
+    >>> from hexdag.core.configurable import ConfigurableMacro, MacroConfig
+    >>> from hexdag.core.domain.dag import DirectedGraph
+    >>>
+    >>> class ResearchMacroConfig(MacroConfig):
+    ...     depth: int = 3
+    ...     enable_synthesis: bool = True
+    >>>
+    >>> class ResearchMacro(ConfigurableMacro):
+    ...     Config = ResearchMacroConfig
+    ...
+    ...     def expand(self, instance_name, inputs, dependencies):
+    ...         # Build and return DirectedGraph
+    ...         return DirectedGraph([...])
+    """
+
+    Config: type[MacroConfig]
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize macro with configuration.
+
+        Parameters
+        ----------
+        **kwargs : Any
+            Configuration options matching Config schema fields
+        """
+        if not hasattr(self.__class__, "Config"):
+            raise AttributeError(
+                f"{self.__class__.__name__} must define a nested Config class (MacroConfig)"
+            )
+
+        config_data = _extract_config_fields(kwargs, self.Config)
+
+        self.config = self.Config(**config_data)
+        self._extra_kwargs = {k: v for k, v in kwargs.items() if k not in self.Config.model_fields}
+
+    def expand(
+        self,
+        instance_name: str,
+        inputs: dict[str, Any],
+        dependencies: list[str],
+    ) -> Any:  # Returns DirectedGraph but avoid circular import
+        """Expand macro into a concrete subgraph.
+
+        This is the core method that subclasses must implement.
+        It transforms the macro template into an actual DirectedGraph
+        with concrete nodes.
+
+        Parameters
+        ----------
+        instance_name : str
+            Unique name for this macro instance (used as prefix for generated nodes).
+            Example: "deep_research" → generates "deep_research_step_1", etc.
+        inputs : dict[str, Any]
+            Input values to bind to macro parameters.
+            Example: {"topic": "AI safety", "depth": 5}
+        dependencies : list[str]
+            External node names that this macro instance depends on.
+            The macro's entry nodes will be connected to these.
+            Example: ["query_parser", "validator"]
+
+        Returns
+        -------
+        DirectedGraph
+            Subgraph containing the expanded nodes with proper dependencies
+
+        Raises
+        ------
+        NotImplementedError
+            If subclass doesn't implement this method
+        ValueError
+            If inputs don't match macro parameter requirements
+        """
+        raise NotImplementedError(f"{self.__class__.__name__} must implement expand() method")
+
+    def validate_inputs(
+        self, inputs: dict[str, Any], required: list[str], optional: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Validate and normalize macro inputs.
+
+        Helper method for subclasses to validate inputs against requirements.
+
+        Parameters
+        ----------
+        inputs : dict[str, Any]
+            Provided input values
+        required : list[str]
+            Names of required input parameters
+        optional : dict[str, Any]
+            Optional parameters with their default values
+
+        Returns
+        -------
+        dict[str, Any]
+            Validated and normalized inputs (with defaults applied)
+
+        Raises
+        ------
+        ValueError
+            If required inputs are missing
+        """
+        # Check required inputs
+        if missing := [name for name in required if name not in inputs]:
+            raise ValueError(
+                f"Missing required inputs for {self.__class__.__name__}: {', '.join(missing)}"
+            )
+
+        # Apply defaults for optional inputs
+        return {**optional, **inputs}
+
+    @classmethod
+    def get_config_class(cls) -> type[BaseModel]:
+        """Get the configuration model class."""
+        return cls.Config
+
+    def __repr__(self) -> str:
+        """Readable representation for debugging."""
+        config_dict = self.config.model_dump() if hasattr(self.config, "model_dump") else {}
+        return f"{self.__class__.__name__}(config={config_dict})"

hexdag/core/context/__init__.py

@@ -0,0 +1,37 @@
+"""Execution context for async-safe cross-cutting concerns."""
+
+from hexdag.core.context.execution_context import (
+    ExecutionContext,
+    clear_execution_context,
+    get_current_graph,
+    get_current_node_name,
+    get_node_results,
+    get_observer_manager,
+    get_port,
+    get_ports,
+    get_run_id,
+    set_current_graph,
+    set_current_node_name,
+    set_node_results,
+    set_observer_manager,
+    set_ports,
+    set_run_id,
+)
+
+__all__ = [
+    "ExecutionContext",
+    "clear_execution_context",
+    "get_current_graph",
+    "get_current_node_name",
+    "get_node_results",
+    "get_observer_manager",
+    "get_port",
+    "get_ports",
+    "get_run_id",
+    "set_current_graph",
+    "set_current_node_name",
+    "set_node_results",
+    "set_observer_manager",
+    "set_ports",
+    "set_run_id",
+]

hexdag/core/context/execution_context.py

@@ -0,0 +1,378 @@
+"""Execution context for orchestrator components.
+
+This module provides async-safe context management for cross-cutting concerns
+like observer management. This eliminates parameter drilling and provides a
+clean way to access these services throughout the execution call stack.
+
+The context is automatically propagated through async call chains, making
+observer_manager and other orchestration services available to all components
+without explicit parameter passing.
+"""
+
+from __future__ import annotations
+
+from contextvars import ContextVar
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from hexdag.core.ports.observer_manager import ObserverManagerPort
+
+# Context variables for orchestrator components (async-safe)
+_observer_manager_context: ContextVar[ObserverManagerPort | None] = ContextVar(
+    "observer_manager", default=None
+)
+
+_run_id_context: ContextVar[str | None] = ContextVar("run_id", default=None)
+
+# Ports stored as immutable MappingProxyType to prevent race conditions in concurrent execution
+_ports_context: ContextVar[MappingProxyType[str, Any] | None] = ContextVar("ports", default=None)
+
+# Dynamic graph context - for runtime expansion support
+_current_graph_context: ContextVar[Any | None] = ContextVar(
+    "current_graph", default=None
+)  # Any to avoid circular import
+
+# Node results context - for accessing intermediate results during execution
+_node_results_context: ContextVar[dict[str, Any] | None] = ContextVar("node_results", default=None)
+
+# Current node name context - for event emission with proper node attribution
+_current_node_name_context: ContextVar[str | None] = ContextVar("current_node_name", default=None)
+
+
+# ============================================================================
+# Observer Manager Context
+# ============================================================================
+
+
+def set_observer_manager(manager: ObserverManagerPort | None) -> None:
+    """Set observer manager for current async execution context.
+
+    This should be called by the orchestrator at the start of DAG execution.
+    All components within this context will automatically have access to
+    the observer manager for event emission.
+
+    Parameters
+    ----------
+    manager : ObserverManagerPort | None
+        Observer manager instance, or None to clear context
+
+    Examples
+    --------
+    Example usage::
+
+        # In orchestrator
+        observer_manager = ports.get("observer_manager")
+        set_observer_manager(observer_manager)
+        # Now all components can emit events
+    """
+    _observer_manager_context.set(manager)
+
+
+def get_observer_manager() -> ObserverManagerPort | None:
+    """Get observer manager from current async execution context.
+
+    This is called by components to access the observer manager for event
+    emission. Returns None if not set or not in orchestrator context.
+
+    Returns
+    -------
+    ObserverManagerPort | None
+        Current observer manager, or None if not in orchestrator context
+
+    Examples
+    --------
+    Example usage::
+
+        # In any component
+        if (observer_manager := get_observer_manager()):
+            await observer_manager.notify(NodeStarted(...))
+    """
+    return _observer_manager_context.get()
+
+
+# ============================================================================
+# Run ID Context
+# ============================================================================
+
+
+def set_run_id(run_id: str | None) -> None:
+    """Set run ID for current async execution context.
+
+    Parameters
+    ----------
+    run_id : str | None
+        Unique run identifier for this execution
+    """
+    _run_id_context.set(run_id)
+
+
+def get_run_id() -> str | None:
+    """Get run ID from current async execution context.
+
+    Returns
+    -------
+    str | None
+        Current run ID, or None if not in orchestrator context
+    """
+    return _run_id_context.get()
+
+
+# ============================================================================
+# Ports Context
+# ============================================================================
+
+
+def set_ports(ports: dict[str, Any] | None) -> None:
+    """Set ports dict for current async execution context.
+
+    This allows adapters and components to access ports without explicit passing.
+
+    IMPORTANT: Ports are stored as immutable MappingProxyType to prevent race
+    conditions when multiple nodes execute concurrently.
+
+    Parameters
+    ----------
+    ports : dict[str, Any] | None
+        Ports dictionary containing all available adapters and services
+    """
+    # Wrap in MappingProxyType to make immutable and prevent concurrent modification
+    _ports_context.set(MappingProxyType(ports) if ports else None)
+
+
+def get_ports() -> MappingProxyType[str, Any] | None:
+    """Get immutable ports mapping from current async execution context.
+
+    Returns
+    -------
+    MappingProxyType[str, Any] | None
+        Immutable view of ports dictionary, or None if not in orchestrator context
+    """
+    return _ports_context.get()
+
+
+def get_port(port_name: str) -> Any:
+    """Get a specific port from current async execution context.
+
+    Parameters
+    ----------
+    port_name : str
+        Name of the port to retrieve (e.g., "llm", "database", "memory")
+
+    Returns
+    -------
+    Any
+        The port adapter, or None if not found or not in orchestrator context
+
+    Examples
+    --------
+    Example usage::
+
+        # In any component
+        if (llm := get_port("llm")):
+            response = await llm.aresponse(messages)
+    """
+    ports = _ports_context.get()
+    if ports is None:
+        return None
+    return ports.get(port_name)
+
+
+# ============================================================================
+# Current Node Name Context
+# ============================================================================
+
+
+def set_current_node_name(node_name: str | None) -> None:
+    """Set current node name for event attribution.
+
+    Parameters
+    ----------
+    node_name : str | None
+        Name of the currently executing node, or None to clear
+    """
+    _current_node_name_context.set(node_name)
+
+
+def get_current_node_name() -> str | None:
+    """Get current node name from execution context.
+
+    Returns
+    -------
+    str | None
+        Name of currently executing node, or None if not set
+    """
+    return _current_node_name_context.get()
+
+
+# ============================================================================
+# Batch Context Management
+# ============================================================================
+
+
+class ExecutionContext:
+    """Context manager for setting up orchestrator execution context.
+
+    This provides a clean way to set all execution context variables at once
+    using a context manager pattern.
+
+    Important
+    ---------
+    The context is automatically cleaned up on exit via __aexit__. This means:
+
+    1. **All async operations (observers, hooks) MUST complete before context exit**
+    2. Observer notifications should always use `await` before exiting context
+    3. Post-DAG hooks must execute INSIDE the context, not after
+    4. Do not create fire-and-forget tasks that outlive the context
+
+    The orchestrator correctly handles this by awaiting all notifications and
+    executing post-hooks inside the context block before cleanup.
+
+    Examples
+    --------
+    Example usage::
+
+        async with ExecutionContext(
+            observer_manager=observer,
+            run_id="run-123",
+            ports=all_ports
+        ):
+            # All components can access context
+            result = await execute_dag(dag, inputs)
+            # IMPORTANT: All observer notifications and hooks complete here
+        # Context cleaned up here - observers no longer accessible
+    """
+
+    def __init__(
+        self,
+        observer_manager: ObserverManagerPort | None = None,
+        run_id: str | None = None,
+        ports: dict[str, Any] | None = None,
+    ):
+        """Initialize execution context.
+
+        Parameters
+        ----------
+        observer_manager : ObserverManagerPort | None
+            Observer manager for event emission
+        run_id : str | None
+            Unique run identifier
+        ports : dict[str, Any] | None
+            Ports dictionary with all adapters
+        """
+        self.observer_manager = observer_manager
+        self.run_id = run_id
+        self.ports = ports
+
+    def __enter__(self) -> ExecutionContext:
+        """Set up execution context (sync context manager)."""
+        set_observer_manager(self.observer_manager)
+        set_run_id(self.run_id)
+        set_ports(self.ports)
+        return self
+
+    def __exit__(self, _exc_type: Any, _exc_val: Any, _exc_tb: Any) -> None:
+        """Clean up execution context (sync context manager)."""
+        clear_execution_context()
+
+    async def __aenter__(self) -> ExecutionContext:
+        """Set up execution context (async context manager)."""
+        set_observer_manager(self.observer_manager)
+        set_run_id(self.run_id)
+        set_ports(self.ports)
+        return self
+
+    async def __aexit__(self, _exc_type: Any, _exc_val: Any, _exc_tb: Any) -> None:
+        """Clean up execution context (async context manager)."""
+        clear_execution_context()
+
+
+# ============================================================================
+# Cleanup
+# ============================================================================
+
+
+def clear_execution_context() -> None:
+    """Clear all execution context variables.
+
+    Useful for cleanup after orchestrator execution or in tests.
+    """
+    _observer_manager_context.set(None)
+    _run_id_context.set(None)
+    _ports_context.set(None)
+    _current_graph_context.set(None)
+    _node_results_context.set(None)
+
+
+# ============================================================================
+# Dynamic Graph Context (for runtime expansion)
+# ============================================================================
+
+
+def set_current_graph(graph: Any) -> None:
+    """Set current graph for dynamic expansion.
+
+    This allows expander nodes to access and modify the graph during execution.
+    Only used when executing DynamicDirectedGraph.
+
+    Parameters
+    ----------
+    graph : Any
+        The current graph being executed (typically DynamicDirectedGraph)
+    """
+    _current_graph_context.set(graph)
+
+
+def get_current_graph() -> Any | None:
+    """Get current graph from execution context.
+
+    Used by expander nodes to inject new nodes during runtime.
+
+    Returns
+    -------
+    Any | None
+        Current graph, or None if not in dynamic execution context
+
+    Examples
+    --------
+    >>> # In an expander node
+    >>> graph = get_current_graph()
+    >>> if graph and hasattr(graph, 'add'):
+    ...     new_node = create_next_step()
+    ...     graph.add(new_node)
+    """
+    return _current_graph_context.get()
+
+
+def set_node_results(results: dict[str, Any]) -> None:
+    """Set accumulated node results for dynamic expansion.
+
+    This allows expander nodes to inspect previous results when deciding
+    whether to expand the graph.
+
+    Parameters
+    ----------
+    results : dict[str, Any]
+        Dictionary mapping node names to their execution results
+    """
+    _node_results_context.set(results)
+
+
+def get_node_results() -> dict[str, Any] | None:
+    """Get accumulated node results from execution context.
+
+    Returns
+    -------
+    dict[str, Any] | None
+        Node results dictionary, or None if not in execution context
+
+    Examples
+    --------
+    >>> # In an expander node
+    >>> results = get_node_results()
+    >>> if results:
+    ...     previous_step = results.get("step_1")
+    ...     if should_continue(previous_step):
+    ...         inject_next_step()
+    """
+    return _node_results_context.get()