hexdag 0.5.0.dev1__py3-none-any.whl

hexdag/core/pipeline_builder/component_instantiator.py

@@ -0,0 +1,386 @@
+"""Component instantiator for creating adapters and policies from YAML specs.
+
+This module handles:
+- Parsing native YAML dict component specifications
+- Resolving components via module paths
+- Instantiating adapters with configuration
+- Instantiating policies with configuration
+- Runtime resolution of deferred environment variables
+"""
+
+import os
+import re
+from collections import namedtuple
+from typing import Any
+
+from hexdag.core.logging import get_logger
+from hexdag.core.resolver import resolve
+
+logger = get_logger(__name__)
+
+# Pattern for deferred environment variables: ${VAR} or ${VAR:default}
+_DEFERRED_ENV_VAR_PATTERN = re.compile(r"\$\{([A-Z_][A-Z0-9_]*)(?::([^}]*))?\}")
+
+# Simple namedtuple for component specification
+ComponentSpec = namedtuple("ComponentSpec", ["module_path", "params"])
+
+
+class ComponentInstantiationError(Exception):
+    """Error instantiating component from specification."""
+
+    pass
+
+
+def _resolve_deferred_env_vars(params: dict[str, Any]) -> dict[str, Any]:
+    """Resolve deferred ${VAR} syntax in parameters at runtime.
+
+    This completes the deferred secret resolution workflow where secrets like
+    ${OPENAI_API_KEY} are preserved at YAML build-time and resolved here
+    at adapter instantiation time.
+
+    Parameters
+    ----------
+    params : dict[str, Any]
+        Parameters potentially containing ${VAR} or ${VAR:default} syntax
+
+    Returns
+    -------
+    dict[str, Any]
+        Parameters with environment variables resolved
+
+    Note
+    ----
+    Raises ComponentInstantiationError (via _resolve_string_value) if a
+    required environment variable is not set and has no default.
+
+    Examples
+    --------
+    >>> import os
+    >>> os.environ["TEST_KEY"] = "secret123"
+    >>> _resolve_deferred_env_vars({"api_key": "${TEST_KEY}"})
+    {'api_key': 'secret123'}
+    >>> _resolve_deferred_env_vars({"model": "${MODEL:gpt-4}"})
+    {'model': 'gpt-4'}
+    """
+    resolved: dict[str, Any] = {}
+    for key, value in params.items():
+        if isinstance(value, str):
+            resolved[key] = _resolve_string_value(value)
+        elif isinstance(value, dict):
+            resolved[key] = _resolve_deferred_env_vars(value)
+        elif isinstance(value, list):
+            resolved[key] = [
+                _resolve_string_value(v)
+                if isinstance(v, str)
+                else _resolve_deferred_env_vars(v)
+                if isinstance(v, dict)
+                else v
+                for v in value
+            ]
+        else:
+            resolved[key] = value
+    return resolved
+
+
+def _resolve_string_value(value: str) -> str:
+    """Resolve ${VAR} or ${VAR:default} patterns in a string value.
+
+    Parameters
+    ----------
+    value : str
+        String potentially containing ${VAR} patterns
+
+    Returns
+    -------
+    str
+        String with all ${VAR} patterns resolved
+
+    Note
+    ----
+    Raises ComponentInstantiationError (via nested replacer) if a required
+    environment variable is not set and has no default.
+    """
+
+    def replacer(match: re.Match[str]) -> str:
+        var_name = match.group(1)
+        default = match.group(2)
+        env_value = os.environ.get(var_name)
+
+        if env_value is None:
+            if default is not None:
+                logger.debug(f"Using default value for ${{{var_name}}}")
+                return default
+            raise ComponentInstantiationError(
+                f"Environment variable '{var_name}' is not set and no default provided. "
+                f"Set the environment variable or use ${{​{var_name}:default}} syntax."
+            )
+
+        logger.debug(f"Resolved ${{{var_name}}} from environment")
+        return env_value
+
+    return _DEFERRED_ENV_VAR_PATTERN.sub(replacer, value)
+
+
+class ComponentInstantiator:
+    """Instantiates adapters and policies from native YAML dict specifications."""
+
+    def __init__(self) -> None:
+        """Initialize component instantiator."""
+        pass  # No bootstrap needed anymore
+
+    def _parse_component_spec(self, spec: dict[str, Any]) -> ComponentSpec:
+        """Parse component specification from native YAML dict format.
+
+        Parameters
+        ----------
+        spec : dict[str, Any]
+            Component specification with either:
+            - "adapter": "module.path.ClassName" and optional "config": {...}
+            - "name": "module.path.ClassName" and optional "params": {...}
+
+        Returns
+        -------
+        ComponentSpec
+            Parsed component specification with module_path and params
+
+        Raises
+        ------
+        ComponentInstantiationError
+            If specification format is invalid
+        """
+        if not isinstance(spec, dict):
+            raise ComponentInstantiationError(
+                f"Component specification must be a dict, got {type(spec).__name__}. "
+                f"Use format: {{adapter: 'module.path.Class', config: {{...}}}}"
+            )
+
+        # Support both "adapter" and "name" keys for module path
+        module_path = spec.get("adapter") or spec.get("name")
+        # Support both "config" and "params" keys for parameters
+        params = spec.get("config") or spec.get("params", {})
+
+        if not module_path:
+            raise ComponentInstantiationError(
+                f"Component specification requires 'adapter' or 'name' field. Got: {spec}"
+            )
+
+        return ComponentSpec(module_path=module_path, params=params)
+
+    def instantiate_adapter(self, spec: dict[str, Any], port_name: str | None = None) -> Any:
+        """Instantiate an adapter from native YAML dict specification.
+
+        Parameters
+        ----------
+        spec : dict[str, Any]
+            Adapter specification: {"adapter": "hexdag.builtin.adapters.mock.MockLLM",
+            "config": {"model": "gpt-4"}}
+        port_name : str | None
+            Optional port name for context in error messages
+
+        Returns
+        -------
+        Any
+            Instantiated adapter instance
+
+        Raises
+        ------
+        ComponentInstantiationError
+            If adapter cannot be instantiated
+
+        Examples
+        --------
+        >>> instantiator = ComponentInstantiator()  # doctest: +SKIP
+        >>> adapter = instantiator.instantiate_adapter({  # doctest: +SKIP
+        ...     "adapter": "hexdag.builtin.adapters.mock.MockLLM",
+        ...     "config": {"model": "gpt-4"}
+        ... })
+        """
+        try:
+            # Parse the specification
+            component_spec = self._parse_component_spec(spec)
+
+            try:
+                adapter_class = resolve(component_spec.module_path)
+            except Exception as e:
+                raise ComponentInstantiationError(
+                    f"Adapter '{component_spec.module_path}' "
+                    f"could not be resolved. "
+                    f"Make sure the module path is correct. Error: {e}"
+                ) from e
+
+            # Instantiate the adapter class with parameters
+            if isinstance(adapter_class, type):
+                try:
+                    # Resolve any deferred environment variables at instantiation time
+                    resolved_params = _resolve_deferred_env_vars(component_spec.params)
+                    adapter_instance = adapter_class(**resolved_params)
+                    logger.info(
+                        f"Instantiated adapter '{component_spec.module_path}' "
+                        f"for port '{port_name}'"
+                    )
+                    return adapter_instance
+                except Exception as e:
+                    raise ComponentInstantiationError(
+                        f"Failed to instantiate adapter "
+                        f"'{component_spec.module_path}' "
+                        f"with params {component_spec.params}. Error: {e}"
+                    ) from e
+            else:
+                # It's already an instance (runtime-registered non-class component)
+                if component_spec.params:  # type: ignore[unreachable]
+                    logger.warning(
+                        f"Adapter '{component_spec.module_path}' "
+                        f"resolved to an instance. Parameters {component_spec.params} "
+                        f"will be ignored."
+                    )
+                logger.info(
+                    f"Using resolved adapter instance "
+                    f"'{component_spec.module_path}' "
+                    f"for port '{port_name}'"
+                )
+                return adapter_class
+
+        except ComponentInstantiationError:
+            raise
+        except Exception as e:
+            raise ComponentInstantiationError(
+                f"Failed to instantiate adapter for port '{port_name}': {e}"
+            ) from e
+
+    def instantiate_policy(self, spec: dict[str, Any], policy_name: str | None = None) -> Any:
+        """Instantiate a policy from native YAML dict specification.
+
+        Parameters
+        ----------
+        spec : dict[str, Any]
+            Policy specification: {"name": "hexdag.builtin.policies.RetryPolicy",
+            "params": {"max_retries": 3}}
+        policy_name : str | None
+            Optional policy name for context in error messages
+
+        Returns
+        -------
+        Any
+            Instantiated policy instance
+
+        Raises
+        ------
+        ComponentInstantiationError
+            If policy cannot be instantiated
+
+        Examples
+        --------
+        >>> instantiator = ComponentInstantiator()  # doctest: +SKIP
+        >>> policy = instantiator.instantiate_policy({  # doctest: +SKIP
+        ...     "name": "hexdag.builtin.policies.execution_policies.RetryPolicy",
+        ...     "params": {"max_retries": 5}
+        ... })
+        """
+        try:
+            # Parse the specification
+            component_spec = self._parse_component_spec(spec)
+
+            try:
+                policy_class = resolve(component_spec.module_path)
+            except Exception as e:
+                raise ComponentInstantiationError(
+                    f"Policy '{component_spec.module_path}' "
+                    f"could not be resolved. "
+                    f"Make sure the module path is correct. Error: {e}"
+                ) from e
+
+            if not isinstance(policy_class, type):
+                raise ComponentInstantiationError(
+                    f"Policy '{component_spec.module_path}' resolved to an instance, "
+                    f"not a class. Cannot instantiate from instance."
+                )
+
+            # Instantiate with parameters
+            try:
+                # Resolve any deferred environment variables at instantiation time
+                resolved_params = _resolve_deferred_env_vars(component_spec.params)
+                policy_instance = policy_class(**resolved_params)
+                logger.info(f"Instantiated policy '{component_spec.module_path}' ('{policy_name}')")
+                return policy_instance
+            except Exception as e:
+                raise ComponentInstantiationError(
+                    f"Failed to instantiate policy "
+                    f"'{component_spec.module_path}' "
+                    f"with params {component_spec.params}. Error: {e}"
+                ) from e
+
+        except ComponentInstantiationError:
+            raise
+        except Exception as e:
+            raise ComponentInstantiationError(
+                f"Failed to instantiate policy '{policy_name}': {e}"
+            ) from e
+
+    def instantiate_ports(self, ports_config: dict[str, dict[str, Any]]) -> dict[str, Any]:
+        """Instantiate all ports (adapters) from configuration.
+
+        Parameters
+        ----------
+        ports_config : dict[str, dict[str, Any]]
+            Dictionary of port_name -> adapter dict spec
+
+        Returns
+        -------
+        dict[str, Any]
+            Dictionary of port_name -> adapter_instance
+
+        Examples
+        --------
+        >>> instantiator = ComponentInstantiator()  # doctest: +SKIP
+        >>> config = {  # doctest: +SKIP
+        ...     "llm": {"adapter": "hexdag.builtin.adapters.mock.MockLLM",
+        ...             "config": {"model": "gpt-4"}},
+        ...     "database": {"adapter": "hexdag.builtin.adapters.mock.MockDatabaseAdapter",
+        ...                  "config": {}}
+        ... }
+        """
+        ports: dict[str, Any] = {}
+
+        for port_name, adapter_spec in ports_config.items():
+            try:
+                ports[port_name] = self.instantiate_adapter(adapter_spec, port_name=port_name)
+            except ComponentInstantiationError as e:
+                logger.error(f"Failed to instantiate adapter for port '{port_name}': {e}")
+                raise
+
+        return ports
+
+    def instantiate_policies(self, policies_config: dict[str, dict[str, Any]]) -> list[Any]:
+        """Instantiate all policies from configuration.
+
+        Parameters
+        ----------
+        policies_config : dict[str, dict[str, Any]]
+            Dictionary of policy_name -> policy dict spec
+
+        Returns
+        -------
+        list[Any]
+            List of policy instances
+
+        Examples
+        --------
+        >>> instantiator = ComponentInstantiator()  # doctest: +SKIP
+        >>> config = {  # doctest: +SKIP
+        ...     "retry": {"name": "hexdag.builtin.policies.RetryPolicy",
+        ...               "params": {"max_retries": 3}},
+        ...     "timeout": {"name": "hexdag.builtin.policies.TimeoutPolicy",
+        ...                 "params": {"timeout_seconds": 300}}
+        ... }
+        """
+        policies: list[Any] = []
+
+        for policy_name, policy_spec in policies_config.items():
+            try:
+                policy = self.instantiate_policy(policy_spec, policy_name=policy_name)
+                policies.append(policy)
+            except ComponentInstantiationError as e:
+                logger.error(f"Failed to instantiate policy '{policy_name}': {e}")
+                raise
+
+        return policies

hexdag/core/pipeline_builder/include_tag.py

@@ -0,0 +1,265 @@
+"""!include YAML custom tag handler for pipeline composition.
+
+This module provides a YAML custom tag constructor that includes content
+from external YAML files, enabling modular pipeline composition.
+
+Examples
+--------
+Include a list of nodes::
+
+    spec:
+      nodes:
+        - kind: expression_node
+          metadata:
+            name: start
+          spec:
+            expressions:
+              ready: "true"
+
+        # Include nodes from external file
+        - !include ./shared/validation_nodes.yaml
+
+        - kind: llm_node
+          metadata:
+            name: final
+          spec:
+            prompt_template: "Finalize: {{input}}"
+
+Include a partial pipeline fragment::
+
+    # In shared/validation_nodes.yaml
+    - kind: function_node
+      metadata:
+        name: validate_input
+      spec:
+        fn: "myapp.validate"
+
+    - kind: expression_node
+      metadata:
+        name: check_result
+      spec:
+        expressions:
+          valid: "validate_input.success"
+
+Include with variable substitution::
+
+    # Use !include with a mapping for variable substitution
+    - !include
+      path: ./templates/processor.yaml
+      vars:
+        node_name: "custom_processor"
+        timeout: 30
+"""
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from hexdag.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+# Thread-local storage for base path during parsing
+_current_base_path: Path | None = None
+
+
+class IncludeTagError(Exception):
+    """Error including external YAML file."""
+
+    pass
+
+
+def set_include_base_path(base_path: Path | None) -> None:
+    """Set the base path for resolving !include paths.
+
+    This should be called before parsing YAML that may contain !include tags.
+
+    Parameters
+    ----------
+    base_path : Path | None
+        Base directory for resolving relative paths
+    """
+    global _current_base_path
+    _current_base_path = base_path
+    if base_path:
+        logger.debug("Set include base path", base_path=str(base_path))
+
+
+def get_include_base_path() -> Path:
+    """Get the current base path for !include resolution.
+
+    Returns
+    -------
+    Path
+        Current base path (defaults to cwd if not set)
+    """
+    return _current_base_path or Path.cwd()
+
+
+def include_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> Any:
+    """Include content from an external YAML file.
+
+    Supports two forms:
+    1. Simple: !include ./path/to/file.yaml
+    2. With vars: !include {path: ./file.yaml, vars: {key: value}}
+
+    Parameters
+    ----------
+    loader : yaml.SafeLoader
+        YAML loader instance
+    node : yaml.Node
+        YAML node (scalar for simple, mapping for vars)
+
+    Returns
+    -------
+    Any
+        Parsed content from the included file
+
+    Raises
+    ------
+    IncludeTagError
+        If the file cannot be found or parsed
+    """
+    base_path = get_include_base_path()
+
+    # Handle simple scalar form: !include ./path.yaml
+    if isinstance(node, yaml.ScalarNode):
+        include_path = loader.construct_scalar(node)
+        if not isinstance(include_path, str):
+            raise IncludeTagError("!include path must be a string")
+        vars_dict: dict[str, Any] = {}
+
+    # Handle mapping form: !include {path: ./path.yaml, vars: {...}}
+    elif isinstance(node, yaml.MappingNode):
+        # Use deep=True to fully construct nested mappings (like vars)
+        mapping = loader.construct_mapping(node, deep=True)
+        include_path = mapping.get("path")
+        if not include_path:
+            raise IncludeTagError("!include mapping requires 'path' key")
+        vars_dict = mapping.get("vars", {})
+
+    else:
+        raise IncludeTagError(f"!include expects scalar or mapping, got {type(node)}")
+
+    # Resolve path relative to base
+    resolved_path = _resolve_include_path(include_path, base_path)
+
+    logger.debug(
+        "Including file",
+        include_path=include_path,
+        resolved_path=str(resolved_path),
+        has_vars=bool(vars_dict),
+    )
+
+    # Read and parse the file
+    try:
+        content = resolved_path.read_text()
+    except FileNotFoundError as e:
+        raise IncludeTagError(
+            f"Include file not found: {resolved_path}\n"
+            f"  (resolved from '{include_path}' relative to '{base_path}')"
+        ) from e
+    except Exception as e:
+        raise IncludeTagError(f"Failed to read include file {resolved_path}: {e}") from e
+
+    # Apply variable substitution if vars provided
+    if vars_dict:
+        content = _substitute_vars(content, vars_dict)
+
+    # Parse the included content
+    try:
+        # Set base path for nested includes
+        old_base = _current_base_path
+        set_include_base_path(resolved_path.parent)
+        try:
+            result = yaml.safe_load(content)
+        finally:
+            set_include_base_path(old_base)
+    except yaml.YAMLError as e:
+        raise IncludeTagError(f"Failed to parse include file {resolved_path}: {e}") from e
+
+    return result
+
+
+def _resolve_include_path(include_path: str, base_path: Path) -> Path:
+    """Resolve include path relative to base path.
+
+    Parameters
+    ----------
+    include_path : str
+        Path from !include tag (may be relative or absolute)
+    base_path : Path
+        Base directory for relative paths
+
+    Returns
+    -------
+    Path
+        Resolved absolute path
+    """
+    path = Path(include_path)
+
+    # If absolute, use as-is
+    if path.is_absolute():
+        return path
+
+    # Resolve relative to base path
+    return (base_path / path).resolve()
+
+
+def _substitute_vars(content: str, vars_dict: dict[str, Any]) -> str:
+    """Substitute variables in content using {{var}} syntax.
+
+    Parameters
+    ----------
+    content : str
+        YAML content with {{var}} placeholders
+    vars_dict : dict[str, Any]
+        Variable values to substitute
+
+    Returns
+    -------
+    str
+        Content with variables substituted
+    """
+    result = content
+    for key, value in vars_dict.items():
+        placeholder = "{{" + key + "}}"
+        # Convert value to YAML-safe string representation
+        if isinstance(value, str):
+            str_value = value
+        elif isinstance(value, bool):
+            str_value = "true" if value else "false"
+        elif value is None:
+            str_value = "null"
+        else:
+            str_value = str(value)
+        result = result.replace(placeholder, str_value)
+    return result
+
+
+def register_include_tag() -> None:
+    """Register the !include custom tag with YAML SafeLoader.
+
+    This function should be called during module initialization to enable
+    !include tag support in YAML parsing.
+
+    Examples
+    --------
+    Import the module to auto-register::
+
+        import hexdag.core.pipeline_builder.include_tag  # Registers !include tag
+
+    Or explicitly register::
+
+        from hexdag.core.pipeline_builder.include_tag import register_include_tag
+        register_include_tag()
+    """
+    # Check if already registered to avoid duplicate registration
+    if "!include" not in yaml.SafeLoader.yaml_constructors:
+        yaml.SafeLoader.add_constructor("!include", include_constructor)
+        logger.debug("Registered !include YAML tag")
+
+
+# Auto-register when module is imported
+register_include_tag()