hexdag 0.5.0.dev1__py3-none-any.whl

hexdag/core/pipeline_builder/pipeline_config.py

@@ -0,0 +1,133 @@
+"""Pipeline configuration models for YAML-based component configuration.
+
+This module provides Pydantic models for declaring and configuring:
+- Global ports (adapters)
+- Global policies
+- Per-type port defaults (type_ports)
+- Per-node port/policy overrides
+
+Note
+----
+All configuration dictionaries use native YAML dict format.
+Formats:
+- ports: port_name -> {namespace: str, name: str, params: dict}
+- type_ports: node_type -> {port_name -> {namespace: str, name: str, params: dict}}
+- policies: policy_name -> {namespace: str, name: str, params: dict}
+"""
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class PipelineConfig(BaseModel):
+    """Complete pipeline configuration including ports and policies.
+
+    This model represents the full configuration extracted from YAML
+    that will be used to instantiate and configure the orchestrator.
+
+    Attributes
+    ----------
+    ports : dict[str, dict[str, Any]]
+        Global port (adapter) configurations using native YAML dict format.
+        Format: {namespace: str, name: str, params: dict}
+    type_ports : dict[str, dict[str, dict[str, Any]]]
+        Per-type port defaults. Maps node_type -> {port_name -> dict spec}
+    policies : dict[str, dict[str, Any]]
+        Global policy configurations using native YAML dict format.
+        Format: {namespace: str, name: str, params: dict}
+    metadata : dict[str, Any]
+        Pipeline metadata (name, description, version, etc.)
+    nodes : list[dict[str, Any]]
+        Node specifications (handled by existing builder)
+
+    Examples
+    --------
+    ```yaml
+    apiVersion: hexdag.omniviser.io/v1alpha1
+    kind: Pipeline
+    metadata:
+      name: my-pipeline
+      description: Example pipeline
+
+    spec:
+      # Global ports (adapters) - native YAML dict format
+      ports:
+        llm:
+          namespace: core
+          name: openai
+          params:
+            model: gpt-4
+            temperature: 0.7
+            api_key: ${OPENAI_API_KEY}
+        database:
+          namespace: core
+          name: postgres
+          params:
+            connection_string: ${DB_URL}
+
+      # Per-type port defaults
+      type_ports:
+        agent:
+          llm:
+            namespace: core
+            name: anthropic
+            params:
+              model: claude-3-5-sonnet
+
+      # Global policies - native YAML dict format
+      policies:
+        retry:
+          name: retry
+          params:
+            max_retries: 3
+        timeout:
+          name: timeout
+          params:
+            timeout_seconds: 300
+
+      # Nodes
+      nodes:
+        - kind: core:agent_node
+          metadata:
+            name: researcher
+          spec:
+            initial_prompt_template: "Research: {{topic}}"
+            max_steps: 10
+            # Node-level port override
+            ports:
+              llm:
+                namespace: core
+                name: openai
+                params:
+                  model: gpt-4o
+            # Node-level policy override
+            policies:
+              timeout:
+                name: timeout
+                params:
+                  timeout_seconds: 600
+    ```
+    """
+
+    # Global configuration
+    ports: dict[str, dict[str, Any]] = Field(
+        default_factory=dict, description="Global adapter configurations"
+    )
+    type_ports: dict[str, dict[str, dict[str, Any]]] = Field(
+        default_factory=dict, description="Per-type port defaults"
+    )
+    policies: dict[str, dict[str, Any]] = Field(
+        default_factory=dict, description="Global policy configurations"
+    )
+
+    # Metadata
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Pipeline metadata")
+
+    # Node configurations (handled by existing builder)
+    nodes: list[dict[str, Any]] = Field(default_factory=list, description="Node specifications")
+
+    class Config:
+        """Pydantic model configuration."""
+
+        extra = "allow"  # Allow additional fields for extensibility

hexdag/core/pipeline_builder/py_tag.py

@@ -0,0 +1,223 @@
+"""!py YAML custom tag handler for inline Python functions.
+
+This module provides a YAML custom tag constructor that compiles inline
+Python code into callable functions, enabling inline Python in YAML pipelines.
+
+Security Warning
+----------------
+The !py tag executes arbitrary Python code. Only use with trusted YAML files.
+For untrusted input, use module path strings instead.
+
+Examples
+--------
+YAML usage::
+
+    body: !py |
+      async def process(item, index, state, **ports):
+          '''Process an item with database lookup.'''
+          db = ports.get('database')
+          if db:
+              context = await db.aquery(item['id'])
+          else:
+              context = {}
+          return {"item": item, "context": context}
+
+The function will be compiled and made available as a callable.
+"""
+
+from collections.abc import Callable
+from typing import Any
+
+import yaml
+
+from hexdag.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class PyTagError(Exception):
+    """Error compiling !py tagged Python code."""
+
+    pass
+
+
+def py_constructor(loader: yaml.SafeLoader, node: yaml.ScalarNode) -> Callable[..., Any]:
+    """Compile !py tagged Python code into a callable function.
+
+    The !py block must define exactly one function. The function can be
+    sync or async, and should follow the signature convention:
+
+        async def process(item, index, state, **ports) -> Any:
+            '''Docstring.'''
+            ...
+
+    Parameters
+    ----------
+    loader : yaml.SafeLoader
+        YAML loader instance
+    node : yaml.ScalarNode
+        YAML node containing Python source code
+
+    Returns
+    -------
+    Callable
+        Compiled Python function
+
+    Raises
+    ------
+    PyTagError
+        If compilation fails or no function is defined
+
+    Examples
+    --------
+    The following YAML::
+
+        body: !py |
+          def double(item, index, state, **ports):
+              return item * 2
+
+    Will compile to a callable function that can be invoked.
+    """
+    source_code = loader.construct_scalar(node)
+
+    if not isinstance(source_code, str) or not source_code.strip():
+        raise PyTagError("!py block must be a non-empty string defining a function.")
+
+    # Compile the source code
+    try:
+        compiled = compile(source_code, "<yaml-!py>", "exec")
+    except SyntaxError as e:
+        raise PyTagError(f"Syntax error in !py block at line {e.lineno}: {e.msg}") from e
+    except Exception as e:
+        raise PyTagError(f"Failed to compile !py block: {e}") from e
+
+    # Execute to get the function
+    namespace: dict[str, Any] = {}
+    try:
+        exec(compiled, namespace)  # noqa: S102
+    except Exception as e:
+        raise PyTagError(f"Failed to execute !py block: {e}") from e
+
+    # Find the defined function (first callable in namespace that's not a builtin)
+    func: Callable[..., Any] | None = None
+    for name, obj in namespace.items():
+        if name.startswith("_"):
+            continue
+        if callable(obj) and not isinstance(obj, type):
+            func = obj
+            break
+
+    if func is None:
+        defined = [k for k in namespace if not k.startswith("_")]
+        raise PyTagError(
+            f"!py block must define a function. Found: {defined if defined else 'nothing'}"
+        )
+
+    logger.debug(
+        "Compiled !py function",
+        function_name=getattr(func, "__name__", "<anonymous>"),
+    )
+
+    return func
+
+
+def validate_py_source(source_code: str) -> list[str]:
+    """Validate !py source code without executing it.
+
+    This function performs static validation of Python source code intended
+    for !py tags. It checks syntax and verifies a function is defined.
+
+    Parameters
+    ----------
+    source_code : str
+        Python source code to validate
+
+    Returns
+    -------
+    list[str]
+        List of validation error messages (empty if valid)
+
+    Examples
+    --------
+    Validate valid code::
+
+        >>> errors = validate_py_source("def process(item, index, state, **ports): return item")
+        >>> assert errors == []
+
+    Validate invalid syntax::
+
+        >>> errors = validate_py_source("def process( invalid")
+        >>> assert "Syntax error" in errors[0]
+
+    Validate missing function::
+
+        >>> errors = validate_py_source("x = 1")
+        >>> assert "must define a function" in errors[0]
+    """
+    errors: list[str] = []
+
+    if not source_code or not source_code.strip():
+        errors.append("!py block is empty. Must define a function.")
+        return errors
+
+    # Check syntax by compiling
+    try:
+        compiled = compile(source_code, "<yaml-!py>", "exec")
+    except SyntaxError as e:
+        errors.append(f"Syntax error at line {e.lineno}: {e.msg}")
+        return errors
+    except Exception as e:
+        errors.append(f"Compilation error: {e}")
+        return errors
+
+    # Execute to check for function definition
+    namespace: dict[str, Any] = {}
+    try:
+        exec(compiled, namespace)  # noqa: S102
+    except Exception as e:
+        errors.append(f"Execution error: {e}")
+        return errors
+
+    # Check that a function is defined
+    has_function = False
+    for name, obj in namespace.items():
+        if name.startswith("_"):
+            continue
+        if callable(obj) and not isinstance(obj, type):
+            has_function = True
+            break
+
+    if not has_function:
+        defined = [k for k in namespace if not k.startswith("_")]
+        errors.append(
+            f"!py block must define a function. Found: {defined if defined else 'nothing'}"
+        )
+
+    return errors
+
+
+def register_py_tag() -> None:
+    """Register the !py custom tag with YAML SafeLoader.
+
+    This function should be called during module initialization to enable
+    !py tag support in YAML parsing.
+
+    Examples
+    --------
+    Import the module to auto-register::
+
+        import hexdag.core.pipeline_builder.py_tag  # Registers !py tag
+
+    Or explicitly register::
+
+        from hexdag.core.pipeline_builder.py_tag import register_py_tag
+        register_py_tag()
+    """
+    # Check if already registered to avoid duplicate registration
+    if "!py" not in yaml.SafeLoader.yaml_constructors:
+        yaml.SafeLoader.add_constructor("!py", py_constructor)
+        logger.debug("Registered !py YAML tag")
+
+
+# Auto-register when module is imported
+register_py_tag()

hexdag/core/pipeline_builder/tag_discovery.py

@@ -0,0 +1,268 @@
+"""Auto-discovery and introspection of YAML custom tags.
+
+This module provides functions to discover available YAML custom tags
+and extract documentation from their constructors.
+
+Usage
+-----
+>>> from hexdag.core.pipeline_builder.tag_discovery import discover_tags
+>>> tags = discover_tags()
+>>> tags["!py"]["name"]
+'!py'
+"""
+
+from __future__ import annotations
+
+import importlib
+import inspect
+from functools import lru_cache
+from typing import Any
+
+import yaml
+
+# Tag metadata registry
+# Maps tag name -> (module, constructor_func, description)
+_TAG_REGISTRY: dict[str, tuple[str, str, str]] = {
+    "!py": (
+        "hexdag.core.pipeline_builder.py_tag",
+        "py_constructor",
+        "Compile inline Python code into callable functions",
+    ),
+    "!include": (
+        "hexdag.core.pipeline_builder.include_tag",
+        "include_constructor",
+        "Include content from external YAML files",
+    ),
+}
+
+
+@lru_cache(maxsize=1)
+def discover_tags() -> dict[str, dict[str, Any]]:
+    """Discover all registered YAML custom tags with their metadata.
+
+    Returns
+    -------
+    dict[str, dict[str, Any]]
+        Mapping of tag name to tag info dict with keys:
+        - name: Tag name (e.g., "!py")
+        - module: Full module path
+        - constructor: Constructor function name
+        - description: Short description
+        - documentation: Full docstring
+        - syntax: Syntax patterns
+        - is_registered: Whether tag is registered with YAML SafeLoader
+
+    Examples
+    --------
+    >>> tags = discover_tags()
+    >>> "!py" in tags
+    True
+    >>> tags["!py"]["description"]
+    'Compile inline Python code into callable functions'
+    """
+    result: dict[str, dict[str, Any]] = {}
+
+    for tag_name, (module_path, constructor_name, description) in _TAG_REGISTRY.items():
+        tag_info = _extract_tag_info(tag_name, module_path, constructor_name, description)
+        result[tag_name] = tag_info
+
+    return result
+
+
+def _extract_tag_info(
+    tag_name: str,
+    module_path: str,
+    constructor_name: str,
+    description: str,
+) -> dict[str, Any]:
+    """Extract comprehensive information about a tag.
+
+    Parameters
+    ----------
+    tag_name : str
+        The YAML tag (e.g., "!py")
+    module_path : str
+        Full module path to the tag module
+    constructor_name : str
+        Name of the constructor function
+    description : str
+        Short description
+
+    Returns
+    -------
+    dict[str, Any]
+        Tag information dictionary
+    """
+    tag_info: dict[str, Any] = {
+        "name": tag_name,
+        "module": module_path,
+        "constructor": constructor_name,
+        "description": description,
+        "documentation": "",
+        "syntax": [],
+        "is_registered": tag_name in yaml.SafeLoader.yaml_constructors,
+    }
+
+    try:
+        module = importlib.import_module(module_path)
+        constructor = getattr(module, constructor_name, None)
+
+        if constructor:
+            # Extract full docstring from constructor
+            docstring = inspect.getdoc(constructor) or ""
+            tag_info["documentation"] = docstring
+
+        # Also get module-level docstring for additional context
+        module_doc = inspect.getdoc(module) or ""
+        if module_doc and not tag_info["documentation"]:
+            tag_info["documentation"] = module_doc
+
+        # Extract syntax patterns
+        tag_info["syntax"] = _get_tag_syntax(tag_name)
+
+    except (ImportError, AttributeError) as e:
+        tag_info["error"] = str(e)
+
+    return tag_info
+
+
+def _get_tag_syntax(tag_name: str) -> list[str]:
+    """Get syntax patterns for a tag.
+
+    Parameters
+    ----------
+    tag_name : str
+        The tag name (e.g., "!py")
+
+    Returns
+    -------
+    list[str]
+        List of syntax pattern descriptions
+    """
+    if tag_name == "!py":
+        return [
+            "!py | <python_code>  # Inline Python code block",
+            "The code block must define exactly one function",
+            "Function signature: async def process(item, index, state, **ports)",
+        ]
+    if tag_name == "!include":
+        return [
+            "!include ./path/to/file.yaml  # Simple file inclusion",
+            "!include {path: ./file.yaml, vars: {key: value}}  # With variable substitution",
+            "Variables use {{var}} placeholder syntax in included files",
+        ]
+    return []
+
+
+def get_tag_schema(tag_name: str) -> dict[str, Any]:
+    """Get JSON Schema-like representation of a tag's usage.
+
+    Parameters
+    ----------
+    tag_name : str
+        Tag name (e.g., "!py" or "!include")
+
+    Returns
+    -------
+    dict[str, Any]
+        Schema-like dict with tag information
+
+    Raises
+    ------
+    ValueError
+        If tag is not found
+
+    Examples
+    --------
+    >>> schema = get_tag_schema("!py")
+    >>> schema["name"]
+    '!py'
+    >>> schema["type"]
+    'yaml_tag'
+    """
+    # Normalize tag name (add ! prefix if missing)
+    normalized_name = tag_name if tag_name.startswith("!") else f"!{tag_name}"
+
+    tags = discover_tags()
+    if normalized_name not in tags:
+        available = list(tags.keys())
+        raise ValueError(f"Unknown tag: {tag_name}. Available: {available}")
+
+    tag_info = tags[normalized_name]
+
+    # Build a schema-like representation
+    schema: dict[str, Any] = {
+        "name": tag_info["name"],
+        "type": "yaml_tag",
+        "description": tag_info["description"],
+        "module": tag_info["module"],
+        "documentation": tag_info["documentation"],
+        "syntax": tag_info["syntax"],
+        "is_registered": tag_info["is_registered"],
+    }
+
+    # Add tag-specific schema information
+    if normalized_name == "!py":
+        schema["input_schema"] = {
+            "type": "string",
+            "format": "python_code",
+            "description": "Python source code defining a single function",
+            "examples": ["async def process(item, index, state, **ports):\n    return item * 2"],
+        }
+        schema["output"] = {
+            "type": "callable",
+            "description": "Compiled Python function",
+        }
+        schema["security_warning"] = (
+            "Executes arbitrary Python code. Only use with trusted YAML files."
+        )
+
+    elif normalized_name == "!include":
+        schema["input_schema"] = {
+            "oneOf": [
+                {
+                    "type": "string",
+                    "description": "Path to YAML file to include",
+                    "examples": ["./shared/nodes.yaml"],
+                },
+                {
+                    "type": "object",
+                    "properties": {
+                        "path": {
+                            "type": "string",
+                            "description": "Path to YAML file",
+                        },
+                        "vars": {
+                            "type": "object",
+                            "description": "Variables to substitute using {{var}} syntax",
+                        },
+                    },
+                    "required": ["path"],
+                },
+            ],
+        }
+        schema["output"] = {
+            "type": "any",
+            "description": "Parsed YAML content from the included file",
+        }
+
+    return schema
+
+
+def get_known_tag_names() -> frozenset[str]:
+    """Get all registered tag names for validation.
+
+    Returns
+    -------
+    frozenset[str]
+        Set of all tag names (e.g., {"!py", "!include"})
+
+    Examples
+    --------
+    >>> names = get_known_tag_names()
+    >>> "!py" in names
+    True
+    >>> "!include" in names
+    True
+    """
+    return frozenset(discover_tags().keys())