hexdag 0.7.0.dev11__tar.gz → 0.7.0.dev13__tar.gz

{hexdag-0.7.0.dev11 → hexdag-0.7.0.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hexdag
-Version: 0.7.0.dev11
+Version: 0.7.0.dev13
 Summary: Lightweight DAG orchestration framework with enterprise pipeline capabilities
 Project-URL: Homepage, https://hexdag.ai
 Project-URL: Repository, https://github.com/omniviser/hexdag
@@ -41,6 +41,7 @@ Requires-Dist: nbformat>=5.9.0; extra == 'all'
 Requires-Dist: openai>=1.0.0; extra == 'all'
 Requires-Dist: pandas>=2.1.0; extra == 'all'
 Requires-Dist: pgvector>=0.3.0; extra == 'all'
+Requires-Dist: redis>=5.0.0; extra == 'all'
 Requires-Dist: sqlalchemy>=2.0.0; extra == 'all'
 Provides-Extra: anthropic
 Requires-Dist: anthropic>=0.25.0; extra == 'anthropic'
@@ -58,6 +59,8 @@ Requires-Dist: nbformat>=5.9.0; extra == 'notebooks'
 Requires-Dist: pandas>=2.1.0; extra == 'notebooks'
 Provides-Extra: openai
 Requires-Dist: openai>=1.0.0; extra == 'openai'
+Provides-Extra: redis
+Requires-Dist: redis>=5.0.0; extra == 'redis'
 Provides-Extra: storage-all
 Requires-Dist: aiomysql>=0.2.0; extra == 'storage-all'
 Requires-Dist: aiosqlite>=0.20.0; extra == 'storage-all'

{hexdag-0.7.0.dev11 → hexdag-0.7.0.dev13}/hexdag/compiler/config_loader.py

@@ -29,6 +29,7 @@ from hexdag.kernel.config.models import (
     HexDAGConfig,
     LoggingConfig,
     ManifestEntry,
+    MCPPermissions,
 )
 from hexdag.kernel.logging import get_logger
 from hexdag.kernel.orchestration.models import OrchestratorConfig
@@ -377,6 +378,15 @@ class ConfigLoader:
             config.caps = DefaultCaps(
                 default_set=caps_data.get("default_set"),
                 deny=caps_data.get("deny"),
+                profiles=caps_data.get("profiles"),
+            )
+
+        # Parse MCP permissions
+        if "mcp_permissions" in data:
+            mcp_data = data["mcp_permissions"]
+            config.mcp_permissions = MCPPermissions(
+                default_profile=mcp_data.get("default_profile", "read-only"),
+                client_profiles=mcp_data.get("client_profiles", {}),
             )
 
         return config

{hexdag-0.7.0.dev11 → hexdag-0.7.0.dev13}/hexdag/compiler/plugins/__init__.py

@@ -1,20 +1,26 @@
 """Entity plugins for the YAML pipeline builder.
 
 These plugins build specific entity types from YAML configuration:
+- AdapterDefinitionPlugin: Process Adapter definitions (kind: Adapter)
 - ConfigDefinitionPlugin: Process Config definitions (kind: Config)
 - MacroDefinitionPlugin: Process Macro definitions (kind: Macro)
 - MacroEntityPlugin: Expand macro invocations into subgraphs
+- MiddlewareDefinitionPlugin: Process Middleware definitions (kind: Middleware)
 - NodeEntityPlugin: Build regular nodes (llm_node, function_node, etc.)
 """
 
+from hexdag.compiler.plugins.adapter_definition import AdapterDefinitionPlugin
 from hexdag.compiler.plugins.config_definition import ConfigDefinitionPlugin
 from hexdag.compiler.plugins.macro_definition import MacroDefinitionPlugin
 from hexdag.compiler.plugins.macro_entity import MacroEntityPlugin
+from hexdag.compiler.plugins.middleware_definition import MiddlewareDefinitionPlugin
 from hexdag.compiler.plugins.node_entity import NodeEntityPlugin
 
 __all__ = [
+    "AdapterDefinitionPlugin",
     "ConfigDefinitionPlugin",
     "MacroDefinitionPlugin",
     "MacroEntityPlugin",
+    "MiddlewareDefinitionPlugin",
     "NodeEntityPlugin",
 ]

hexdag-0.7.0.dev13/hexdag/compiler/plugins/adapter_definition.py

@@ -0,0 +1,140 @@
+"""Plugin for handling Adapter definitions (kind: Adapter).
+
+Standalone ``kind: Adapter`` manifests define reusable adapter configurations
+that can be referenced by name from ``spec.ports.<name>.ref``.
+
+Example YAML::
+
+    apiVersion: hexdag/v1
+    kind: Adapter
+    metadata:
+      name: production-openai
+      description: Production OpenAI with low temperature
+    spec:
+      class: hexdag.stdlib.adapters.openai.OpenAIAdapter
+      config:
+        model: gpt-4o
+        temperature: 0.3
+        api_key: ${OPENAI_API_KEY}
+
+Reference in a pipeline::
+
+    spec:
+      ports:
+        llm:
+          ref: production-openai
+          config:          # optional overrides merged on top
+            temperature: 0.9
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from hexdag.kernel.exceptions import YamlPipelineBuilderError
+from hexdag.kernel.logging import get_logger
+
+if TYPE_CHECKING:
+    from hexdag.compiler.yaml_builder import YamlPipelineBuilder
+    from hexdag.kernel.domain.dag import DirectedGraph, NodeSpec
+
+logger = get_logger(__name__)
+
+# Module-level registry for named adapter definitions
+_adapter_registry: dict[str, dict[str, Any]] = {}
+
+
+def get_adapter_definition(name: str) -> dict[str, Any] | None:
+    """Look up a registered adapter definition by name.
+
+    Parameters
+    ----------
+    name : str
+        Adapter name (from ``metadata.name``)
+
+    Returns
+    -------
+    dict[str, Any] | None
+        Adapter definition dict with ``class`` and ``config`` keys,
+        or None if not registered
+    """
+    return _adapter_registry.get(name)
+
+
+def clear_adapter_registry() -> None:
+    """Clear all registered adapter definitions (for testing)."""
+    _adapter_registry.clear()
+
+
+class AdapterDefinitionPlugin:
+    """Plugin for handling Adapter definitions (kind: Adapter).
+
+    Processes ``kind: Adapter`` YAML documents and registers the
+    adapter configuration for later reference by name.  Does not add
+    nodes to the graph.
+    """
+
+    def can_handle(self, node_config: dict[str, Any]) -> bool:
+        """Handle Adapter kind."""
+        return node_config.get("kind") == "Adapter"
+
+    def build(
+        self, node_config: dict[str, Any], builder: YamlPipelineBuilder, graph: DirectedGraph
+    ) -> NodeSpec | None:
+        """Register named adapter definition.
+
+        Parameters
+        ----------
+        node_config : dict[str, Any]
+            Adapter definition configuration
+        builder : YamlPipelineBuilder
+            Builder instance (unused)
+        graph : DirectedGraph
+            Graph instance (unused — definitions don't add nodes)
+
+        Returns
+        -------
+        None
+            Adapter definitions don't add nodes to the graph
+        """
+        metadata = node_config.get("metadata", {})
+        name = metadata.get("name")
+        if not name:
+            raise YamlPipelineBuilderError("Adapter definition missing 'metadata.name'")
+
+        spec = node_config.get("spec", {})
+        adapter_class = spec.get("class")
+
+        if not adapter_class:
+            raise YamlPipelineBuilderError(
+                f"Adapter '{name}' missing 'spec.class'. "
+                f"Adapter definitions must specify the adapter class module path."
+            )
+
+        if not isinstance(adapter_class, str):
+            raise YamlPipelineBuilderError(
+                f"Adapter '{name}' spec.class must be a module path string, "
+                f"got {type(adapter_class).__name__}"
+            )
+
+        config = spec.get("config", {})
+        if not isinstance(config, dict):
+            raise YamlPipelineBuilderError(
+                f"Adapter '{name}' spec.config must be a dict, got {type(config).__name__}"
+            )
+
+        capabilities = spec.get("capabilities")
+
+        entry: dict[str, Any] = {"class": adapter_class, "config": config}
+        if capabilities is not None:
+            entry["capabilities"] = capabilities
+
+        _adapter_registry[name] = entry
+
+        logger.info(
+            "Registered adapter definition '{}' (class={})",
+            name,
+            adapter_class,
+        )
+
+        return None

{hexdag-0.7.0.dev11 → hexdag-0.7.0.dev13}/hexdag/compiler/system_builder.py

@@ -125,6 +125,12 @@ class SystemBuilder:
                 "ports": spec.get("ports", {}),
                 "policies": spec.get("policies", {}),
                 "services": spec.get("services", {}),
+                "state_machines": spec.get("state_machines", {}),
+                "states": spec.get("states", {}),
+                "on_transition": spec.get("on_transition", {}),
+                "observers": spec.get("observers", []),
+                "memory": spec.get("memory", {}),
+                "gc": spec.get("gc", {}),
             })
         except Exception as e:
             raise SystemBuildError(f"System validation error: {e}") from e
@@ -132,15 +138,23 @@ class SystemBuilder:
         # 5. Validate pipeline paths exist
         self._validate_pipeline_paths(system_config, resolved_base)
 
-        # 6. Validate pipe DAG is acyclic
-        self._validate_no_cycles(system_config)
+        # 6. Validate pipe DAG is acyclic (only for non-lifecycle systems)
+        if not system_config.is_lifecycle:
+            self._validate_no_cycles(system_config)
+
+        # 7. Validate lifecycle constraints (if state machines declared)
+        if system_config.is_lifecycle:
+            self._validate_lifecycle(system_config)
 
         system_name = metadata.get("name", "unnamed")
+        mode = "lifecycle" if system_config.is_lifecycle else "dag"
         logger.info(
-            "Built system '{}': {} processes, {} pipes",
+            "Built system '{}' ({}): {} processes, {} pipes, {} state machines",
             system_name,
+            mode,
             len(system_config.processes),
             len(system_config.pipes),
+            len(system_config.state_machines),
         )
 
         return system_config
@@ -201,6 +215,75 @@ class SystemBuilder:
                 "Cycle detected in pipe DAG. Processes must form a directed acyclic graph."
             )
 
+    @staticmethod
+    def _validate_lifecycle(config: SystemConfig) -> None:
+        """Validate lifecycle-mode constraints.
+
+        Ensures that ``states`` and ``on_transition`` reference valid
+        process names and valid state machine states.
+
+        Raises
+        ------
+        SystemBuildError
+            If lifecycle configuration is invalid.
+        """
+        process_names = set(config.process_names)
+
+        # Collect all valid states from state machines
+        all_states: set[str] = set()
+        for sm_spec in config.state_machines.values():
+            transitions = sm_spec.get("transitions", {})
+            for from_state, targets in transitions.items():
+                all_states.add(from_state)
+                if isinstance(targets, list):
+                    for t in targets:
+                        if isinstance(t, str):
+                            all_states.add(t)
+                        elif isinstance(t, dict) and "to" in t:
+                            all_states.add(t["to"])
+            initial = sm_spec.get("initial")
+            if initial:
+                all_states.add(initial)
+
+        # Validate spec.states references
+        for state_name, state_spec in config.states.items():
+            if state_name not in all_states:
+                raise SystemBuildError(
+                    f"spec.states references unknown state '{state_name}'. "
+                    f"Valid states: {sorted(all_states)}"
+                )
+            on_enter = state_spec.get("on_enter")
+            if on_enter and on_enter not in process_names:
+                raise SystemBuildError(
+                    f"State '{state_name}' references unknown process "
+                    f"'{on_enter}'. Declared processes: {sorted(process_names)}"
+                )
+
+        # Validate spec.on_transition references
+        for transition_key, transition_spec in config.on_transition.items():
+            parts = [p.strip() for p in transition_key.split("->")]
+            if len(parts) != 2:  # noqa: PLR2004
+                raise SystemBuildError(
+                    f"Invalid on_transition key '{transition_key}'. "
+                    "Expected format: 'FROM_STATE -> TO_STATE'"
+                )
+            from_state, to_state = parts
+            if from_state not in all_states:
+                raise SystemBuildError(
+                    f"on_transition '{transition_key}' references unknown state '{from_state}'"
+                )
+            if to_state not in all_states:
+                raise SystemBuildError(
+                    f"on_transition '{transition_key}' references unknown state '{to_state}'"
+                )
+            process_name = transition_spec.get("process")
+            if process_name and process_name not in process_names:
+                raise SystemBuildError(
+                    f"on_transition '{transition_key}' references unknown "
+                    f"process '{process_name}'. "
+                    f"Declared processes: {sorted(process_names)}"
+                )
+
     @staticmethod
     def topological_order(config: SystemConfig) -> list[str]:
         """Compute topological execution order from the pipe DAG.

{hexdag-0.7.0.dev11 → hexdag-0.7.0.dev13}/hexdag/compiler/yaml_builder.py

@@ -20,6 +20,7 @@ import yaml
 
 # Re-export extracted classes for backward compatibility.
 # Existing code that imports from ``yaml_builder`` continues to work.
+from hexdag.compiler.plugins.adapter_definition import AdapterDefinitionPlugin  # noqa: F401
 from hexdag.compiler.plugins.config_definition import ConfigDefinitionPlugin  # noqa: F401
 from hexdag.compiler.plugins.macro_definition import MacroDefinitionPlugin  # noqa: F401
 from hexdag.compiler.plugins.macro_entity import MacroEntityPlugin  # noqa: F401
@@ -158,6 +159,7 @@ class YamlPipelineBuilder:
 
         # Entity plugins (build specific entity types)
         self.entity_plugins.append(ConfigDefinitionPlugin())  # Process Config definitions
+        self.entity_plugins.append(AdapterDefinitionPlugin())  # Process Adapter definitions
         self.entity_plugins.append(MiddlewareDefinitionPlugin())  # Process Middleware definitions
         self.entity_plugins.append(MacroDefinitionPlugin())  # Process Macro definitions
         self.entity_plugins.append(MacroEntityPlugin())  # Then macro invocations
@@ -230,6 +232,13 @@ class YamlPipelineBuilder:
                     if not isinstance(plugin, TemplatePlugin):
                         processed_doc = plugin.process(processed_doc)
                 self._process_config_definition(processed_doc)
+            elif kind == "Adapter":
+                # Process Adapter definition
+                processed_doc = doc
+                for plugin in self.preprocess_plugins:
+                    if not isinstance(plugin, TemplatePlugin):
+                        processed_doc = plugin.process(processed_doc)
+                self._process_adapter_definition(processed_doc)
             elif kind == "Middleware":
                 # Process Middleware definition
                 processed_doc = doc
@@ -308,7 +317,9 @@ class YamlPipelineBuilder:
         """
         # Filter out Config, Macro, and Middleware definitions - they're processed separately
         pipeline_docs = [
-            doc for doc in documents if doc.get("kind") not in ("Macro", "Config", "Middleware")
+            doc
+            for doc in documents
+            if doc.get("kind") not in ("Macro", "Config", "Middleware", "Adapter")
         ]
 
         if not pipeline_docs:
@@ -416,6 +427,27 @@ class YamlPipelineBuilder:
 
         middleware_plugin.build(middleware_doc, self, temp_graph)
 
+    def _process_adapter_definition(self, adapter_doc: dict[str, Any]) -> None:
+        """Process a kind: Adapter document and register its definition.
+
+        Parameters
+        ----------
+        adapter_doc : dict[str, Any]
+            Validated Adapter document
+        """
+        temp_graph = DirectedGraph()
+
+        adapter_plugin = next(
+            (p for p in self.entity_plugins if isinstance(p, AdapterDefinitionPlugin)), None
+        )
+
+        if adapter_plugin is None:
+            raise YamlPipelineBuilderError(
+                "AdapterDefinitionPlugin not found. Cannot process Adapter definitions."
+            )
+
+        adapter_plugin.build(adapter_doc, self, temp_graph)
+
     def _process_macro_definitions(self, macro_configs: list[dict[str, Any]]) -> None:
         """Process macro definitions and register them.
 

{hexdag-0.7.0.dev11 → hexdag-0.7.0.dev13}/hexdag/compiler/yaml_validator.py

@@ -1,10 +1,13 @@
 """YAML Pipeline Validator - Validates pipeline configurations."""
 
+import difflib
 from collections.abc import Iterator
 from typing import Any
 
+from hexdag.compiler.reference_resolver import _BUILTIN_NAMES, _NODE_FIELD_RE, _RESERVED_PREFIXES
 from hexdag.kernel.domain.dag import DirectedGraph
 from hexdag.kernel.domain.pipeline_config import BaseNodeConfig
+from hexdag.kernel.expression_parser import ALLOWED_FUNCTIONS
 
 # Separator for namespace:name format
 NAMESPACE_SEPARATOR = ":"
@@ -319,6 +322,9 @@ class YamlValidator:
         # Reuse cached node_ids and macro_instances for dependency validation
         self._validate_dependencies_with_cache(nodes, result, node_ids, macro_instances)
 
+        # Validate expression/mapping naming to prevent ambiguous resolution
+        self._validate_naming_collisions(nodes, result, node_ids, macro_instances)
+
         return result
 
     def _validate_manifest_structure(self, config: Any, result: ValidationReport) -> None:
@@ -368,6 +374,14 @@ class YamlValidator:
             # Skip pipeline-specific validation for Middleware kind
             return
 
+        if kind == "Adapter":
+            # Adapter has: metadata, spec.class (no nodes)
+            spec = config.get("spec", {})
+            if not isinstance(spec, dict) or "class" not in spec:
+                result.add_error("Adapter definition must contain 'spec.class' field")
+            # Skip pipeline-specific validation for Adapter kind
+            return
+
         # For Pipeline and other kinds, validate spec
         if "spec" not in config:
             result.add_error("Configuration must contain 'spec' field")
@@ -639,3 +653,118 @@ class YamlValidator:
         # Check for cycles using DirectedGraph's public static method
         if cycle_message := DirectedGraph.detect_cycle(dependency_graph):
             result.add_error(cycle_message)
+
+    def _validate_naming_collisions(
+        self,
+        nodes: list[dict[str, Any]],
+        result: ValidationReport,
+        node_ids: set[str],
+        macro_instances: set[str],
+    ) -> None:
+        """Validate that expression variables and input_mapping aliases don't collide.
+
+        Checks (all hard errors):
+        1. Expression variable name == node name
+        2. Expression variable name == builtin function name
+        3. input_mapping alias == node name
+        4. First path segment in input_mapping/expressions is a known reference
+        """
+        builtin_names = frozenset(ALLOWED_FUNCTIONS.keys())
+
+        for node in nodes:
+            node_id = node.get("metadata", {}).get("name")
+            if not node_id:
+                continue
+
+            spec = node.get("spec", {})
+            expressions = spec.get("expressions", {})
+            input_mapping = spec.get("input_mapping", {})
+
+            if not isinstance(expressions, dict):
+                expressions = {}
+            if not isinstance(input_mapping, dict):
+                input_mapping = {}
+
+            # Check 1: Expression variable names vs node names
+            for var_name in expressions:
+                if var_name in node_ids:
+                    result.add_error(
+                        f"Node '{node_id}': Expression variable '{var_name}' "
+                        f"collides with node '{var_name}'. Rename the variable "
+                        f"or the node to avoid ambiguous resolution."
+                    )
+
+            # Check 2: Expression variable names vs builtin functions
+            for var_name in expressions:
+                if var_name in builtin_names:
+                    result.add_error(
+                        f"Node '{node_id}': Expression variable '{var_name}' "
+                        f"collides with built-in function '{var_name}'. "
+                        f"Choose a different variable name."
+                    )
+
+            # Check 3: input_mapping alias vs node names
+            for alias in input_mapping:
+                if alias in node_ids:
+                    result.add_error(
+                        f"Node '{node_id}': input_mapping alias '{alias}' "
+                        f"collides with node '{alias}'. Choose a different alias."
+                    )
+
+            # Check 4: First path segment validation in expressions
+            # Build the set of valid first segments for this node
+            valid_first_segments = (
+                node_ids
+                | set(expressions.keys())
+                | set(input_mapping.keys())
+                | _RESERVED_PREFIXES
+                | _BUILTIN_NAMES
+                | {"input", "state"}
+            )
+
+            for var_name, expr in expressions.items():
+                if not isinstance(expr, str):
+                    continue
+                self._check_first_segments(
+                    expr, var_name, node_id, valid_first_segments, node_ids, macro_instances, result
+                )
+
+            for alias, source in input_mapping.items():
+                if not isinstance(source, str):
+                    continue
+                # Skip $input references and expressions
+                if source.startswith("$input") or source == "$input":
+                    continue
+                self._check_first_segments(
+                    source, alias, node_id, valid_first_segments, node_ids, macro_instances, result
+                )
+
+    @staticmethod
+    def _check_first_segments(
+        text: str,
+        field_name: str,
+        node_id: str,
+        valid_first_segments: set[str],
+        node_ids: set[str],
+        macro_instances: set[str],
+        result: "ValidationReport",
+    ) -> None:
+        """Check that first path segments in text are known references."""
+        for match in _NODE_FIELD_RE.finditer(text):
+            candidate = match.group(1)
+            if candidate in _RESERVED_PREFIXES or candidate in _BUILTIN_NAMES:
+                continue
+            if candidate in valid_first_segments:
+                continue
+            # Check if candidate is a macro-expanded node name
+            if any(candidate.startswith(f"{mi}_") for mi in macro_instances):
+                continue
+            # Unknown first segment — likely a typo
+            close_matches = difflib.get_close_matches(candidate, sorted(node_ids), n=3, cutoff=0.6)
+            suggestion = ""
+            if close_matches:
+                suggestion = f" Did you mean: {', '.join(close_matches)}?"
+            result.add_error(
+                f"Node '{node_id}': Unknown reference '{candidate}' "
+                f"in field '{field_name}'.{suggestion}"
+            )

{hexdag-0.7.0.dev11 → hexdag-0.7.0.dev13}/hexdag/drivers/pipeline_spawner/local.py

@@ -15,6 +15,7 @@ from uuid import uuid4
 from hexdag.kernel.logging import get_logger
 
 if TYPE_CHECKING:
+    from hexdag.kernel.domain.caps import CapSet
     from hexdag.kernel.pipeline_runner import PipelineRunner
 
 logger = get_logger(__name__)
@@ -46,9 +47,11 @@ class LocalPipelineSpawner:
         self,
         runner: PipelineRunner,
         pipeline_dir: str | None = None,
+        cap_set: CapSet | None = None,
     ) -> None:
         self._runner = runner
         self._pipeline_dir = pipeline_dir
+        self._cap_set: CapSet | None = cap_set
         self._runs: dict[str, _RunState] = {}
         self._tasks: dict[str, asyncio.Task[dict[str, Any]]] = {}
 
@@ -62,12 +65,22 @@ class LocalPipelineSpawner:
         parent_run_id: str | None = None,
         wait: bool = False,
         timeout: float | None = None,
+        cap_set: CapSet | None = None,
     ) -> str:
         """Spawn a pipeline run.
 
         If ``wait=True``, blocks until the pipeline completes (or times out).
         Otherwise, runs in the background and returns immediately.
+
+        If ``cap_set`` is provided, the child's effective capabilities are
+        the intersection of the parent spawner's caps and the requested caps
+        (narrowing-only — the child can never have more caps than the parent).
         """
+        # Compute effective child caps via narrowing chain
+        effective_caps = self._narrow_caps(cap_set)
+        # Store for future use when we pass caps to child runners
+        _ = effective_caps  # reserved for child runner integration
+
         run_id = str(uuid4())
         self._runs[run_id] = _RunState(
             run_id=run_id,
@@ -160,6 +173,21 @@ class LocalPipelineSpawner:
     # Internal
     # ------------------------------------------------------------------
 
+    def _narrow_caps(self, child_caps: CapSet | None) -> CapSet | None:
+        """Compute effective child capabilities via narrowing chain.
+
+        If neither the spawner nor the caller provides caps, returns None
+        (unrestricted). If only one side provides caps, uses that. If both
+        provide caps, intersects (narrows).
+        """
+        if self._cap_set is None and child_caps is None:
+            return None
+        if self._cap_set is None:
+            return child_caps
+        if child_caps is None:
+            return self._cap_set
+        return self._cap_set.intersect(child_caps)
+
     async def _execute(
         self,
         run_id: str,