hexdag 0.7.0.dev13__tar.gz → 0.8.0.dev1__tar.gz

{hexdag-0.7.0.dev13 → hexdag-0.8.0.dev1}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.4
 Name: hexdag
-Version: 0.7.0.dev13
-Summary: Lightweight DAG orchestration framework with enterprise pipeline capabilities
+Version: 0.8.0.dev1
+Summary: Operating system for AI agents — pipelines, ports, drivers, and stdlib so agents don't reinvent infrastructure
 Project-URL: Homepage, https://hexdag.ai
 Project-URL: Repository, https://github.com/omniviser/hexdag
 Project-URL: Documentation, https://hexdag.ai/docs
 Author-email: hexDAG Team <developers@omniviser.ai>
 License: Apache-2.0
 License-File: LICENSE
-Keywords: async,dag,hexagonal-architecture,orchestration,pipeline,workflow
+Keywords: ai-agents,async,dag,llm,operating-system,orchestration,pipeline,workflow
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.12
+Requires-Dist: aiofiles>=23.0.0
 Requires-Dist: jinja2>=3.1.0
 Requires-Dist: loguru>=0.7.0
 Requires-Dist: orjson>=3.9.0

{hexdag-0.7.0.dev13 → hexdag-0.8.0.dev1}/hexdag/__init__.py

@@ -25,8 +25,10 @@ from hexdag.kernel import (
     FewShotPromptTemplate,
     PipelineRunner,
     PromptTemplate,
+    System,
     ToolRouter,
 )
+from hexdag.kernel.domain.pipeline_result import PipelineResult
 from hexdag.stdlib.nodes import (
     FunctionNode,
     LLMNode,
@@ -121,7 +123,9 @@ __all__ = [
     # Version
     "__version__",
     # Primary API
+    "PipelineResult",
     "PipelineRunner",
+    "System",
     # Node Factories
     "FunctionNode",
     "LLMNode",

{hexdag-0.7.0.dev13 → hexdag-0.8.0.dev1}/hexdag/api/processes.py

@@ -26,7 +26,6 @@ if TYPE_CHECKING:
     from hexdag.kernel import PipelineSpawner
     from hexdag.stdlib.lib.entity_state import EntityState
     from hexdag.stdlib.lib.process_registry import ProcessRegistry
-    from hexdag.stdlib.lib.scheduler import Scheduler
 
 
 # ---------------------------------------------------------------------------
@@ -119,94 +118,6 @@ async def spawn_pipeline(
     return {"run_id": run_id, "pipeline_name": pipeline_name, "status": "spawned"}
 
 
-# ---------------------------------------------------------------------------
-# Scheduler tools
-# ---------------------------------------------------------------------------
-
-
-async def schedule_pipeline(
-    scheduler: Scheduler,
-    pipeline_name: str,
-    initial_input: dict[str, Any] | None = None,
-    *,
-    delay_seconds: float | None = None,
-    interval_seconds: float | None = None,
-    ref_id: str | None = None,
-    ref_type: str | None = None,
-) -> dict[str, Any]:
-    """Schedule a pipeline for delayed or recurring execution.
-
-    Provide ``delay_seconds`` for one-shot or ``interval_seconds`` for
-    recurring. If both given, recurring takes precedence.
-
-    Args
-    ----
-        scheduler: Scheduler instance.
-        pipeline_name: Name of the pipeline to execute.
-        initial_input: Input data for the pipeline.
-        delay_seconds: Seconds to wait before one-shot execution.
-        interval_seconds: Seconds between recurring executions.
-        ref_id: Business reference ID.
-        ref_type: Business reference type.
-
-    Returns
-    -------
-        Dict with task_id and schedule details.
-    """
-    if interval_seconds is not None:
-        return await scheduler.aschedule_recurring(
-            pipeline_name,
-            initial_input,
-            interval_seconds=interval_seconds,
-            ref_id=ref_id,
-            ref_type=ref_type,
-        )
-    return await scheduler.aschedule_once(
-        pipeline_name,
-        initial_input,
-        delay_seconds=delay_seconds or 0.0,
-        ref_id=ref_id,
-        ref_type=ref_type,
-    )
-
-
-async def cancel_scheduled(
-    scheduler: Scheduler,
-    task_id: str,
-) -> dict[str, Any]:
-    """Cancel a scheduled task.
-
-    Args
-    ----
-        scheduler: Scheduler instance.
-        task_id: The task ID to cancel.
-
-    Returns
-    -------
-        Dict with task_id and cancellation status.
-    """
-    return await scheduler.acancel(task_id)
-
-
-async def list_scheduled(
-    scheduler: Scheduler,
-    *,
-    ref_id: str | None = None,
-) -> list[dict[str, Any]]:
-    """List scheduled tasks, optionally filtered by ref_id.
-
-    Args
-    ----
-        scheduler: Scheduler instance.
-        ref_id: Filter by business reference ID.
-
-    Returns
-    -------
-        List of scheduled task dicts.
-    """
-    return await scheduler.alist_scheduled(ref_id=ref_id)
-
-
 # ---------------------------------------------------------------------------
 # EntityState tools
 # ---------------------------------------------------------------------------

{hexdag-0.7.0.dev13 → hexdag-0.8.0.dev1}/hexdag/api/validation.py

@@ -6,7 +6,10 @@ Delegates to core YamlValidator for structural validation.
 
 from __future__ import annotations
 
-from typing import Any
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pathlib import Path
 
 import yaml
 
@@ -14,7 +17,11 @@ from hexdag.compiler import YamlPipelineBuilder
 from hexdag.compiler.yaml_validator import YamlValidator
 
 
-def validate(yaml_content: str, lenient: bool = False) -> dict[str, Any]:
+def validate(
+    yaml_content: str,
+    lenient: bool = False,
+    base_path: Path | None = None,
+) -> dict[str, Any]:
     """Validate a YAML pipeline configuration.
 
     Parameters
@@ -25,6 +32,10 @@ def validate(yaml_content: str, lenient: bool = False) -> dict[str, Any]:
         If True, validate structure only without requiring environment variables.
         Useful for CI/CD validation where secrets aren't available.
         Default: False
+    base_path : Path | None
+        Base directory for resolving ``!include`` directives.
+        Required for full validation of pipelines that use includes.
+        Default: None (uses current working directory)
 
     Returns
     -------
@@ -58,13 +69,16 @@ def validate(yaml_content: str, lenient: bool = False) -> dict[str, Any]:
     """
     if lenient:
         return _validate_lenient(yaml_content)
-    return _validate_full(yaml_content)
+    return _validate_full(yaml_content, base_path=base_path)
 
 
-def _validate_full(yaml_content: str) -> dict[str, Any]:
+def _validate_full(
+    yaml_content: str,
+    base_path: Path | None = None,
+) -> dict[str, Any]:
     """Full validation with YamlPipelineBuilder."""
     try:
-        builder = YamlPipelineBuilder()
+        builder = YamlPipelineBuilder(base_path=base_path)
         graph, config = builder.build_from_yaml_string(yaml_content)
 
         return {
@@ -82,6 +96,46 @@ def _validate_full(yaml_content: str) -> dict[str, Any]:
         }
 
 
+def _strip_includes(
+    obj: Any,
+    warnings: list[str],
+    *,
+    _counter: list[int] | None = None,
+) -> Any:
+    """Strip ``{"!include": "..."}`` entries from parsed YAML.
+
+    In node lists, replaces includes with placeholder data_node entries.
+    Elsewhere, replaces with None.  Collects warnings for each stripped include.
+    """
+    if _counter is None:
+        _counter = [0]
+
+    if isinstance(obj, dict):
+        if "!include" in obj and len(obj) == 1:
+            path = obj["!include"]
+            warnings.append(f"!include '{path}' skipped during lenient validation")
+            return None
+        return {k: _strip_includes(v, warnings, _counter=_counter) for k, v in obj.items()}
+
+    if isinstance(obj, list):
+        result = []
+        for item in obj:
+            if isinstance(item, dict) and "!include" in item and len(item) == 1:
+                path = item["!include"]
+                warnings.append(f"!include '{path}' skipped during lenient validation")
+                _counter[0] += 1
+                result.append({
+                    "kind": "data_node",
+                    "metadata": {"name": f"__included_{_counter[0]}__"},
+                    "spec": {"data": {}},
+                })
+            else:
+                result.append(_strip_includes(item, warnings, _counter=_counter))
+        return result
+
+    return obj
+
+
 def _validate_lenient(yaml_content: str) -> dict[str, Any]:
     """Structure-only validation using core YamlValidator.
 
@@ -90,6 +144,10 @@ def _validate_lenient(yaml_content: str) -> dict[str, Any]:
     - Manifest structure (kind, metadata, spec)
     - Node structure and dependencies
     - Cycle detection
+
+    ``!include`` directives (dict-key syntax) are stripped and replaced
+    with placeholder nodes so that structural validation can proceed
+    without requiring access to included files.
     """
     try:
         parsed = yaml.safe_load(yaml_content)
@@ -100,6 +158,10 @@ def _validate_lenient(yaml_content: str) -> dict[str, Any]:
                 "error_type": "ParseError",
             }
 
+        # Strip !include directives before validation
+        include_warnings: list[str] = []
+        parsed = _strip_includes(parsed, include_warnings)
+
         # Use core validator for structural validation
         validator = YamlValidator()
         report = validator.validate(parsed)
@@ -122,7 +184,7 @@ def _validate_lenient(yaml_content: str) -> dict[str, Any]:
             "message": "Pipeline structure is valid",
             "node_count": len(nodes),
             "nodes": nodes,
-            "warnings": report.warnings,
+            "warnings": include_warnings + report.warnings,
         }
 
     except yaml.YAMLError as e:

{hexdag-0.7.0.dev13 → hexdag-0.8.0.dev1}/hexdag/api/vfs.py

@@ -27,20 +27,17 @@ from hexdag.drivers.vfs.providers import (
     LibProvider,
     ProcEntitiesProvider,
     ProcRunsProvider,
-    ProcScheduledProvider,
 )
 
 if TYPE_CHECKING:
     from hexdag.kernel import VFS
     from hexdag.stdlib.lib.entity_state import EntityState
     from hexdag.stdlib.lib.process_registry import ProcessRegistry
-    from hexdag.stdlib.lib.scheduler import Scheduler
 
 
 def create_vfs(
     *,
     registry: ProcessRegistry | None = None,
-    scheduler: Scheduler | None = None,
     entity_state: EntityState | None = None,
 ) -> VFS:
     """Create a VFS instance with all available providers mounted.
@@ -52,8 +49,6 @@ def create_vfs(
     ----------
     registry : ProcessRegistry | None
         Mount ``/proc/runs/`` if provided.
-    scheduler : Scheduler | None
-        Mount ``/proc/scheduled/`` if provided.
     entity_state : EntityState | None
         Mount ``/proc/entities/`` if provided.
 
@@ -68,8 +63,6 @@ def create_vfs(
 
     if registry is not None:
         vfs.mount("/proc/runs/", ProcRunsProvider(registry))
-    if scheduler is not None:
-        vfs.mount("/proc/scheduled/", ProcScheduledProvider(scheduler))
     if entity_state is not None:
         vfs.mount("/proc/entities/", ProcEntitiesProvider(entity_state))
 

{hexdag-0.7.0.dev13 → hexdag-0.8.0.dev1}/hexdag/cli/main.py

@@ -19,7 +19,7 @@ import hexdag.cli.commands as _commands_pkg
 
 app = typer.Typer(
     name="hexdag",
-    help="HexDAG - Lightweight DAG orchestration framework with hexagonal architecture.",
+    help="hexDAG — Operating system for AI agents.",
     no_args_is_help=True,
     rich_markup_mode="rich",
     pretty_exceptions_enable=False,
@@ -56,7 +56,7 @@ def callback(
         help="Show version and exit",
     ),
 ) -> None:
-    """HexDAG CLI - Modular DAG orchestration framework."""
+    """hexDAG CLI — Operating system for AI agents."""
     if version:
         # Read version from package metadata
         try:

{hexdag-0.7.0.dev13 → hexdag-0.8.0.dev1}/hexdag/compiler/__init__.py

@@ -6,9 +6,7 @@ from declarative YAML configurations, plus the configuration loader.
 """
 
 # Import tags to register YAML custom tags
-from . import include_tag as _include_tag  # noqa: F401
 from . import py_tag as _py_tag  # noqa: F401
-from .include_tag import set_include_base_path
 from .tag_discovery import discover_tags, get_known_tag_names, get_tag_schema
 from .yaml_builder import YamlPipelineBuilder
 
@@ -34,5 +32,4 @@ __all__ = [
     "discover_tags",
     "get_known_tag_names",
     "get_tag_schema",
-    "set_include_base_path",
 ]

{hexdag-0.7.0.dev13 → hexdag-0.8.0.dev1}/hexdag/compiler/plugins/macro_entity.py

@@ -37,6 +37,10 @@ class MacroEntityPlugin:
         self, node_config: dict[str, Any], builder: YamlPipelineBuilder, graph: DirectedGraph
     ) -> NodeSpec | None:
         """Expand macro into subgraph and merge into main graph."""
+        # Reset exit nodes upfront so a failed expansion doesn't leave
+        # stale state from the previous macro for the next node's implicit deps.
+        self.last_exit_nodes = []
+
         instance_name = node_config["metadata"]["name"]
         spec = node_config.get("spec", {})
         macro_ref = spec.get("macro")

{hexdag-0.7.0.dev13 → hexdag-0.8.0.dev1}/hexdag/compiler/plugins/node_entity.py

@@ -84,7 +84,7 @@ class NodeEntityPlugin:
 
         # Backward compat: if no explicit settings section, auto-partition flat spec
         if not settings_values:
-            settings_values, spec = self._partition_spec(factory_obj, spec)
+            settings_values, spec = self._partition_spec(factory, spec)
 
         # Merge for factory call (factory signature unchanged)
         factory_kwargs = {**settings_values, **spec}

hexdag-0.8.0.dev1/hexdag/compiler/reference_resolver.py

@@ -0,0 +1,282 @@
+"""Reference resolver — extracts node references from mapping values, expressions, and templates.
+
+Used by the YAML builder to infer dependencies from ``input_mapping``,
+``expressions``, and ``prompt_template`` / ``template`` fields so that
+users no longer need to specify redundant ``dependencies``.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from hexdag.kernel.context.execution_context import RESERVED_NAMES
+from hexdag.kernel.expression_parser import ALLOWED_FUNCTIONS
+
+# Pattern: identifier.identifier (but not $input.field)
+_NODE_FIELD_RE = re.compile(r"(?<!\$)\b([A-Za-z_][A-Za-z0-9_]*)\.[A-Za-z_][A-Za-z0-9_.]*")
+
+# Pattern: splits a source string into identifier tokens and non-identifier chars.
+# Used to find bare node names in expressions like "analyzer + 1".
+_IDENT_SPLIT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+
+# Jinja2 variable pattern: {{node.field}} or {{ node.field }}
+_JINJA_VAR_RE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\.[A-Za-z_][A-Za-z0-9_.]*\s*\}\}")
+
+# Names that are never node references — expression namespaces (from kernel)
+# plus Python/YAML keyword-like literals and built-in functions.
+_BUILTIN_NAMES = frozenset(ALLOWED_FUNCTIONS.keys()) | frozenset({
+    "self",
+    "type",
+    "true",
+    "false",
+    "none",
+    "True",
+    "False",
+    "None",
+    "null",
+})
+
+
+def extract_refs_from_mapping(
+    input_mapping: dict[str, Any],
+    known_nodes: frozenset[str],
+    macro_instances: frozenset[str] = frozenset(),
+) -> set[str]:
+    """Extract node names referenced in ``input_mapping`` values.
+
+    Recognises the ``node_name.field`` pattern used by
+    :pyclass:`ExecutionCoordinator._apply_input_mapping`.
+    Skips ``$input.*`` references (pipeline input, not a node).
+
+    Parameters
+    ----------
+    input_mapping : dict[str, Any]
+        Mapping of ``{target_field: "source_path"}``.
+    known_nodes : frozenset[str]
+        Set of all node names in the pipeline (used for validation).
+    macro_instances : frozenset[str]
+        Macro invocation names for prefix-based matching.
+
+    Returns
+    -------
+    set[str]
+        Node names referenced in the mapping.
+    """
+    refs: set[str] = set()
+    for source_path in input_mapping.values():
+        if isinstance(source_path, str):
+            refs.update(_extract_node_refs(source_path, known_nodes, macro_instances))
+        elif isinstance(source_path, list):
+            for item in source_path:
+                if isinstance(item, str):
+                    refs.update(_extract_node_refs(item, known_nodes, macro_instances))
+    return refs
+
+
+def extract_refs_from_expressions(
+    expressions: dict[str, Any],
+    known_nodes: frozenset[str],
+    macro_instances: frozenset[str] = frozenset(),
+) -> set[str]:
+    """Extract node names referenced in expression strings.
+
+    Scans for ``node_name.field`` tokens where ``node_name`` matches
+    a known node.
+
+    Parameters
+    ----------
+    expressions : dict[str, str]
+        Mapping of ``{variable_name: expression_string}``.
+    known_nodes : frozenset[str]
+        Set of all node names in the pipeline.
+    macro_instances : frozenset[str]
+        Macro invocation names for prefix-based matching.
+
+    Returns
+    -------
+    set[str]
+        Node names referenced in the expressions.
+    """
+    refs: set[str] = set()
+    for expr in expressions.values():
+        if not isinstance(expr, str):
+            continue
+        refs.update(_extract_node_refs(expr, known_nodes, macro_instances))
+    return refs
+
+
+def extract_refs_from_template(
+    template: str,
+    known_nodes: frozenset[str],
+    macro_instances: frozenset[str] = frozenset(),
+) -> set[str]:
+    """Extract node names referenced in Jinja2-style templates.
+
+    Looks for ``{{node_name.field}}`` patterns where ``node_name``
+    matches a known node.
+
+    Parameters
+    ----------
+    template : str
+        Jinja2 template string.
+    known_nodes : frozenset[str]
+        Set of all node names in the pipeline.
+    macro_instances : frozenset[str]
+        Macro invocation names for prefix-based matching.
+
+    Returns
+    -------
+    set[str]
+        Node names referenced in the template.
+    """
+    refs: set[str] = set()
+    for match in _JINJA_VAR_RE.finditer(template):
+        candidate = match.group(1)
+        if candidate in _BUILTIN_NAMES:
+            continue
+        if candidate in known_nodes:
+            refs.add(candidate)
+        elif macro_instances:
+            for mi in macro_instances:
+                if candidate.startswith(f"{mi}_"):
+                    refs.add(mi)
+                    break
+    return refs
+
+
+def extract_refs_from_string(
+    source: str,
+    known_nodes: frozenset[str],
+    macro_instances: frozenset[str] = frozenset(),
+) -> set[str]:
+    """Extract node references from a single string (condition, items, etc.).
+
+    Public wrapper around :func:`_extract_node_refs` for use by
+    ``_infer_deps`` when scanning composite node fields.
+
+    Parameters
+    ----------
+    source : str
+        A source path or expression string.
+    known_nodes : frozenset[str]
+        Set of all node names in the pipeline.
+    macro_instances : frozenset[str]
+        Macro invocation names for prefix-based matching.
+
+    Returns
+    -------
+    set[str]
+        Node names found.
+    """
+    return _extract_node_refs(source, known_nodes, macro_instances)
+
+
+# Pattern: $input.field_name — captures the field name after $input.
+_INPUT_FIELD_RE = re.compile(r"\$input\.([A-Za-z_][A-Za-z0-9_]*)")
+
+
+def extract_input_refs_from_mapping(input_mapping: dict[str, Any]) -> set[str]:
+    """Extract ``$input.X`` field names from ``input_mapping`` values.
+
+    Unlike :func:`extract_refs_from_mapping` which *skips* ``$input`` references,
+    this function *collects* them — returning the field names (without the
+    ``$input.`` prefix) so the validator can cross-check them against the
+    pipeline's declared ``input_schema`` or sibling nodes.
+
+    Parameters
+    ----------
+    input_mapping : dict[str, Any]
+        Mapping of ``{target_field: "source_path"}``.
+
+    Returns
+    -------
+    set[str]
+        Field names referenced via ``$input.X``.
+    """
+    fields: set[str] = set()
+    for val in input_mapping.values():
+        if isinstance(val, str):
+            for match in _INPUT_FIELD_RE.finditer(val):
+                fields.add(match.group(1))
+        elif isinstance(val, list):
+            for item in val:
+                if isinstance(item, str):
+                    for match in _INPUT_FIELD_RE.finditer(item):
+                        fields.add(match.group(1))
+    return fields
+
+
+def _extract_node_refs(
+    source: str,
+    known_nodes: frozenset[str],
+    macro_instances: frozenset[str] = frozenset(),
+) -> set[str]:
+    """Extract node references from a single source string.
+
+    Parameters
+    ----------
+    source : str
+        A source path or expression string.
+    known_nodes : frozenset[str]
+        Set of all node names in the pipeline.
+    macro_instances : frozenset[str]
+        Macro invocation names for prefix-based matching.
+
+    Returns
+    -------
+    set[str]
+        Node names found.
+    """
+    refs: set[str] = set()
+
+    # Skip pure $input references
+    stripped = source.strip()
+    if stripped == "$input" or stripped.startswith("$input."):
+        return refs
+
+    # Sort macro instances by length descending so longer prefixes match first.
+    # Without this, "extract" could greedily match "extract_rate_node" before
+    # the correct macro "extract_rate" gets a chance.
+    sorted_macros = sorted(macro_instances, key=len, reverse=True) if macro_instances else []
+
+    for match in _NODE_FIELD_RE.finditer(source):
+        candidate = match.group(1)
+        if candidate in RESERVED_NAMES or candidate in _BUILTIN_NAMES:
+            continue
+        if candidate in known_nodes:
+            refs.add(candidate)
+        elif sorted_macros:
+            for mi in sorted_macros:
+                if candidate.startswith(f"{mi}_"):
+                    refs.add(mi)
+                    break
+
+    # Bare identifiers — detect known nodes and macro-prefixed names that
+    # the dotted regex above cannot match.  Handles both simple bare names
+    # ("analyzer") and bare names inside expressions ("analyzer + 1").
+    # Extract all identifier tokens, then look them up in known_nodes directly.
+    # Tokens that are part of a "node.field" chain are skipped because _NODE_FIELD_RE
+    # already handled those and they appear with a trailing or leading dot in the
+    # source — but we filter by checking the char before/after each token position.
+    for token_match in _IDENT_SPLIT_RE.finditer(stripped):
+        start = token_match.start()
+        end = token_match.end()
+        # Skip if preceded by '.' or '$' (part of node.field or $input)
+        if start > 0 and stripped[start - 1] in (".", "$"):
+            continue
+        # Skip if followed by '.' (this is the owner in a node.field pattern)
+        if end < len(stripped) and stripped[end] == ".":
+            continue
+        candidate = token_match.group(0)
+        if candidate in _BUILTIN_NAMES or candidate in RESERVED_NAMES:
+            continue
+        if candidate in known_nodes:
+            refs.add(candidate)
+        elif sorted_macros:
+            for mi in sorted_macros:
+                if candidate.startswith(f"{mi}_"):
+                    refs.add(mi)
+                    break
+
+    return refs