hexdag 0.7.0.dev10__tar.gz → 0.7.0.dev12__tar.gz

{hexdag-0.7.0.dev10 → hexdag-0.7.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hexdag
-Version: 0.7.0.dev10
+Version: 0.7.0.dev12
 Summary: Lightweight DAG orchestration framework with enterprise pipeline capabilities
 Project-URL: Homepage, https://hexdag.ai
 Project-URL: Repository, https://github.com/omniviser/hexdag
@@ -41,6 +41,7 @@ Requires-Dist: nbformat>=5.9.0; extra == 'all'
 Requires-Dist: openai>=1.0.0; extra == 'all'
 Requires-Dist: pandas>=2.1.0; extra == 'all'
 Requires-Dist: pgvector>=0.3.0; extra == 'all'
+Requires-Dist: redis>=5.0.0; extra == 'all'
 Requires-Dist: sqlalchemy>=2.0.0; extra == 'all'
 Provides-Extra: anthropic
 Requires-Dist: anthropic>=0.25.0; extra == 'anthropic'
@@ -58,6 +59,8 @@ Requires-Dist: nbformat>=5.9.0; extra == 'notebooks'
 Requires-Dist: pandas>=2.1.0; extra == 'notebooks'
 Provides-Extra: openai
 Requires-Dist: openai>=1.0.0; extra == 'openai'
+Provides-Extra: redis
+Requires-Dist: redis>=5.0.0; extra == 'redis'
 Provides-Extra: storage-all
 Requires-Dist: aiomysql>=0.2.0; extra == 'storage-all'
 Requires-Dist: aiosqlite>=0.20.0; extra == 'storage-all'

{hexdag-0.7.0.dev10 → hexdag-0.7.0.dev12}/hexdag/api/__init__.py

@@ -32,6 +32,7 @@ Available submodules
 - documentation: Guides and references
 - export: Project export
 - processes: Process management (pipeline runs, scheduling, entity state)
+- logs: Log querying (query, summarize, tail JSON log files)
 - vfs: Virtual filesystem (uniform path-based introspection)
 """
 
@@ -40,6 +41,7 @@ from hexdag.api import (
     documentation,
     execution,
     export,
+    logs,
     pipeline,
     processes,
     validation,
@@ -51,6 +53,7 @@ __all__ = [
     "documentation",
     "execution",
     "export",
+    "logs",
     "pipeline",
     "processes",
     "validation",

hexdag-0.7.0.dev12/hexdag/api/logs.py

@@ -0,0 +1,221 @@
+"""Log query API — MCP/Studio tools for querying hexDAG JSON log files.
+
+Provides unified functions that both the MCP server and hexdag-studio
+REST API consume. Reads Loguru JSON log files (written by
+``configure_logging(output_file=...)``) and filters on structured fields.
+
+MCP server usage::
+
+    from hexdag.api import logs
+
+    @mcp.tool()
+    async def query_logs(
+        level: str | None = None,
+        pipeline_name: str | None = None,
+        contains: str | None = None,
+    ):
+        return await logs.query_logs(
+            "hexdag.log", level=level, pipeline_name=pipeline_name, contains=contains,
+        )
+
+Studio REST API::
+
+    @router.get("/logs")
+    async def get_logs(level: str | None = None, pipeline_name: str | None = None):
+        return await logs.query_logs("hexdag.log", level=level, pipeline_name=pipeline_name)
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import aiofiles
+
+_LEVEL_ORDER = {
+    "TRACE": 0,
+    "DEBUG": 1,
+    "INFO": 2,
+    "WARNING": 3,
+    "ERROR": 4,
+    "CRITICAL": 5,
+}
+
+
+async def query_logs(
+    log_file: str | Path,
+    *,
+    level: str | None = None,
+    pipeline_name: str | None = None,
+    node_id: str | None = None,
+    contains: str | None = None,
+    limit: int = 100,
+) -> list[dict[str, Any]]:
+    """Query log records from a JSON log file with filters.
+
+    Each line in the log file is a JSON record produced by Loguru's
+    ``serialize=True`` mode. Structured fields (``pipeline_name``,
+    ``node``, ``node_type``) are extracted from ``record.extra``.
+
+    Args
+    ----
+        log_file: Path to the JSON log file.
+        level: Minimum log level (DEBUG, INFO, WARNING, ERROR, CRITICAL).
+        pipeline_name: Filter by pipeline name (exact match on extra.pipeline_name).
+        node_id: Filter by node name (exact match on extra.node).
+        contains: Substring match on the log message text.
+        limit: Maximum number of records to return (default 100).
+
+    Returns
+    -------
+        List of matching log records, newest first.
+    """
+    path = Path(log_file)
+    if not path.exists():
+        return []
+
+    min_level = _LEVEL_ORDER.get(level.upper(), 0) if level else 0
+    contains_lower = contains.lower() if contains else None
+
+    records = await _read_json_lines(path)
+
+    results: list[dict[str, Any]] = []
+    for rec in reversed(records):
+        if len(results) >= limit:
+            break
+
+        record = rec.get("record", {})
+        extra = record.get("extra", {})
+        rec_level = record.get("level", {}).get("name", "INFO")
+
+        # Level filter
+        if _LEVEL_ORDER.get(rec_level, 0) < min_level:
+            continue
+
+        # Pipeline filter
+        if pipeline_name and extra.get("pipeline_name") != pipeline_name:
+            continue
+
+        # Node filter
+        if node_id and extra.get("node") != node_id:
+            continue
+
+        # Substring match
+        message = record.get("message", rec.get("text", ""))
+        if contains_lower and contains_lower not in message.lower():
+            continue
+
+        results.append(_summarize_record(rec))
+
+    return results
+
+
+async def get_log_summary(
+    log_file: str | Path,
+    *,
+    pipeline_name: str | None = None,
+) -> dict[str, Any]:
+    """Aggregate log counts by level from a JSON log file.
+
+    Args
+    ----
+        log_file: Path to the JSON log file.
+        pipeline_name: Scope summary to a specific pipeline.
+
+    Returns
+    -------
+        Dict with counts per level, total count, and time range.
+    """
+    path = Path(log_file)
+    if not path.exists():
+        return {"total": 0, "counts": {}, "pipeline_name": pipeline_name}
+
+    records = await _read_json_lines(path)
+
+    counts: dict[str, int] = {}
+    total = 0
+
+    for rec in records:
+        record = rec.get("record", {})
+        extra = record.get("extra", {})
+
+        if pipeline_name and extra.get("pipeline_name") != pipeline_name:
+            continue
+
+        rec_level = record.get("level", {}).get("name", "UNKNOWN")
+        counts[rec_level] = counts.get(rec_level, 0) + 1
+        total += 1
+
+    return {
+        "total": total,
+        "counts": counts,
+        "pipeline_name": pipeline_name,
+    }
+
+
+async def tail_logs(
+    log_file: str | Path,
+    *,
+    n: int = 20,
+    level: str | None = None,
+) -> list[dict[str, Any]]:
+    """Get the N most recent log records from a JSON log file.
+
+    Args
+    ----
+        log_file: Path to the JSON log file.
+        n: Number of records to return (default 20).
+        level: Minimum log level filter.
+
+    Returns
+    -------
+        List of most recent records, newest first.
+    """
+    return await query_logs(log_file, level=level, limit=n)
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+async def _read_json_lines(path: Path) -> list[dict[str, Any]]:
+    """Read a JSON-lines log file, skipping malformed lines."""
+    records: list[dict[str, Any]] = []
+    async with aiofiles.open(path, encoding="utf-8") as f:
+        async for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    return records
+
+
+def _summarize_record(rec: dict[str, Any]) -> dict[str, Any]:
+    """Extract key fields from a Loguru JSON record for API output."""
+    record = rec.get("record", {})
+    extra = record.get("extra", {})
+    level_info = record.get("level", {})
+    time_info = record.get("time", {})
+
+    result: dict[str, Any] = {
+        "timestamp": time_info.get("repr", ""),
+        "level": level_info.get("name", ""),
+        "message": record.get("message", rec.get("text", "")),
+        "module": extra.get("module", record.get("name", "")),
+        "function": record.get("function", ""),
+        "line": record.get("line", 0),
+    }
+
+    if pipeline_name := extra.get("pipeline_name"):
+        result["pipeline_name"] = pipeline_name
+    if node := extra.get("node"):
+        result["node"] = node
+    if node_type := extra.get("node_type"):
+        result["node_type"] = node_type
+
+    return result

{hexdag-0.7.0.dev10 → hexdag-0.7.0.dev12}/hexdag/compiler/config_loader.py

@@ -29,6 +29,7 @@ from hexdag.kernel.config.models import (
     HexDAGConfig,
     LoggingConfig,
     ManifestEntry,
+    MCPPermissions,
 )
 from hexdag.kernel.logging import get_logger
 from hexdag.kernel.orchestration.models import OrchestratorConfig
@@ -377,6 +378,15 @@ class ConfigLoader:
             config.caps = DefaultCaps(
                 default_set=caps_data.get("default_set"),
                 deny=caps_data.get("deny"),
+                profiles=caps_data.get("profiles"),
+            )
+
+        # Parse MCP permissions
+        if "mcp_permissions" in data:
+            mcp_data = data["mcp_permissions"]
+            config.mcp_permissions = MCPPermissions(
+                default_profile=mcp_data.get("default_profile", "read-only"),
+                client_profiles=mcp_data.get("client_profiles", {}),
             )
 
         return config

{hexdag-0.7.0.dev10 → hexdag-0.7.0.dev12}/hexdag/compiler/plugins/__init__.py

@@ -1,20 +1,26 @@
 """Entity plugins for the YAML pipeline builder.
 
 These plugins build specific entity types from YAML configuration:
+- AdapterDefinitionPlugin: Process Adapter definitions (kind: Adapter)
 - ConfigDefinitionPlugin: Process Config definitions (kind: Config)
 - MacroDefinitionPlugin: Process Macro definitions (kind: Macro)
 - MacroEntityPlugin: Expand macro invocations into subgraphs
+- MiddlewareDefinitionPlugin: Process Middleware definitions (kind: Middleware)
 - NodeEntityPlugin: Build regular nodes (llm_node, function_node, etc.)
 """
 
+from hexdag.compiler.plugins.adapter_definition import AdapterDefinitionPlugin
 from hexdag.compiler.plugins.config_definition import ConfigDefinitionPlugin
 from hexdag.compiler.plugins.macro_definition import MacroDefinitionPlugin
 from hexdag.compiler.plugins.macro_entity import MacroEntityPlugin
+from hexdag.compiler.plugins.middleware_definition import MiddlewareDefinitionPlugin
 from hexdag.compiler.plugins.node_entity import NodeEntityPlugin
 
 __all__ = [
+    "AdapterDefinitionPlugin",
     "ConfigDefinitionPlugin",
     "MacroDefinitionPlugin",
     "MacroEntityPlugin",
+    "MiddlewareDefinitionPlugin",
     "NodeEntityPlugin",
 ]

hexdag-0.7.0.dev12/hexdag/compiler/plugins/adapter_definition.py

@@ -0,0 +1,140 @@
+"""Plugin for handling Adapter definitions (kind: Adapter).
+
+Standalone ``kind: Adapter`` manifests define reusable adapter configurations
+that can be referenced by name from ``spec.ports.<name>.ref``.
+
+Example YAML::
+
+    apiVersion: hexdag/v1
+    kind: Adapter
+    metadata:
+      name: production-openai
+      description: Production OpenAI with low temperature
+    spec:
+      class: hexdag.stdlib.adapters.openai.OpenAIAdapter
+      config:
+        model: gpt-4o
+        temperature: 0.3
+        api_key: ${OPENAI_API_KEY}
+
+Reference in a pipeline::
+
+    spec:
+      ports:
+        llm:
+          ref: production-openai
+          config:          # optional overrides merged on top
+            temperature: 0.9
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from hexdag.kernel.exceptions import YamlPipelineBuilderError
+from hexdag.kernel.logging import get_logger
+
+if TYPE_CHECKING:
+    from hexdag.compiler.yaml_builder import YamlPipelineBuilder
+    from hexdag.kernel.domain.dag import DirectedGraph, NodeSpec
+
+logger = get_logger(__name__)
+
+# Module-level registry for named adapter definitions
+_adapter_registry: dict[str, dict[str, Any]] = {}
+
+
+def get_adapter_definition(name: str) -> dict[str, Any] | None:
+    """Look up a registered adapter definition by name.
+
+    Parameters
+    ----------
+    name : str
+        Adapter name (from ``metadata.name``)
+
+    Returns
+    -------
+    dict[str, Any] | None
+        Adapter definition dict with ``class`` and ``config`` keys,
+        or None if not registered
+    """
+    return _adapter_registry.get(name)
+
+
+def clear_adapter_registry() -> None:
+    """Clear all registered adapter definitions (for testing)."""
+    _adapter_registry.clear()
+
+
+class AdapterDefinitionPlugin:
+    """Plugin for handling Adapter definitions (kind: Adapter).
+
+    Processes ``kind: Adapter`` YAML documents and registers the
+    adapter configuration for later reference by name.  Does not add
+    nodes to the graph.
+    """
+
+    def can_handle(self, node_config: dict[str, Any]) -> bool:
+        """Handle Adapter kind."""
+        return node_config.get("kind") == "Adapter"
+
+    def build(
+        self, node_config: dict[str, Any], builder: YamlPipelineBuilder, graph: DirectedGraph
+    ) -> NodeSpec | None:
+        """Register named adapter definition.
+
+        Parameters
+        ----------
+        node_config : dict[str, Any]
+            Adapter definition configuration
+        builder : YamlPipelineBuilder
+            Builder instance (unused)
+        graph : DirectedGraph
+            Graph instance (unused — definitions don't add nodes)
+
+        Returns
+        -------
+        None
+            Adapter definitions don't add nodes to the graph
+        """
+        metadata = node_config.get("metadata", {})
+        name = metadata.get("name")
+        if not name:
+            raise YamlPipelineBuilderError("Adapter definition missing 'metadata.name'")
+
+        spec = node_config.get("spec", {})
+        adapter_class = spec.get("class")
+
+        if not adapter_class:
+            raise YamlPipelineBuilderError(
+                f"Adapter '{name}' missing 'spec.class'. "
+                f"Adapter definitions must specify the adapter class module path."
+            )
+
+        if not isinstance(adapter_class, str):
+            raise YamlPipelineBuilderError(
+                f"Adapter '{name}' spec.class must be a module path string, "
+                f"got {type(adapter_class).__name__}"
+            )
+
+        config = spec.get("config", {})
+        if not isinstance(config, dict):
+            raise YamlPipelineBuilderError(
+                f"Adapter '{name}' spec.config must be a dict, got {type(config).__name__}"
+            )
+
+        capabilities = spec.get("capabilities")
+
+        entry: dict[str, Any] = {"class": adapter_class, "config": config}
+        if capabilities is not None:
+            entry["capabilities"] = capabilities
+
+        _adapter_registry[name] = entry
+
+        logger.info(
+            "Registered adapter definition '{}' (class={})",
+            name,
+            adapter_class,
+        )
+
+        return None

{hexdag-0.7.0.dev10 → hexdag-0.7.0.dev12}/hexdag/compiler/plugins/macro_definition.py

@@ -67,6 +67,7 @@ class MacroDefinitionPlugin:
             Macro definitions don't add nodes to the graph
         """
         from hexdag.kernel.yaml_macro import (  # lazy: mutual cycle with yaml_builder
+            PortRequirement,
             YamlMacro,
             YamlMacroConfig,
             YamlMacroParameterSpec,
@@ -85,22 +86,31 @@ class MacroDefinitionPlugin:
         raw_parameters = node_config.get("parameters", [])
         parameters = [YamlMacroParameterSpec(**p) for p in raw_parameters]
 
-        # Extract nodes
+        # Extract nodes — support both static list and raw YAML template string
         nodes = node_config.get("nodes", [])
-        if not nodes:
+        nodes_raw = node_config.get("nodes_raw")
+
+        if not nodes and not nodes_raw:
             raise YamlPipelineBuilderError(
-                f"Macro '{macro_name}' has no nodes. Macros must define at least one node."
+                f"Macro '{macro_name}' has no nodes. "
+                f"Macros must define 'nodes' (list) or 'nodes_raw' (template string)."
             )
 
         # Extract outputs (optional)
         outputs = node_config.get("outputs")
 
+        # Extract port requirements (optional)
+        raw_ports = node_config.get("requires_ports", [])
+        requires_ports = [PortRequirement(**p) for p in raw_ports]
+
         # Create YamlMacroConfig
         macro_config = YamlMacroConfig(
             macro_name=macro_name,
             macro_description=macro_description,
             parameters=parameters,
             nodes=nodes,
+            nodes_raw=nodes_raw,
+            requires_ports=requires_ports,
             outputs=outputs,
         )
 
@@ -108,12 +118,24 @@ class MacroDefinitionPlugin:
         # Create a dynamic class that pre-fills the YamlMacro config
         config_dict = macro_config.model_dump()
 
+        # Structural fields that must not be overridden at invocation time
+        _structural = frozenset({
+            "macro_name",
+            "macro_description",
+            "parameters",
+            "nodes",
+            "nodes_raw",
+            "requires_ports",
+            "outputs",
+        })
+
         class DynamicYamlMacro(YamlMacro):
             """Dynamically generated YamlMacro with pre-filled configuration."""
 
             def __init__(self, **kwargs: Any) -> None:
-                # Merge pre-filled config with any override kwargs
-                merged_config = {**config_dict, **kwargs}
+                # Only allow non-structural overrides (parameter values)
+                safe_kwargs = {k: v for k, v in kwargs.items() if k not in _structural}
+                merged_config = {**config_dict, **safe_kwargs}
                 super().__init__(**merged_config)
 
         # Set class name for better debugging