abstractgateway 0.1.0__py3-none-any.whl

abstractgateway/hosts/visualflow_host.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from abstractruntime import Runtime
+
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_visual_listener_id(workflow_id: str) -> Optional[tuple[str, str]]:
+    """Return (flow_id, node_id) for a visual_event_listener_* workflow_id."""
+    prefix = "visual_event_listener_"
+    if not isinstance(workflow_id, str) or not workflow_id.startswith(prefix):
+        return None
+    rest = workflow_id[len(prefix) :]
+    parts = rest.split("_", 1)
+    if len(parts) != 2:
+        return None
+    flow_id, node_id = parts[0], parts[1]
+    if not flow_id or not node_id:
+        return None
+    return (flow_id, node_id)
+
+
+def _parse_visual_agent_id(workflow_id: str) -> Optional[tuple[str, str]]:
+    """Return (flow_id, node_id) for a visual_react_agent_* workflow_id."""
+    prefix = "visual_react_agent_"
+    if not isinstance(workflow_id, str) or not workflow_id.startswith(prefix):
+        return None
+    rest = workflow_id[len(prefix) :]
+    parts = rest.split("_", 1)
+    if len(parts) != 2:
+        return None
+    flow_id, node_id = parts[0], parts[1]
+    if not flow_id or not node_id:
+        return None
+    return (flow_id, node_id)
+
+
+def _require_visualflow_deps() -> None:
+    try:
+        import abstractflow  # noqa: F401
+    except Exception as e:  # pragma: no cover
+        raise RuntimeError(
+            "VisualFlow execution requires the AbstractFlow compiler library (not the web UI). "
+            "Install: `pip install abstractgateway[visualflow]`"
+        ) from e
+
+
+@dataclass(frozen=True)
+class VisualFlowRegistry:
+    """Load VisualFlow JSON files from a directory."""
+
+    flows_dir: Path
+
+    def load(self) -> Dict[str, Any]:
+        _require_visualflow_deps()
+        from abstractflow.visual.models import VisualFlow
+
+        base = Path(self.flows_dir).expanduser().resolve()
+        if not base.exists():
+            raise FileNotFoundError(f"flows_dir does not exist: {base}")
+
+        flows: Dict[str, Any] = {}
+        for p in sorted(base.glob("*.json")):
+            try:
+                raw = p.read_text(encoding="utf-8")
+                data = json.loads(raw)
+                flow = VisualFlow.model_validate(data)
+                flow_id = str(getattr(flow, "id", "") or "").strip() or p.stem
+                flows[flow_id] = flow
+            except Exception as e:
+                logger.warning("Failed to load flow file %s: %s", p, e)
+        return flows
+
+
+class VisualFlowGatewayHost:
+    """Gateway host that starts/ticks runs from VisualFlow JSON.
+
+    This host is optional. For dependency-light deployments prefer bundle mode (`.flow` bundles),
+    which compiles VisualFlow via `abstractruntime.visualflow_compiler` without importing `abstractflow`.
+    """
+
+    def __init__(
+        self,
+        *,
+        flows_dir: Path,
+        flows: Dict[str, Any],
+        run_store: Any,
+        ledger_store: Any,
+        artifact_store: Any,
+    ) -> None:
+        _require_visualflow_deps()
+        self._flows_dir = Path(flows_dir).expanduser().resolve()
+        self._flows = dict(flows)
+        self._run_store = run_store
+        self._ledger_store = ledger_store
+        self._artifact_store = artifact_store
+
+    @property
+    def flows_dir(self) -> Path:
+        return self._flows_dir
+
+    @property
+    def flows(self) -> Dict[str, Any]:
+        return dict(self._flows)
+
+    @property
+    def run_store(self) -> Any:
+        return self._run_store
+
+    @property
+    def ledger_store(self) -> Any:
+        return self._ledger_store
+
+    @property
+    def artifact_store(self) -> Any:
+        return self._artifact_store
+
+    def start_run(
+        self,
+        *,
+        flow_id: str,
+        input_data: Dict[str, Any],
+        actor_id: str = "gateway",
+        bundle_id: Optional[str] = None,
+    ) -> str:
+        # bundle_id is ignored for VisualFlow sources (kept for API compatibility with bundle mode).
+        del bundle_id
+        _require_visualflow_deps()
+        from abstractflow.visual.executor import create_visual_runner
+        from abstractflow.visual.workspace_scoped_tools import WorkspaceScope, build_scoped_tool_executor
+
+        fid = str(flow_id or "").strip()
+        if not fid:
+            raise ValueError("flow_id is required")
+        visual_flow = self._flows.get(fid)
+        if visual_flow is None:
+            raise KeyError(f"Flow '{fid}' not found in {self._flows_dir}")
+
+        data = dict(input_data or {})
+        scope = WorkspaceScope.from_input_data(data)
+        tool_executor = build_scoped_tool_executor(scope=scope) if scope is not None else None
+
+        vis_runner = create_visual_runner(
+            visual_flow,
+            flows=self._flows,
+            run_store=self._run_store,
+            ledger_store=self._ledger_store,
+            artifact_store=self._artifact_store,
+            tool_executor=tool_executor,
+        )
+        return str(vis_runner.start(data, actor_id=actor_id))
+
+    def runtime_and_workflow_for_run(self, run_id: str) -> tuple[Runtime, Any]:
+        """Return (runtime, workflow_spec) for the given run, ensuring derived VisualFlow workflows are registered."""
+
+        run = self._run_store.load(run_id)
+        if run is None:
+            raise KeyError(f"Run '{run_id}' not found")
+        workflow_id = getattr(run, "workflow_id", None)
+        if not isinstance(workflow_id, str) or not workflow_id:
+            raise ValueError(f"Run '{run_id}' missing workflow_id")
+
+        # Determine which VisualFlow root we need to compile to ensure this workflow_id is registered.
+        root_flow_id = workflow_id
+        parsed = _parse_visual_listener_id(workflow_id)
+        if parsed is not None:
+            root_flow_id = parsed[0]
+        parsed2 = _parse_visual_agent_id(workflow_id)
+        if parsed2 is not None:
+            root_flow_id = parsed2[0]
+
+        visual_flow = self._flows.get(root_flow_id)
+        if visual_flow is None:
+            raise KeyError(f"Flow '{root_flow_id}' not found (needed for workflow '{workflow_id}')")
+
+        # Rebuild tool scope from persisted vars (best-effort).
+        _require_visualflow_deps()
+        from abstractflow.visual.executor import create_visual_runner
+        from abstractflow.visual.workspace_scoped_tools import WorkspaceScope, build_scoped_tool_executor
+
+        vars0 = getattr(run, "vars", None)
+        scope = WorkspaceScope.from_input_data(vars0) if isinstance(vars0, dict) else None
+        tool_executor = build_scoped_tool_executor(scope=scope) if scope is not None else None
+
+        runner = create_visual_runner(
+            visual_flow,
+            flows=self._flows,
+            run_store=self._run_store,
+            ledger_store=self._ledger_store,
+            artifact_store=self._artifact_store,
+            tool_executor=tool_executor,
+        )
+        runtime = runner.runtime
+
+        # WorkflowSpec lookup: root uses runner.workflow; derived workflows use runtime.workflow_registry.
+        if getattr(runner.workflow, "workflow_id", None) == workflow_id:
+            return (runtime, runner.workflow)
+
+        reg = getattr(runtime, "workflow_registry", None)
+        if reg is None or not hasattr(reg, "get"):
+            raise KeyError(f"Runtime has no workflow_registry for workflow '{workflow_id}'")
+        wf = reg.get(workflow_id)
+        if wf is None:
+            raise KeyError(f"Workflow '{workflow_id}' not registered")
+        return (runtime, wf)
+

abstractgateway/routes/__init__.py

@@ -0,0 +1,5 @@
+from .gateway import router as gateway_router
+
+__all__ = ["gateway_router"]
+
+

abstractgateway/routes/gateway.py

@@ -0,0 +1,393 @@
+"""Run Gateway API (HTTP + SSE).
+
+Backlog 307: Durable Run Gateway (Command Inbox + Ledger Stream)
+
+This is intentionally replay-first:
+- The durable ledger is the source of truth.
+- SSE is an optimization; clients must be able to reconnect and replay by cursor.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from typing import Any, Dict, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel, Field
+
+from abstractruntime.storage.commands import CommandRecord
+
+from ..service import get_gateway_service, run_summary
+
+
+router = APIRouter(prefix="/gateway", tags=["gateway"])
+logger = logging.getLogger(__name__)
+
+
+class StartRunRequest(BaseModel):
+    bundle_id: Optional[str] = Field(
+        default=None,
+        description="Bundle id (when workflow source is 'bundle'). Optional if flow_id is already namespaced as 'bundle:flow'.",
+    )
+    flow_id: Optional[str] = Field(
+        default=None,
+        description=(
+            "Workflow id to start (flow id or 'bundle:flow'). Optional when bundle_id is provided and the bundle has a single entrypoint."
+        ),
+    )
+    input_data: Dict[str, Any] = Field(default_factory=dict)
+
+
+class StartRunResponse(BaseModel):
+    run_id: str
+
+
+class SubmitCommandRequest(BaseModel):
+    command_id: str = Field(..., description="Client-supplied idempotency key (UUID recommended).")
+    run_id: str = Field(..., description="Target run id (or session id for emit_event).")
+    type: str = Field(..., description="pause|resume|cancel|emit_event")
+    payload: Dict[str, Any] = Field(default_factory=dict)
+    ts: Optional[str] = Field(default=None, description="ISO timestamp (optional).")
+    client_id: Optional[str] = None
+
+
+class SubmitCommandResponse(BaseModel):
+    accepted: bool
+    duplicate: bool
+    seq: int
+
+
+def _require_bundle_host(svc: Any) -> Any:
+    host = getattr(svc, "host", None)
+    bundles = getattr(host, "bundles", None)
+    if not isinstance(bundles, dict):
+        raise HTTPException(status_code=400, detail="Bundle workflow source is not enabled on this gateway")
+    return host
+
+
+def _extract_entrypoint_inputs_from_visualflow(raw: Any) -> list[Dict[str, Any]]:
+    """Best-effort derive start input definitions from a VisualFlow JSON object."""
+    if not isinstance(raw, dict):
+        return []
+    nodes = raw.get("nodes")
+    if not isinstance(nodes, list):
+        return []
+
+    start_node: Optional[Dict[str, Any]] = None
+    for n in nodes:
+        if not isinstance(n, dict):
+            continue
+        t = n.get("type")
+        t_str = t.value if hasattr(t, "value") else str(t or "")
+        data = n.get("data") if isinstance(n.get("data"), dict) else {}
+        node_type = str(data.get("nodeType") or t_str or "").strip()
+        if node_type == "on_flow_start" or t_str == "on_flow_start":
+            start_node = n
+            break
+
+    if start_node is None:
+        return []
+
+    data = start_node.get("data") if isinstance(start_node.get("data"), dict) else {}
+    outputs = data.get("outputs")
+    pin_defaults = data.get("pinDefaults") if isinstance(data.get("pinDefaults"), dict) else {}
+    if not isinstance(outputs, list):
+        return []
+
+    out: list[Dict[str, Any]] = []
+    for p in outputs:
+        if not isinstance(p, dict):
+            continue
+        pid = str(p.get("id") or "").strip()
+        if not pid:
+            continue
+        ptype = str(p.get("type") or "").strip()
+        if ptype == "execution" or pid in {"exec-out", "exec"}:
+            continue
+        label = str(p.get("label") or pid).strip() or pid
+        item: Dict[str, Any] = {"id": pid, "label": label, "type": ptype or "unknown"}
+        if pid in pin_defaults:
+            item["default"] = pin_defaults.get(pid)
+        out.append(item)
+    return out
+
+
+def _extract_node_index_from_visualflow(raw: Any) -> Dict[str, Dict[str, Any]]:
+    """Best-effort node metadata index {node_id -> {type,label,headerColor}}."""
+    if not isinstance(raw, dict):
+        return {}
+    nodes = raw.get("nodes")
+    if not isinstance(nodes, list):
+        return {}
+    out: Dict[str, Dict[str, Any]] = {}
+    for n in nodes:
+        if not isinstance(n, dict):
+            continue
+        node_id = str(n.get("id") or "").strip()
+        if not node_id:
+            continue
+        t = n.get("type")
+        t_str = t.value if hasattr(t, "value") else str(t or "")
+        data = n.get("data") if isinstance(n.get("data"), dict) else {}
+        node_type = str(data.get("nodeType") or t_str or "").strip() or "unknown"
+        label = str(data.get("label") or n.get("label") or node_id).strip() or node_id
+        header_color = data.get("headerColor") or n.get("headerColor")
+        item: Dict[str, Any] = {"type": node_type, "label": label}
+        if isinstance(header_color, str) and header_color.strip():
+            item["headerColor"] = header_color.strip()
+        out[node_id] = item
+    return out
+
+
+@router.get("/bundles")
+async def list_bundles() -> Dict[str, Any]:
+    svc = get_gateway_service()
+    host = _require_bundle_host(svc)
+
+    items: list[Dict[str, Any]] = []
+    bundles = getattr(host, "bundles", {}) or {}
+    for bid, b in bundles.items():
+        man = getattr(b, "manifest", None)
+        if man is None:
+            continue
+        entrypoints = getattr(man, "entrypoints", None) or []
+        eps: list[Dict[str, Any]] = []
+        for ep in entrypoints:
+            eps.append(
+                {
+                    "flow_id": getattr(ep, "flow_id", None),
+                    "name": getattr(ep, "name", None),
+                    "description": getattr(ep, "description", "") or "",
+                    "interfaces": list(getattr(ep, "interfaces", None) or []),
+                }
+            )
+        items.append(
+            {
+                "bundle_id": str(bid),
+                "bundle_version": getattr(man, "bundle_version", "0.0.0"),
+                "created_at": getattr(man, "created_at", ""),
+                "default_entrypoint": str(getattr(man, "default_entrypoint", "") or "") or None,
+                "entrypoints": eps,
+            }
+        )
+
+    items.sort(key=lambda x: str(x.get("bundle_id") or ""))
+    return {"items": items, "default_bundle_id": getattr(host, "_default_bundle_id", None)}
+
+
+@router.get("/bundles/{bundle_id}")
+async def get_bundle(bundle_id: str) -> Dict[str, Any]:
+    svc = get_gateway_service()
+    host = _require_bundle_host(svc)
+
+    bid = str(bundle_id or "").strip()
+    if not bid:
+        raise HTTPException(status_code=400, detail="bundle_id is required")
+
+    bundles = getattr(host, "bundles", {}) or {}
+    bundle = bundles.get(bid)
+    if bundle is None:
+        raise HTTPException(status_code=404, detail=f"Bundle '{bid}' not found")
+
+    man = bundle.manifest
+    entrypoints_out: list[Dict[str, Any]] = []
+    for ep in list(man.entrypoints or []):
+        fid = str(getattr(ep, "flow_id", "") or "").strip()
+        rel = man.flow_path_for(fid) if fid else None
+        raw = bundle.read_json(rel) if isinstance(rel, str) and rel.strip() else None
+        entrypoints_out.append(
+            {
+                "flow_id": fid,
+                "workflow_id": f"{bid}:{fid}" if fid else None,
+                "name": getattr(ep, "name", None),
+                "description": str(getattr(ep, "description", "") or ""),
+                "interfaces": list(getattr(ep, "interfaces", None) or []),
+                "inputs": _extract_entrypoint_inputs_from_visualflow(raw),
+                "node_index": _extract_node_index_from_visualflow(raw),
+            }
+        )
+
+    flow_ids = sorted([str(k) for k in (man.flows or {}).keys() if isinstance(k, str) and k.strip()])
+    return {
+        "bundle_id": str(man.bundle_id),
+        "bundle_version": str(man.bundle_version or "0.0.0"),
+        "created_at": str(man.created_at or ""),
+        "default_entrypoint": str(getattr(man, "default_entrypoint", "") or "") or None,
+        "entrypoints": entrypoints_out,
+        "flows": flow_ids,
+        "metadata": dict(getattr(man, "metadata", None) or {}),
+    }
+
+
+@router.get("/bundles/{bundle_id}/flows/{flow_id}")
+async def get_bundle_flow(bundle_id: str, flow_id: str) -> Dict[str, Any]:
+    """Return a VisualFlow JSON from a bundle (best-effort; intended for thin clients)."""
+    svc = get_gateway_service()
+    host = _require_bundle_host(svc)
+
+    bid = str(bundle_id or "").strip()
+    if not bid:
+        raise HTTPException(status_code=400, detail="bundle_id is required")
+
+    fid = str(flow_id or "").strip()
+    if not fid:
+        raise HTTPException(status_code=400, detail="flow_id is required")
+
+    # Accept "bundle:flow" for convenience but enforce matching bundle_id.
+    if ":" in fid:
+        parsed = fid.split(":", 1)
+        if len(parsed) == 2 and parsed[0] and parsed[1]:
+            if parsed[0] != bid:
+                raise HTTPException(status_code=400, detail="flow_id bundle prefix does not match bundle_id")
+            fid = parsed[1].strip()
+        else:
+            raise HTTPException(status_code=400, detail="Invalid flow_id")
+
+    bundles = getattr(host, "bundles", {}) or {}
+    bundle = bundles.get(bid)
+    if bundle is None:
+        raise HTTPException(status_code=404, detail=f"Bundle '{bid}' not found")
+
+    rel = bundle.manifest.flow_path_for(fid)
+    if not isinstance(rel, str) or not rel.strip():
+        raise HTTPException(status_code=404, detail=f"Flow '{fid}' not found in bundle '{bid}'")
+
+    raw = bundle.read_json(rel)
+    if not isinstance(raw, dict):
+        raise HTTPException(status_code=500, detail="Flow JSON is invalid")
+
+    return {"bundle_id": bid, "flow_id": fid, "workflow_id": f"{bid}:{fid}", "flow": raw}
+
+
+@router.post("/runs/start", response_model=StartRunResponse)
+async def start_run(req: StartRunRequest) -> StartRunResponse:
+    svc = get_gateway_service()
+    svc.runner.start()
+
+    flow_id = str(req.flow_id or "").strip()
+    bundle_id = str(req.bundle_id).strip() if isinstance(req.bundle_id, str) and str(req.bundle_id).strip() else None
+    if not flow_id and not bundle_id:
+        raise HTTPException(status_code=400, detail="flow_id is required (or provide bundle_id to start a bundle entrypoint)")
+
+    try:
+        run_id = svc.host.start_run(
+            flow_id=flow_id,
+            bundle_id=bundle_id,
+            input_data=dict(req.input_data or {}),
+            actor_id="gateway",
+        )
+    except KeyError:
+        # Best-effort error message: in bundle mode, KeyError can refer to either a bundle or a flow.
+        msg = f"Flow '{flow_id}' not found" if flow_id else "Bundle not found"
+        raise HTTPException(status_code=404, detail=msg)
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Failed to start run: {e}")
+    return StartRunResponse(run_id=str(run_id))
+
+
+@router.get("/runs/{run_id}")
+async def get_run(run_id: str) -> Dict[str, Any]:
+    svc = get_gateway_service()
+    rs = svc.host.run_store
+    try:
+        run = rs.load(str(run_id))
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to load run: {e}")
+    if run is None:
+        raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
+    return run_summary(run)
+
+
+@router.get("/runs/{run_id}/ledger")
+async def get_ledger(
+    run_id: str,
+    after: int = Query(0, ge=0, description="Cursor: number of records already consumed."),
+    limit: int = Query(200, ge=1, le=2000),
+) -> Dict[str, Any]:
+    svc = get_gateway_service()
+    ledger = svc.host.ledger_store.list(str(run_id))
+    if not isinstance(ledger, list):
+        ledger = []
+    a = int(after or 0)
+    items = ledger[a : a + int(limit)]
+    next_after = a + len(items)
+    return {"items": items, "next_after": next_after}
+
+
+@router.get("/runs/{run_id}/ledger/stream")
+async def stream_ledger(
+    run_id: str,
+    after: int = Query(0, ge=0, description="Cursor: number of records already consumed."),
+    heartbeat_s: float = Query(5.0, gt=0.1, le=60.0),
+) -> StreamingResponse:
+    svc = get_gateway_service()
+    run_id2 = str(run_id)
+
+    async def _gen():
+        cursor = int(after or 0)
+        last_emit = asyncio.get_event_loop().time()
+        while True:
+            ledger = svc.host.ledger_store.list(run_id2)
+            if not isinstance(ledger, list):
+                ledger = []
+            if cursor < 0:
+                cursor = 0
+            if cursor < len(ledger):
+                while cursor < len(ledger):
+                    item = ledger[cursor]
+                    data = json.dumps({"cursor": cursor + 1, "record": item}, ensure_ascii=False)
+                    yield f"id: {cursor + 1}\n".encode("utf-8")
+                    yield b"event: step\n"
+                    yield f"data: {data}\n\n".encode("utf-8")
+                    cursor += 1
+                    last_emit = asyncio.get_event_loop().time()
+            else:
+                now = asyncio.get_event_loop().time()
+                if (now - last_emit) >= float(heartbeat_s):
+                    yield b": keep-alive\n\n"
+                    last_emit = now
+                await asyncio.sleep(0.25)
+
+    return StreamingResponse(_gen(), media_type="text/event-stream")
+
+
+@router.post("/commands", response_model=SubmitCommandResponse)
+async def submit_command(req: SubmitCommandRequest) -> SubmitCommandResponse:
+    svc = get_gateway_service()
+    typ = str(req.type or "").strip()
+    if typ not in {"pause", "resume", "cancel", "emit_event"}:
+        raise HTTPException(status_code=400, detail="type must be one of pause|resume|cancel|emit_event")
+
+    record = CommandRecord(
+        command_id=str(req.command_id),
+        run_id=str(req.run_id),
+        type=typ,
+        payload=dict(req.payload or {}),
+        ts=str(req.ts) if isinstance(req.ts, str) and req.ts else "",
+        client_id=str(req.client_id) if isinstance(req.client_id, str) and req.client_id else None,
+        seq=0,
+    )
+
+    try:
+        res = svc.runner.command_store.append(record)
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Failed to append command: {e}")
+
+    try:
+        logger.info(
+            "gateway_command accepted=%s dup=%s seq=%s run_id=%s type=%s command_id=%s client_id=%s",
+            bool(res.accepted),
+            bool(res.duplicate),
+            int(res.seq),
+            str(req.run_id),
+            str(req.type),
+            str(req.command_id),
+            str(req.client_id) if req.client_id else "",
+        )
+    except Exception:
+        pass
+
+    return SubmitCommandResponse(accepted=bool(res.accepted), duplicate=bool(res.duplicate), seq=int(res.seq))