flowforge-map 0.1.0__py3-none-any.whl

flowforge/main.py

@@ -0,0 +1,137 @@
+"""FastAPI app: start-up wiring and small routes. Runs: api/runs.py. Connectors: api/connectors.py.
+
+GET  /workflows              → names of the example workflows
+GET  /workflows/{name}       → one example workflow's JSON
+GET  /, /classic, /assets/*  → the dashboard and the V1 status page (spa.py)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Any
+
+from dotenv import load_dotenv
+from fastapi import FastAPI, HTTPException
+
+from flowforge import example_data, spa
+from flowforge import state as app_state_module
+from flowforge.api import agent as agent_api
+from flowforge.api import checks as checks_api
+from flowforge.api import connectors as connectors_api
+from flowforge.api import flowmap as flowmap_api
+from flowforge.api import runs as runs_api
+from flowforge.api import scan as scan_api
+from flowforge.api import security as security_api
+from flowforge.api import stats as stats_api
+from flowforge.connectors.models import Connector
+from flowforge.connectors.registry import Registry, check_references, load_connectors_file
+from flowforge.connectors.runtime import build_nodes  # noqa: F401 (re-exported for callers)
+from flowforge.flowmap.planner import CodeFlowPlanner, ScanPlanner
+from flowforge.flowmap.service import FlowService
+from flowforge.home import database_path, flowforge_home, resource
+from flowforge.security.logfilter import Redactor, install_log_redaction, secret_values_for
+from flowforge.state import AppState, app_state
+from flowforge.storage import Storage
+
+load_dotenv()
+
+WORKFLOWS_DIR = resource("workflows")   # inside the package when installed (D34)
+MCP_CHECK_TIMEOUT_S = runs_api.MCP_CHECK_TIMEOUT_S
+
+
+def __getattr__(name: str) -> Any:
+    """`main.state` stays readable for callers and tests written before api/ existed."""
+    if name == "state":
+        return app_state_module.current
+    raise AttributeError(name)
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+    example = os.getenv("FLOWFORGE_EXAMPLE") == "1"
+    redactor = Redactor()
+    if example:
+        flowforge_home().mkdir(parents=True, exist_ok=True)
+    storage = Storage(database_path(example), redact=redactor.redact)
+    registry = Registry(storage)
+    registry.ensure_defaults()
+    from_file = load_connectors_file(flowforge_home() / "connectors.json")
+    for connector in from_file:
+        registry.save(connector)
+    connectors = registry.all()
+    check_references(connectors)
+    redactor.set_values(secret_values_for(connectors))
+    restore_logging = install_log_redaction(redactor)
+    storage.mark_interrupted()
+    nodes, buckets = build_nodes(connectors)
+    s = app_state_module.current = AppState(
+        nodes=nodes, rate_limits=buckets, storage=storage, registry=registry, redactor=redactor,
+        file_managed={c.id for c in from_file}, example=example)
+    # the maps (D17, D21, D25): what the repo depends on, and the order its code calls them. Both come from code
+    # (D23: nothing drawn by hand reaches a map).
+    def connected() -> dict[str, Connector]:
+        return {c.id: c for c in registry.all()}
+
+    drawn = storage.flows("flow").latest_flow_version("accepted")
+    if example and drawn and (drawn["flow"] or {}).get("source") != "code":
+        storage.clear_map("flow")   # an example.db from before D25 still holds a hand-drawn flow
+    s.flows = {"deps": FlowService(storage.flows("deps"), connected, None),
+               "flow": FlowService(storage.flows("flow"), connected, None)}
+    if example:
+        s.flows["deps"].planner = ScanPlanner(scan_api.EXAMPLE_REPO)
+        s.flows["flow"].planner = CodeFlowPlanner(scan_api.EXAMPLE_REPO, lambda: example_data.latencies(storage))
+    else:   # your folder: from --repo or the last one chosen, mapped now if it has no maps yet
+        scan_api.startup_repo(storage, s.flows)
+    if example:  # offline nodes and a seeded history, never mixed with real data (D16)
+        example_data.install(s)
+        example_data.seed_flow(s)
+        seeding = asyncio.create_task(example_data.seed(s))
+        s.tasks.add(seeding)
+        seeding.add_done_callback(s.tasks.discard)
+    try:
+        yield
+    finally:
+        s.shutting_down = True
+        for task in list(s.tasks):
+            task.cancel()
+        if s.tasks:
+            await asyncio.gather(*s.tasks, return_exceptions=True)
+        for node in {id(n): n for n in [*s.nodes.values(), *s.retired_nodes]}.values():
+            await node.aclose()
+        restore_logging()
+
+
+app = FastAPI(title="FlowForge", lifespan=lifespan)
+app.include_router(runs_api.router)
+app.include_router(connectors_api.router)
+app.include_router(stats_api.router)
+app.include_router(flowmap_api.router)
+app.include_router(scan_api.router)
+app.include_router(checks_api.router)
+app.include_router(agent_api.router)
+app.include_router(security_api.router)
+
+
+@app.get("/workflows")
+def list_workflows() -> list[str]:
+    names = {p.stem for p in WORKFLOWS_DIR.glob("*.json")}
+    if app_state().example:
+        names |= set(example_data.workflows())
+    return sorted(names)
+
+
+@app.get("/workflows/{name}")
+def get_workflow(name: str) -> dict[str, Any]:
+    if app_state().example and name in example_data.workflows():
+        return example_data.workflows()[name]
+    path = WORKFLOWS_DIR / f"{name}.json"
+    if path.parent != WORKFLOWS_DIR or not path.exists():
+        raise HTTPException(404, "unknown workflow")
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+spa.install(app)  # last: the built dashboard, /classic and the HTML-vs-JSON routing (D15)

flowforge/nodes/__init__.py

@@ -0,0 +1,3 @@
+from flowforge.nodes.base import Node, NodeError, TransientNodeError
+
+__all__ = ["Node", "NodeError", "TransientNodeError"]

flowforge/nodes/base.py

@@ -0,0 +1,74 @@
+"""The interface every step type implements, plus the error split the executor retries on (D7)."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from contextvars import ContextVar
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
+from typing import Any
+
+# The run a node call belongs to, for naming artifact folders (D12). Set by the API before
+# run_workflow; step tasks inherit it. None outside a run.
+current_run_id: ContextVar[str | None] = ContextVar("current_run_id", default=None)
+
+
+class NodeError(Exception):
+    """Permanent failure — do not retry (bad request, auth error, ...)."""
+
+
+class TransientNodeError(NodeError):
+    """Retryable failure: 429, 5xx, timeout, connection error.
+
+    `retry_after_s` carries a server-provided Retry-After hint when there is one.
+    """
+
+    def __init__(self, message: str, retry_after_s: float | None = None):
+        super().__init__(message)
+        self.retry_after_s = retry_after_s
+
+
+class Node(ABC):
+    type: str
+    # Name of the rate-limit bucket this node draws from (None = unlimited). Per instance
+    # when built from a connector (D10).
+    rate_limit_key: str | None = None
+    # Set when built from a connector (D10). Nodes built without one keep V1 behaviour.
+    connector_id: str | None = None
+    fallback: str | None = None  # connector id tried after transient exhaustion (D10)
+    default_for_type: bool = False  # the type's default connector keeps the V1 cache namespace
+
+    @property
+    def cache_namespace(self) -> str:
+        """Prefix of this node's cache keys: two connectors never share an answer (D10)."""
+        if self.connector_id is None or self.default_for_type:
+            return self.type
+        return f"{self.type}:{self.connector_id}"
+
+    @abstractmethod
+    async def run(self, params: dict[str, Any]) -> Any:
+        """Execute with fully resolved params and return a JSON-serialisable output."""
+
+    def cacheable_across_runs(self, params: dict[str, Any]) -> bool:
+        """Per-type default for the persistent cache (D3). Overridden by step.cache."""
+        return False
+
+    async def aclose(self) -> None:  # noqa: B027 — optional hook; most nodes hold nothing
+        """Release connections/processes held across calls."""
+
+
+def parse_retry_after(value: str | None) -> float | None:
+    """Retry-After is either delta-seconds or an HTTP date."""
+    if not value:
+        return None
+    try:
+        return max(0.0, float(value))
+    except ValueError:
+        pass
+    try:
+        when = parsedate_to_datetime(value)
+    except (TypeError, ValueError):
+        return None
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=UTC)
+    return max(0.0, (when - datetime.now(UTC)).total_seconds())

flowforge/nodes/check_node.py

@@ -0,0 +1,50 @@
+"""The `check` step (D24): one health check per step, so the scheduler runs a check run like any other."""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from flowforge.checks import CheckTarget, check, new_client
+from flowforge.nodes.base import Node, NodeError
+
+# MCP servers are processes; at most this many start together (D24). The scheduler's slots are per step type and
+# every check is one type, so the limit lives here; check_workflow() gives MCP steps a timeout that covers the wait.
+MCP_AT_ONCE = 3
+
+
+class CheckNode(Node):
+    type = "check"
+
+    def __init__(self, transport: httpx.AsyncBaseTransport | None = None) -> None:
+        self._transport = transport   # tests answer through a MockTransport
+        self._http: httpx.AsyncClient | None = None
+        self._mcp = asyncio.Semaphore(MCP_AT_ONCE)
+
+    def _client(self) -> httpx.AsyncClient:
+        if self._http is None:
+            self._http = new_client(self._transport)
+        return self._http
+
+    async def run(self, params: dict[str, Any]) -> Any:
+        """params: `target` (a CheckTarget) and `root` (the scanned folder). A dependency that is down is a
+        result; only a check that can't run at all fails the step."""
+        try:
+            target = CheckTarget.model_validate(params["target"])
+            root = Path(params["root"])
+        except (KeyError, ValueError) as exc:
+            raise NodeError(f"check step: {exc}") from exc
+        if target.kind == "mcp":
+            async with self._mcp:
+                result = await check(target, root, http=self._client())
+        else:
+            result = await check(target, root, http=self._client())
+        return {**result.model_dump(mode="json"), "kind": target.kind}
+
+    async def aclose(self) -> None:
+        if self._http is not None:
+            await self._http.aclose()
+            self._http = None

flowforge/nodes/http_node.py

@@ -0,0 +1,161 @@
+"""Plain HTTP call via httpx (D9).
+
+params: method (default GET), url, headers?, json?, max_chars? (default 20000),
+        extract_text? (default: true for HTML responses — strips tags/scripts so an LLM
+        step downstream gets the readable text, not markup)
+output: {"status": int, "body": str | parsed JSON}
+
+With a connector (D10): a relative params.url is joined to the connector's base_url, and
+the auth header is built from its secret_ref (bearer / basic / raw). The key is only ever
+sent to the base_url's origin: an absolute URL elsewhere is refused, step headers cannot
+replace the auth header, and the key is redacted from error messages.
+
+429 / 5xx / timeouts / connection errors → TransientNodeError (honours Retry-After);
+other 4xx → NodeError.
+"""
+
+from __future__ import annotations
+
+import base64
+import re
+from html.parser import HTMLParser
+from typing import Any
+
+import httpx
+
+from flowforge.connectors.models import HTTPConnector
+from flowforge.connectors.secrets import SecretRefError, resolve_secret
+from flowforge.nodes.base import Node, NodeError, TransientNodeError, parse_retry_after
+
+USER_AGENT = "FlowForge/0.1 (+https://github.com/Vishwa-Thangapandiyan/flowforge-agent-orchestrator)"
+
+
+class _TextExtractor(HTMLParser):
+    SKIP = {"script", "style", "noscript", "svg", "head", "template"}
+    BLOCK = {"p", "div", "br", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "section", "article", "pre", "table"}
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.parts: list[str] = []
+        self._skipping = 0
+
+    def handle_starttag(self, tag: str, attrs: Any) -> None:
+        if tag in self.SKIP:
+            self._skipping += 1
+        elif tag in self.BLOCK:
+            self.parts.append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in self.SKIP and self._skipping:
+            self._skipping -= 1
+
+    def handle_data(self, data: str) -> None:
+        if not self._skipping:
+            self.parts.append(data)
+
+
+def html_to_text(html: str) -> str:
+    parser = _TextExtractor()
+    parser.feed(html)
+    text = "".join(parser.parts)
+    text = re.sub(r"[ \t\r\f\v]+", " ", text)
+    return re.sub(r"\n\s*\n+", "\n\n", text).strip()
+
+
+class HTTPNode(Node):
+    type = "http"
+
+    def __init__(self, client: httpx.AsyncClient | None = None) -> None:
+        self._client = client
+        self.connector: HTTPConnector | None = None
+
+    @classmethod
+    def from_connector(cls, connector: HTTPConnector, client: httpx.AsyncClient | None = None) -> HTTPNode:
+        node = cls(client)
+        node.connector = connector
+        node.connector_id = connector.id
+        node.rate_limit_key = connector.id
+        node.fallback = connector.fallback
+        return node
+
+    @property
+    def client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(follow_redirects=True, headers={"User-Agent": USER_AGENT})
+        return self._client
+
+    async def aclose(self) -> None:
+        if self._client is not None:
+            await self._client.aclose()
+
+    def cacheable_across_runs(self, params: dict[str, Any]) -> bool:
+        return params.get("method", "GET").upper() == "GET"
+
+    def _prepare(self, params: dict[str, Any]) -> tuple[str, dict[str, str] | None, list[str]]:
+        """(url, headers, secret strings to redact) for this call."""
+        url, headers = params["url"], params.get("headers")
+        if self.connector is None:
+            return url, headers, []
+        conn = self.connector.connection
+        label = f"http connector '{self.connector.id}'"
+        absolute = "://" in url or url.startswith("//")
+        if not absolute:
+            if not conn.base_url:
+                raise NodeError(f"{label}: relative url '{url}' needs a base_url on the connector")
+            url = conn.base_url.rstrip("/") + "/" + url.lstrip("/")
+        try:
+            secret = resolve_secret(self.connector.secret_ref)
+        except SecretRefError as exc:
+            raise NodeError(f"{label}: {exc}") from exc
+        if secret is None:
+            return url, headers, []
+        if not conn.base_url:
+            raise NodeError(f"{label}: a connector with a secret needs base_url, the only place its key is sent")
+        base = httpx.URL(conn.base_url)
+        target = httpx.URL("https:" + url if url.startswith("//") else url)
+        if (target.scheme, target.host, target.port) != (base.scheme, base.host, base.port):
+            raise NodeError(f"{label} only sends its key to {base.scheme}://{base.host}; refusing {target.host}")
+        header = conn.auth_header or "Authorization"
+        scheme = conn.auth_scheme or "bearer"
+        value = {"bearer": f"Bearer {secret}",
+                 "basic": "Basic " + base64.b64encode(secret.encode()).decode(),
+                 "raw": secret}[scheme]
+        merged = {k: v for k, v in (headers or {}).items() if k.lower() != header.lower()}
+        merged[header] = value
+        return url, merged, [secret, value]
+
+    async def run(self, params: dict[str, Any]) -> Any:
+        if "url" not in params:
+            raise NodeError("http step needs params.url")
+        method = params.get("method", "GET").upper()
+        url, headers, secrets = self._prepare(params)
+
+        def redact(text: str) -> str:
+            for secret in secrets:
+                text = text.replace(secret, "[REDACTED]")
+            return text
+
+        try:
+            response = await self.client.request(method, url, headers=headers, json=params.get("json"))
+        except (httpx.TimeoutException, httpx.TransportError) as exc:
+            raise TransientNodeError(redact(f"{type(exc).__name__}: {exc}")) from exc
+
+        status = response.status_code
+        if status == 429 or status >= 500:
+            raise TransientNodeError(
+                f"HTTP {status} from {params['url']}", parse_retry_after(response.headers.get("retry-after"))
+            )
+        if status >= 400:
+            # redact before cutting, so a key straddling the cut can't leak its first half
+            raise NodeError(f"HTTP {status} from {params['url']}: {redact(response.text)[:200]}")
+
+        content_type = response.headers.get("content-type", "")
+        if "json" in content_type:
+            try:
+                return {"status": status, "body": response.json()}
+            except ValueError:
+                pass
+        body = response.text
+        if params.get("extract_text", "html" in content_type):
+            body = html_to_text(body)
+        return {"status": status, "body": body[: params.get("max_chars", 20000)]}

flowforge/nodes/llm_node.py

@@ -0,0 +1,158 @@
+"""LLM chat completion for any provider (D8, D10).
+
+params: prompt (str) | messages (list), model?, temperature?, max_tokens?
+output: {"text": str, "model": str, "usage": {"prompt_tokens", "completion_tokens"[, "credits"]}}
+
+`LLMNode()` is the V1 node: NVIDIA NIM from env (D8), output unchanged.
+`LLMNode.from_connector(c)` builds a node for one connector (D10):
+  openai_compatible — NIM, Gemini's OpenAI endpoint, Ollama, vLLM, OpenRouter (openai SDK)
+  anthropic         — Claude via the official anthropic SDK; `system` messages move to the
+                      `system` parameter, sampling parameters are not sent (current Claude
+                      models reject them), and server-side refusal fallback is enabled on
+                      the models that support it
+Connector nodes add usage.credits (0 until a provider reports credits; no invented prices).
+
+429 / 5xx / timeouts / connection errors → TransientNodeError (with Retry-After);
+everything else (bad request, auth, refusal) → NodeError. The SDKs' own retries are off —
+the executor owns retries so they go through the rate limiter.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import anthropic
+import openai
+
+from flowforge.connectors.models import LLMConnector
+from flowforge.connectors.secrets import SecretRefError, resolve_secret
+from flowforge.nodes.base import Node, NodeError, TransientNodeError, parse_retry_after
+
+ANTHROPIC_DEFAULT_MAX_TOKENS = 16000  # thinking is always on; a small budget truncates answers
+# Models that accept the server-side refusal fallback ("default" routing by refusal category).
+ANTHROPIC_SERVER_FALLBACK_MODELS = {"claude-fable-5-1", "claude-opus-5-5", "claude-opus-5", "claude-sonnet-5-5"}
+ANTHROPIC_FALLBACK_BETA = "server-side-fallback-2026-07-01"
+
+
+class LLMNode(Node):
+    type = "llm"
+    rate_limit_key = "nim"
+
+    def __init__(self, client: Any | None = None) -> None:
+        """The V1 default: NVIDIA NIM configured from env (D8)."""
+        self.provider = "openai_compatible"
+        self.label = "NIM"
+        self.base_url: str | None = os.getenv("NIM_BASE_URL", "https://integrate.api.nvidia.com/v1")
+        self.default_model = os.getenv("NIM_DEFAULT_MODEL", "meta/llama-3.1-8b-instruct")
+        self.temperature_default = 0.0
+        self.secret_ref: str | None = None  # None here = the V1 NVIDIA_API_KEY lookup
+        self.report_credits = False
+        self._client = client
+
+    @classmethod
+    def from_connector(cls, connector: LLMConnector, client: Any | None = None) -> LLMNode:
+        node = cls(client)
+        conn = connector.connection
+        node.connector_id = connector.id
+        node.rate_limit_key = connector.id
+        node.fallback = connector.fallback
+        node.provider = conn.provider
+        node.label = f"{connector.id} ({conn.provider})"
+        node.base_url = conn.base_url
+        node.default_model = conn.model
+        node.temperature_default = conn.temperature_default
+        node.secret_ref = connector.secret_ref
+        node.report_credits = True
+        return node
+
+    @property
+    def client(self) -> Any:
+        if self._client is None:
+            self._client = self._make_client()
+        return self._client
+
+    def _make_client(self) -> Any:
+        if self.connector_id is None:
+            api_key = os.getenv("NVIDIA_API_KEY")
+            if not api_key:
+                raise NodeError("NVIDIA_API_KEY is not set — get a free key at build.nvidia.com and put it in .env")
+            return openai.AsyncOpenAI(api_key=api_key, base_url=self.base_url, max_retries=0)
+        try:
+            api_key = resolve_secret(self.secret_ref)
+        except SecretRefError as exc:
+            raise NodeError(f"{self.label}: {exc}") from exc
+        if self.provider == "anthropic":
+            # no key configured → the SDK resolves its own credentials (env, `ant auth login` profile)
+            return anthropic.AsyncAnthropic(**({"api_key": api_key} if api_key else {}), max_retries=0)
+        # keyless local servers (Ollama) still need a non-empty value for the client
+        return openai.AsyncOpenAI(api_key=api_key or "unused", base_url=self.base_url, max_retries=0)
+
+    def cacheable_across_runs(self, params: dict[str, Any]) -> bool:
+        return params.get("temperature", self.temperature_default) == 0
+
+    async def run(self, params: dict[str, Any]) -> Any:
+        if "messages" in params:
+            messages = params["messages"]
+        elif "prompt" in params:
+            messages = [{"role": "user", "content": str(params["prompt"])}]
+        else:
+            raise NodeError("llm step needs params.prompt or params.messages")
+
+        sdk = anthropic if self.provider == "anthropic" else openai
+        try:
+            if self.provider == "anthropic":
+                text, model, prompt_tokens, completion_tokens = await self._anthropic(params, messages)
+            else:
+                text, model, prompt_tokens, completion_tokens = await self._openai(params, messages)
+        except sdk.RateLimitError as exc:
+            raise TransientNodeError(
+                f"{self.label} rate limit: {exc}", parse_retry_after(exc.response.headers.get("retry-after"))
+            ) from exc
+        except sdk.APIStatusError as exc:
+            if exc.status_code >= 500:
+                raise TransientNodeError(
+                    f"{self.label} {exc.status_code}: {exc}", parse_retry_after(exc.response.headers.get("retry-after"))
+                ) from exc
+            raise NodeError(f"{self.label} {exc.status_code}: {exc}") from exc
+        except sdk.APIConnectionError as exc:  # includes APITimeoutError
+            raise TransientNodeError(f"{self.label} connection: {exc}") from exc
+
+        usage: dict[str, Any] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens}
+        if self.report_credits:
+            usage["credits"] = 0
+        return {"text": text, "model": model, "usage": usage}
+
+    async def _openai(self, params: dict[str, Any], messages: list[Any]) -> tuple[str, str, int, int]:
+        completion = await self.client.chat.completions.create(
+            model=params.get("model", self.default_model),
+            messages=messages,
+            temperature=params.get("temperature", self.temperature_default),
+            max_tokens=params.get("max_tokens", 512),
+        )
+        usage = completion.usage
+        return (
+            completion.choices[0].message.content or "",
+            completion.model,
+            usage.prompt_tokens if usage else 0,
+            usage.completion_tokens if usage else 0,
+        )
+
+    async def _anthropic(self, params: dict[str, Any], messages: list[Any]) -> tuple[str, str, int, int]:
+        system = "\n\n".join(str(m["content"]) for m in messages if m.get("role") == "system")
+        model = params.get("model", self.default_model)
+        request: dict[str, Any] = {
+            "model": model,
+            "max_tokens": params.get("max_tokens", ANTHROPIC_DEFAULT_MAX_TOKENS),
+            "messages": [m for m in messages if m.get("role") != "system"],
+        }
+        if system:
+            request["system"] = system
+        if model in ANTHROPIC_SERVER_FALLBACK_MODELS:
+            request |= {"betas": [ANTHROPIC_FALLBACK_BETA], "fallbacks": "default"}
+        message = await self.client.beta.messages.create(**request)
+        if message.stop_reason == "refusal":
+            category = getattr(message.stop_details, "category", None) if message.stop_details else None
+            raise NodeError(f"{self.label} refused the request" + (f" ({category})" if category else ""))
+        text = "".join(block.text for block in message.content if block.type == "text")
+        return text, message.model, message.usage.input_tokens, message.usage.output_tokens