flowforge-map 0.1.0__py3-none-any.whl

flowforge/connectors/catalog.py

@@ -0,0 +1,84 @@
+"""The connector catalog (D17): every app FlowForge knows, whether it is on the map, and a mock test run.
+
+Brands live only in `presets.py`. Test runs here are offline and canned ("mock"): they show what a
+real test would do and what the Planner learns from an app, and they never touch a network or a key.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from flowforge.connectors.models import Connector
+from flowforge.connectors.presets import CATALOG, CATEGORIES
+
+TYPE_NAME = {"llm": "LLM", "mcp": "MCP server", "http": "HTTP API", "local": "Local command"}
+
+
+def catalog(connectors: dict[str, Connector], on_map: set[str]) -> dict[str, Any]:
+    """Catalog entries with a status each, plus connected apps the catalog doesn't list (your own).
+
+    status: `on_map` (the code calls it), `connected` (connected, not in your code) or `none`.
+    """
+    def status(app_id: str) -> str:
+        if app_id in on_map:
+            return "on_map"
+        return "connected" if app_id in connectors else "none"
+
+    items = [{**entry, "kind": TYPE_NAME[entry["type"]], "status": status(entry["id"])} for entry in CATALOG]
+    listed = {entry["id"] for entry in CATALOG}
+    own = [{"id": c.id, "name": c.name, "category": "custom", "type": c.type, "kind": TYPE_NAME[c.type],
+            "live": True, "description": c.role or "One of your own apps.", "meta": f"{TYPE_NAME[c.type]} · yours",
+            "status": status(c.id), "color": c.style.color if c.style else None}
+           for c in connectors.values() if c.id not in listed]
+    return {"categories": CATEGORIES, "items": items, "own": own}
+
+
+def find(app_id: str, connectors: dict[str, Connector]) -> dict[str, Any] | None:
+    entry = next((e for e in CATALOG if e["id"] == app_id), None)
+    if entry is not None:
+        return entry
+    c = connectors.get(app_id)
+    if c is None:
+        return None
+    return {"id": c.id, "name": c.name, "category": "custom", "type": c.type, "live": True}
+
+
+def mock_test(entry: dict[str, Any]) -> dict[str, Any]:
+    """The canned steps, result and capability summary for one app. No network, no key, no secret text."""
+    kind, category, name = entry["type"], entry["category"], entry["name"]
+    if category == "pay":
+        steps = [("Connect in test mode", "a live key is refused on a test connector"),
+                 ("Create a test order", "POST /v1/orders · ₹499.00"), ("Read it back", "GET /v1/orders/{id}")]
+        result = '{ "id": "order_example", "amount": 49900, "status": "created" }'
+        stats = "0.3 s · test mode, no money moves"
+        learns = ["create order", "fetch payment", "refund · always behind a gate"]
+    elif kind == "llm":
+        steps = [("Connect", "chat/completions"), ("Send a tiny prompt", '"Say hello from FlowForge in one line."'),
+                 ("Read the answer", "check the reply and the token count")]
+        result, stats = '"Hello from FlowForge. Test run OK."', "0.4 s · 18 tokens"
+        learns = ["provider and model", "rate limit", "token counts per call"]
+    elif kind == "mcp":
+        steps = [("Start the server", "on this machine, as its own process"), ("List its tools", "tools/list"),
+                 ("Check the schemas", "arguments are validated before any run")]
+        result, stats = "tools listed with their input schemas", "0.6 s · server stopped after"
+        learns = ["every tool and its arguments", "which tools only read", "tools that write wait for you"]
+    elif kind == "local":
+        steps = [("Check the command", "it exists and isn't a shell script"),
+                 ("Run it in its folder", "no shell · env passed explicitly"),
+                 ("Read the output", "stdout, stderr, exit code")]
+        result, stats = "exit 0", "1.2 s"
+        learns = ["the command and its arguments", "the folder it may use", "always behind a gate"]
+    elif category == "media":
+        steps = [("Connect", f"{name} API"), ("Make one sample", '"a soft blue baby blanket"'),
+                 ("Save the file", "kept on disk, not in the run log")]
+        result, stats = '{ "file": "artifacts/test/sample.png", "width": 1024 }', "0.5 s · example output"
+        learns = ["makes images, audio or video", "returns a file", "cost per call, when it reports one"]
+    else:
+        steps = [("Connect", "GET {base_url}/health"), ("Read its endpoints", "from OpenAPI when you give it"),
+                 ("One harmless call", "GET only")]
+        result, stats = '{ "ok": true }', "0.2 s"
+        learns = ["endpoints and their parameters", "which calls write data (gated)", "key name only, never the value"]
+    mode = ("Live: works today with your own key or on your machine. This test run is a mock, so nothing is sent."
+            if entry.get("live") else "Demo only: example data, no network, no key needed.")
+    return {"id": entry["id"], "name": name, "mock": True, "mode": mode,
+            "steps": [{"title": t, "text": x} for t, x in steps], "result": result, "stats": stats, "learns": learns}

flowforge/connectors/models.py

@@ -0,0 +1,149 @@
+"""Connector models (D10; MCP D12; local D13). Shape follows CLAUDE.md §7.1.
+
+A connector is configuration: who to call, how to connect, which secret (by reference
+only) and how it looks. Nodes are built from connectors; the executor never sees these.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator
+
+from flowforge.connectors.secrets import SECRET_REF_PATTERN, check_ref
+
+CONNECTOR_ID_PATTERN = r"^[A-Za-z][A-Za-z0-9_-]{0,63}$"
+ENV_VAR_PATTERN = r"^[A-Za-z_][A-Za-z0-9_]*$"
+# Step types name their default node in the executor's node map (D10), so no connector may use them.
+RESERVED_IDS = frozenset({"llm", "mcp", "http", "local", "mock"})
+
+
+class _Model(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+# --- style (stored now, rendered from Phase 2) --------------------------------------------------
+
+class LettersLogo(_Model):
+    type: Literal["letters"] = "letters"
+    text: str = Field(max_length=2)
+
+
+class UploadLogo(_Model):
+    type: Literal["upload"] = "upload"
+    file: str
+
+
+class NoLogo(_Model):
+    type: Literal["none"] = "none"
+
+
+class Style(_Model):
+    color: str | None = Field(default=None, pattern=r"^#[0-9A-Fa-f]{6}$")
+    logo: Annotated[LettersLogo | UploadLogo | NoLogo, Field(discriminator="type")] = NoLogo()
+
+
+# --- connections, one per type ------------------------------------------------------------------
+
+class LLMConnection(_Model):
+    provider: Literal["openai_compatible", "anthropic"]
+    base_url: str | None = None  # required for openai_compatible
+    model: str
+    temperature_default: float = Field(default=0, ge=0, le=2)
+
+    @model_validator(mode="after")
+    def _base_url_for_openai_compatible(self) -> LLMConnection:
+        if self.provider == "openai_compatible" and not self.base_url:
+            raise ValueError("openai_compatible connections need base_url")
+        return self
+
+
+class MCPConnection(_Model):
+    command: str
+    args: list[str] = Field(default_factory=list)
+    env: dict[Annotated[str, Field(pattern=ENV_VAR_PATTERN)], str] = Field(default_factory=dict)
+    env_refs: dict[Annotated[str, Field(pattern=ENV_VAR_PATTERN)],
+                   Annotated[str, Field(pattern=SECRET_REF_PATTERN)]] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _refs_are_names(self) -> MCPConnection:
+        _check_env_refs(self.env_refs)
+        return self
+
+
+def _check_env_refs(refs: dict[str, str]) -> None:
+    for ref in refs.values():
+        check_ref(ref)
+
+
+class HTTPConnection(_Model):
+    base_url: str | None = None
+    auth_header: str | None = None  # header name only, e.g. "Authorization"
+    auth_scheme: Literal["bearer", "basic", "raw"] | None = None
+
+
+class LocalConnection(_Model):
+    command: list[str] = Field(min_length=1)
+    cwd: str
+    env: dict[Annotated[str, Field(pattern=ENV_VAR_PATTERN)], str] = Field(default_factory=dict)
+    env_refs: dict[Annotated[str, Field(pattern=ENV_VAR_PATTERN)],
+                   Annotated[str, Field(pattern=SECRET_REF_PATTERN)]] = Field(default_factory=dict)
+    max_output_bytes: int = Field(default=1_000_000, gt=0)
+
+    @model_validator(mode="after")
+    def _refs_are_names(self) -> LocalConnection:
+        _check_env_refs(self.env_refs)
+        return self
+
+
+# --- connectors ---------------------------------------------------------------------------------
+
+class ConnectorBase(_Model):
+    id: str = Field(pattern=CONNECTOR_ID_PATTERN)
+    name: str
+    role: str = ""
+    slot: str | None = None
+    style: Style = Field(default_factory=Style)
+    secret_ref: str | None = Field(default=None, pattern=SECRET_REF_PATTERN)
+    mode: Literal["test", "live"] | None = None
+    rate_limit_rpm: float | None = Field(default=None, gt=0)
+    data_sent: Literal["all", "redacted_only"] = "all"  # behaviour arrives in Phase 4
+    fallback: str | None = Field(default=None, pattern=CONNECTOR_ID_PATTERN)
+    created_at: str | None = None
+
+    @model_validator(mode="after")
+    def _fallback_not_self(self) -> ConnectorBase:
+        check_ref(self.secret_ref)
+        if self.id in RESERVED_IDS:
+            raise ValueError(f"'{self.id}' is reserved for the {self.id} type's default node; pick another id")
+        if self.fallback == self.id:
+            raise ValueError(f"connector '{self.id}' cannot fall back to itself")
+        return self
+
+
+class LLMConnector(ConnectorBase):
+    type: Literal["llm"]
+    connection: LLMConnection
+
+
+class MCPConnector(ConnectorBase):
+    type: Literal["mcp"]
+    connection: MCPConnection
+
+
+class HTTPConnector(ConnectorBase):
+    type: Literal["http"]
+    connection: HTTPConnection = Field(default_factory=HTTPConnection)
+
+
+class LocalConnector(ConnectorBase):
+    type: Literal["local"]
+    connection: LocalConnection
+
+
+Connector = Annotated[LLMConnector | MCPConnector | HTTPConnector | LocalConnector, Field(discriminator="type")]
+_ADAPTER: TypeAdapter[Connector] = TypeAdapter(Connector)
+
+
+def parse_connector(data: Any) -> Connector:
+    return _ADAPTER.validate_python(data)

flowforge/connectors/presets.py

@@ -0,0 +1,192 @@
+"""Preset connectors (CLAUDE.md §7.3). The only place in the code where brands appear.
+
+Each preset is an ordinary Connector holding secret *references*. Default models and URLs
+come from each provider's own docs (checked 2026-10-07); change them per connector.
+Brand colours are approximate tiles with a two-letter monogram, never real logos.
+"""
+
+from __future__ import annotations
+
+import os
+
+from flowforge.connectors.models import Connector, parse_connector
+
+DEFAULT_CONNECTORS = ("nim", "fetch")  # seeded on first start: the default llm and mcp (D10)
+
+
+def _letters(text: str, color: str) -> dict:
+    return {"color": color, "logo": {"type": "letters", "text": text}}
+
+
+PRESETS: dict[str, Connector] = {c.id: c for c in map(parse_connector, [
+    {"id": "nim", "type": "llm", "name": "NVIDIA NIM", "role": "free hosted models (default)",
+     "style": _letters("Nv", "#76B900"), "rate_limit_rpm": 40, "secret_ref": "env:NVIDIA_API_KEY",
+     "connection": {"provider": "openai_compatible", "base_url": "https://integrate.api.nvidia.com/v1",
+                    "model": "meta/llama-3.1-8b-instruct"}},
+    {"id": "gemini", "type": "llm", "name": "Gemini", "role": "Google's models",
+     "style": _letters("Ge", "#4285F4"), "secret_ref": "env:GEMINI_API_KEY",
+     "connection": {"provider": "openai_compatible",
+                    "base_url": "https://generativelanguage.googleapis.com/v1beta/openai/",
+                    "model": "gemini-3.8-flash"}},
+    {"id": "claude", "type": "llm", "name": "Claude", "role": "Anthropic's models",
+     "style": _letters("Cl", "#D97757"), "secret_ref": "env:ANTHROPIC_API_KEY",
+     "connection": {"provider": "anthropic", "model": "claude-opus-5-5"}},
+    {"id": "ollama", "type": "llm", "name": "Ollama", "role": "models on your machine",
+     "style": _letters("Ol", "#555555"),  # no key: a local server; set model to one you've pulled
+     "connection": {"provider": "openai_compatible", "base_url": "http://localhost:11434/v1",
+                    "model": "gpt-oss:20b"}},
+    {"id": "razorpay", "type": "http", "name": "Razorpay", "role": "payments", "slot": "payments",
+     "style": _letters("Rz", "#3395FF"), "mode": "test", "secret_ref": "env:RAZORPAY_KEY",  # "key_id:key_secret"
+     "connection": {"base_url": "https://api.razorpay.com/v1", "auth_header": "Authorization",
+                    "auth_scheme": "basic"}},
+    {"id": "stripe", "type": "http", "name": "Stripe", "role": "payments", "slot": "payments",
+     "style": _letters("St", "#635BFF"), "mode": "test", "secret_ref": "env:STRIPE_SECRET_KEY",
+     "connection": {"base_url": "https://api.stripe.com/v1", "auth_header": "Authorization",
+                    "auth_scheme": "bearer"}},
+    {"id": "fetch", "type": "mcp", "name": "Fetch", "role": "reads web pages (default MCP)",
+     "style": _letters("Fe", "#8A8A8A"),
+     "connection": {"command": "uvx", "args": ["mcp-server-fetch"]}},
+])}
+
+# The connector catalog (D17): many apps, each with a mock test run. `live` means FlowForge can call it
+# today with the user's own key or on their machine; the rest are "Demo only" until their integration lands.
+# Fields: id, name, category, connector type, live, one-line description, meta line.
+CATEGORIES = {"llm": "LLMs", "pay": "Payments", "dev": "Dev tools", "data": "Data and apps",
+              "media": "Media and AI services", "custom": "Your own"}
+_CATALOG_ROWS: list[tuple[str, str, str, str, bool, str, str]] = [
+    ("nim", "NVIDIA NIM", "llm", "llm", True, "Fast open models on a free tier.", "openai-compatible · 40 rpm free"),
+    ("gemini", "Google Gemini", "llm", "llm", True, "Quick general model with a generous free tier.", "openai-compatible · 15 rpm free"),
+    ("claude", "Claude", "llm", "llm", True, "Careful writing, code and long documents.", "anthropic · your key"),
+    ("openai", "OpenAI", "llm", "llm", True, "GPT models through the standard API.", "openai-compatible · your key"),
+    ("ollama", "Ollama", "llm", "llm", True, "Models on your own machine. Nothing leaves it.", "local · port 11434"),
+    ("mistral", "Mistral", "llm", "llm", True, "European models, from small to large.", "openai-compatible · your key"),
+    ("groq", "Groq", "llm", "llm", True, "Very fast answers from open models.", "openai-compatible · free tier"),
+    ("openrouter", "OpenRouter", "llm", "llm", True, "One key for hundreds of models.", "openai-compatible · your key"),
+    ("deepseek", "DeepSeek", "llm", "llm", True, "Strong reasoning at a low price.", "openai-compatible · your key"),
+    ("perplexity", "Perplexity", "llm", "http", False, "Answers with web sources attached.", "HTTP · search"),
+    ("huggingface", "Hugging Face", "llm", "http", False, "Inference for thousands of open models.", "HTTP · inference"),
+    ("together", "Together AI", "llm", "llm", False, "Hosted open models and fine-tunes.", "openai-compatible"),
+    ("cohere", "Cohere", "llm", "http", False, "Embeddings and reranking for search.", "HTTP · embed, rerank"),
+    ("razorpay", "Razorpay", "pay", "http", False, "Payments in India. Starts in test mode.", "HTTP · money moves only behind a gate"),
+    ("stripe", "Stripe", "pay", "http", False, "Global payments. Starts in test mode.", "HTTP · money moves only behind a gate"),
+    ("paypal", "PayPal", "pay", "http", False, "PayPal wallets and cards.", "HTTP · money moves only behind a gate"),
+    ("square", "Square", "pay", "http", False, "In-person and online payments.", "HTTP · money moves only behind a gate"),
+    ("adyen", "Adyen", "pay", "http", False, "Payments for large shops.", "HTTP · money moves only behind a gate"),
+    ("github", "GitHub", "dev", "mcp", True, "Issues, pull requests and code search.", "MCP · 26 tools"),
+    ("sirius", "Sirius", "dev", "local", True, "Your security scanner, run on this machine.", "local · no shell · one folder"),
+    ("playwright", "Playwright", "dev", "mcp", True, "Drives a real browser to test pages.", "MCP · 21 tools"),
+    ("filesystem", "Filesystem", "dev", "mcp", True, "Reads and writes files in one folder.", "MCP · 11 tools"),
+    ("gitlab", "GitLab", "dev", "mcp", False, "Merge requests, pipelines and issues.", "MCP · demo"),
+    ("docker", "Docker", "dev", "mcp", False, "Start, stop and inspect containers.", "MCP · demo"),
+    ("sentry", "Sentry", "dev", "mcp", False, "Errors and traces from production.", "MCP · demo"),
+    ("vercel", "Vercel", "dev", "mcp", False, "Deploys, domains and logs.", "MCP · demo"),
+    ("linear", "Linear", "dev", "mcp", False, "Issues and projects.", "MCP · demo"),
+    ("jira", "Jira", "dev", "http", False, "Tickets and sprints.", "HTTP · demo"),
+    ("fetch", "Fetch", "data", "mcp", True, "Fetches any web page as clean text.", "MCP · uvx mcp-server-fetch"),
+    ("postgres", "PostgreSQL", "data", "mcp", True, "Asks your database questions, read-only.", "MCP · 3 tools · read-only"),
+    ("supabase", "Supabase", "data", "mcp", False, "Postgres, auth and storage in one.", "MCP · demo"),
+    ("mongodb", "MongoDB", "data", "mcp", False, "Document database queries.", "MCP · demo"),
+    ("redis", "Redis", "data", "http", False, "Fast key-value cache.", "HTTP · demo"),
+    ("notion", "Notion", "data", "mcp", False, "Pages and databases in your workspace.", "MCP · demo"),
+    ("slack", "Slack", "data", "mcp", False, "Post and read messages in channels.", "MCP · demo · sending waits for you"),
+    ("gmail", "Gmail", "data", "mcp", False, "Send and read email.", "MCP · demo · sending waits for you"),
+    ("gdrive", "Google Drive", "data", "mcp", False, "Find and read files in Drive.", "MCP · demo"),
+    ("gsheets", "Google Sheets", "data", "http", False, "Read and append rows.", "HTTP · demo"),
+    ("airtable", "Airtable", "data", "http", False, "Tables your team already uses.", "HTTP · demo"),
+    ("discord", "Discord", "data", "http", False, "Messages and webhooks.", "HTTP · demo"),
+    ("telegram", "Telegram", "data", "http", False, "Bots and chat messages.", "HTTP · demo"),
+    ("whatsapp", "WhatsApp", "data", "http", False, "Business messages to customers.", "HTTP · demo · sending waits for you"),
+    ("twilio", "Twilio", "data", "http", False, "SMS and voice calls.", "HTTP · demo · sending waits for you"),
+    ("resend", "Resend", "data", "http", False, "Email from your app.", "HTTP · demo"),
+    ("shopify", "Shopify", "data", "http", False, "Products, orders and customers.", "HTTP · demo"),
+    ("firebase", "Firebase", "data", "http", False, "App data and auth.", "HTTP · demo"),
+    ("higgsfield", "Higgsfield", "media", "http", False, "Images and video from a prompt.", "HTTP · demo"),
+    ("elevenlabs", "ElevenLabs", "media", "http", False, "Natural voices and speech.", "HTTP · demo"),
+    ("replicate", "Replicate", "media", "http", False, "Run open models by API.", "HTTP · demo"),
+    ("stability", "Stability AI", "media", "http", False, "Image generation and editing.", "HTTP · demo"),
+    ("runway", "Runway", "media", "http", False, "Video generation and editing.", "HTTP · demo"),
+    ("deepgram", "Deepgram", "media", "http", False, "Speech to text, fast.", "HTTP · demo"),
+    ("assemblyai", "AssemblyAI", "media", "http", False, "Speech to text and audio insights.", "HTTP · demo"),
+    ("custom_mcp", "Any MCP server", "custom", "mcp", True, "A command and its arguments. Tools are found for you.", "MCP · your colour and logo"),
+    ("custom_http", "Any HTTP API", "custom", "http", True, "A base URL and the name of its key.", "HTTP · OpenAPI optional"),
+    ("custom_local", "Local command", "custom", "local", True, "Any script on your machine. No shell, one folder.", "local · your colour and logo"),
+]
+CATALOG: list[dict] = [
+    {"id": r[0], "name": r[1], "category": r[2], "type": r[3], "live": r[4], "description": r[5], "meta": r[6]}
+    for r in _CATALOG_ROWS
+]
+
+# What the repo scanner recognises (D21). One row per service: id (the catalog id where there is one), name,
+# kind, hosts it answers on (a leading "." matches any subdomain; "host:port" for local servers) and its SDK
+# packages (Python import or dist names, npm names, Dart packages; compared case-insensitively with - _ . alike).
+_SERVICE_ROWS: list[tuple[str, str, str, tuple[str, ...], tuple[str, ...]]] = [
+    ("ollama", "Ollama", "llm", ("localhost:11434", "127.0.0.1:11434"), ("ollama", "langchain_ollama", "ollama-ai-provider")),
+    ("nim", "NVIDIA NIM", "llm", ("integrate.api.nvidia.com",), ("langchain_nvidia_ai_endpoints",)),
+    ("openai", "OpenAI", "llm", ("api.openai.com",), ("openai", "langchain_openai", "@ai-sdk/openai", "dart_openai")),
+    ("claude", "Claude", "llm", ("api.anthropic.com",), ("anthropic", "@anthropic-ai/sdk", "langchain_anthropic", "@ai-sdk/anthropic")),
+    ("gemini", "Google Gemini", "llm", ("generativelanguage.googleapis.com",),
+     ("google.generativeai", "google.genai", "google-generativeai", "google-genai", "@google/generative-ai", "@google/genai",
+      "langchain_google_genai", "google_generative_ai", "@ai-sdk/google")),
+    ("mistral", "Mistral", "llm", ("api.mistral.ai",), ("mistralai", "@mistralai/mistralai", "langchain_mistralai")),
+    ("cohere", "Cohere", "llm", ("api.cohere.com", "api.cohere.ai"), ("cohere", "cohere-ai", "langchain_cohere")),
+    ("groq", "Groq", "llm", ("api.groq.com",), ("groq", "groq-sdk", "langchain_groq")),
+    ("openrouter", "OpenRouter", "llm", ("openrouter.ai",), ("@openrouter/ai-sdk-provider",)),
+    ("deepseek", "DeepSeek", "llm", ("api.deepseek.com",), ()),
+    ("supabase", "Supabase", "api", (".supabase.co",), ("supabase", "@supabase/supabase-js", "supabase_flutter")),
+    ("stripe", "Stripe", "api", ("api.stripe.com",), ("stripe", "flutter_stripe", "@stripe/stripe-js")),
+    ("razorpay", "Razorpay", "api", ("api.razorpay.com",), ("razorpay", "razorpay_flutter")),
+    ("firebase", "Firebase", "api", (".firebaseio.com", "firestore.googleapis.com", ".firebaseapp.com"),
+     ("firebase_admin", "firebase", "firebase-admin", "firebase_core", "cloud_firestore", "firebase_auth")),
+    ("google_maps", "Google Maps", "api", ("maps.googleapis.com",), ("googlemaps", "google_maps_flutter", "@googlemaps/js-api-loader")),
+    ("osm_tiles", "OpenStreetMap tiles", "api", ("tile.openstreetmap.org",), ()),
+    ("nominatim", "Nominatim", "api", ("nominatim.openstreetmap.org",), ()),
+    ("overpass", "Overpass API", "api", ("overpass-api.de",), ()),
+    ("osrm", "OSRM", "api", ("router.project-osrm.org",), ()),
+    ("open_meteo", "Open-Meteo", "api", ("api.open-meteo.com",), ()),
+    ("wikipedia", "Wikipedia", "api", (".wikipedia.org",), ()),
+    ("wikimedia", "Wikimedia", "api", (".wikimedia.org",), ()),
+]
+SERVICES: list[dict] = [{"id": r[0], "name": r[1], "kind": r[2], "hosts": list(r[3]), "sdks": list(r[4])}
+                        for r in _SERVICE_ROWS]
+# Where a health check reads an LLM's model list, with no key (D24). "{host}" is the address the code uses.
+MODEL_LISTS: dict[str, str] = {
+    "ollama": "http://{host}/api/tags", "nim": "https://integrate.api.nvidia.com/v1/models",
+    "openai": "https://api.openai.com/v1/models", "claude": "https://api.anthropic.com/v1/models",
+    "gemini": "https://generativelanguage.googleapis.com/v1beta/models", "mistral": "https://api.mistral.ai/v1/models",
+    "cohere": "https://api.cohere.com/v1/models", "groq": "https://api.groq.com/openai/v1/models",
+    "openrouter": "https://openrouter.ai/api/v1/models", "deepseek": "https://api.deepseek.com/models",
+}
+# How a service's SDK client is reached in Dart/JS/TS code (D32): a call through it, or through a name bound to it,
+# is a call to that service
+CLIENTS: dict[str, tuple[str, ...]] = {
+    "supabase": (r"Supabase\.instance\.client", r"\bcreateClient\s*\(", r"\bSupabaseClient\s*\("),
+    "firebase": (r"FirebaseFirestore\.instance", r"\bgetFirestore\s*\(", r"FirebaseAuth\.instance"),
+}
+# SDKs that route to many providers (D21): the provider comes from the model name, by these prefixes
+ROUTER_SDKS = ("litellm",)
+MODEL_PREFIXES: dict[str, tuple[str, ...]] = {
+    "claude": ("anthropic/", "claude-"), "openai": ("openai/", "gpt-", "o1", "o3", "o4"),
+    "gemini": ("gemini/", "gemini-", "vertex_ai/gemini"), "ollama": ("ollama/", "ollama_chat/"), "groq": ("groq/",),
+    "mistral": ("mistral/", "mistral-", "codestral"), "openrouter": ("openrouter/",), "nim": ("nvidia_nim/",),
+    "deepseek": ("deepseek/",), "cohere": ("cohere/", "command-"),
+}
+
+# Starting points for the "Custom" choice of each type; the user fills in id, name and connection.
+CUSTOM: dict[str, dict] = {
+    "llm": {"type": "llm", "connection": {"provider": "openai_compatible", "base_url": "", "model": ""}},
+    "mcp": {"type": "mcp", "connection": {"command": "", "args": []}},
+    "http": {"type": "http", "connection": {"base_url": "", "auth_header": "Authorization", "auth_scheme": "bearer"}},
+    "local": {"type": "local", "connection": {"command": [], "cwd": ""}},
+}
+
+
+def with_env_overrides(connector: Connector) -> Connector:
+    """The default NIM connector keeps honouring NIM_RPM, NIM_BASE_URL and NIM_DEFAULT_MODEL (D8, D10)."""
+    if connector.id != "nim" or connector.type != "llm":
+        return connector
+    connection = connector.connection.model_copy(update={
+        "base_url": os.getenv("NIM_BASE_URL") or connector.connection.base_url,
+        "model": os.getenv("NIM_DEFAULT_MODEL") or connector.connection.model,
+    })
+    rpm = float(os.getenv("NIM_RPM")) if os.getenv("NIM_RPM") else connector.rate_limit_rpm
+    return connector.model_copy(update={"connection": connection, "rate_limit_rpm": rpm})

flowforge/connectors/registry.py

@@ -0,0 +1,85 @@
+"""Connector registry: the `connectors` table, default seeding and connectors.json (D10).
+
+Phase 1 has no write routes. Connectors come from the seeded defaults (nim, fetch) and the
+optional FLOWFORGE_HOME/connectors.json, a JSON list of connectors validated by the same
+models and upserted at start-up. A bad file stops start-up with a clear message rather
+than running with half the connectors.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from flowforge.connectors.models import Connector, parse_connector
+from flowforge.connectors.presets import DEFAULT_CONNECTORS, PRESETS
+from flowforge.storage import Storage
+
+
+class ConnectorFileError(ValueError):
+    pass
+
+
+class Registry:
+    def __init__(self, storage: Storage):
+        self.storage = storage
+
+    def all(self) -> list[Connector]:
+        return [parse_connector(json.loads(raw)) for _, raw in self.storage.connector_rows()]
+
+    def get(self, connector_id: str) -> Connector | None:
+        return next((c for c in self.all() if c.id == connector_id), None)
+
+    def save(self, connector: Connector) -> None:
+        self.storage.put_connector(connector.id, connector.model_dump_json())
+
+    def delete(self, connector_id: str) -> None:
+        self.storage.delete_connector(connector_id)
+
+    def ensure_defaults(self) -> None:
+        """Seed the default connectors once; later edits to them are kept."""
+        existing = {c.id for c in self.all()}
+        for connector_id in DEFAULT_CONNECTORS:
+            if connector_id not in existing:
+                self.save(PRESETS[connector_id])
+
+
+def _describe(exc: ValidationError) -> str:
+    """Field paths and messages only: never echo input values (a pasted key must not be repeated)."""
+    return "; ".join(
+        f"{'.'.join(map(str, e['loc'])) or 'connector'}: {e['msg']}"
+        for e in exc.errors(include_input=False, include_url=False)
+    )
+
+
+def load_connectors_file(path: Path) -> list[Connector]:
+    if not path.exists():
+        return []
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ConnectorFileError(f"{path}: not valid JSON (line {exc.lineno}, column {exc.colno})") from None
+    if not isinstance(data, list):
+        raise ConnectorFileError(f"{path}: expected a JSON list of connectors")
+    connectors: list[Connector] = []
+    for i, item in enumerate(data):
+        try:
+            connectors.append(parse_connector(item))
+        except ValidationError as exc:
+            raise ConnectorFileError(f"{path}: connector #{i + 1}: {_describe(exc)}") from None
+    return connectors
+
+
+def check_references(connectors: list[Connector]) -> None:
+    """Every fallback names an existing connector of the same type (D10)."""
+    by_id = {c.id: c for c in connectors}
+    for c in connectors:
+        if c.fallback is None:
+            continue
+        target = by_id.get(c.fallback)
+        if target is None:
+            raise ConnectorFileError(f"connector '{c.id}' falls back to '{c.fallback}', which does not exist")
+        if target.type != c.type:
+            raise ConnectorFileError(f"connector '{c.id}' ({c.type}) falls back to '{c.fallback}' ({target.type})")

flowforge/connectors/runtime.py

@@ -0,0 +1,47 @@
+"""Connectors → runtime nodes and rate-limit buckets (D10, D16)."""
+
+from __future__ import annotations
+
+import os
+
+from flowforge.connectors.models import Connector
+from flowforge.connectors.presets import with_env_overrides
+from flowforge.nodes.base import Node
+from flowforge.nodes.check_node import CheckNode
+from flowforge.nodes.http_node import HTTPNode
+from flowforge.nodes.llm_node import LLMNode
+from flowforge.nodes.local_node import LocalNode
+from flowforge.nodes.mcp_node import MCPNode
+from flowforge.nodes.mock_node import MockNode
+from flowforge.scheduler.rate_limit import TokenBucket
+
+NODE_BUILDERS = {"llm": LLMNode.from_connector, "mcp": MCPNode.from_connector,
+                 "http": HTTPNode.from_connector, "local": LocalNode.from_connector}
+
+
+def build_node(connector: Connector) -> tuple[Node, TokenBucket | None]:
+    """The node for one connector, and its own bucket if it has a rate limit."""
+    connector = with_env_overrides(connector)
+    node = NODE_BUILDERS[connector.type](connector)
+    if connector.id == "nim" and isinstance(node, LLMNode):
+        node.default_for_type = True  # the default llm keeps the V1 cache namespace
+    bucket = TokenBucket(connector.rate_limit_rpm) if connector.rate_limit_rpm and node.rate_limit_key else None
+    return node, bucket
+
+
+def build_nodes(connectors: list[Connector]) -> tuple[dict[str, Node], dict[str, TokenBucket]]:
+    """One node per connector, keyed by id, plus each type's default under the type name (D10).
+
+    The default llm is the `nim` connector; it keeps the V1 cache namespace and the "nim"
+    bucket. mcp and http keep their V1 nodes as defaults (a step may still pass params.server).
+    """
+    nodes: dict[str, Node] = {"http": HTTPNode(), "mcp": MCPNode(), "mock": MockNode(), "check": CheckNode()}
+    buckets: dict[str, TokenBucket] = {}
+    for connector in connectors:
+        node, bucket = build_node(connector)
+        nodes[connector.id] = node
+        if bucket is not None and node.rate_limit_key:
+            buckets[node.rate_limit_key] = bucket
+    nodes["llm"] = nodes["nim"] if isinstance(nodes.get("nim"), LLMNode) else LLMNode()
+    buckets.setdefault("nim", TokenBucket(float(os.getenv("NIM_RPM", "40"))))
+    return nodes, buckets

flowforge/connectors/secrets.py

@@ -0,0 +1,53 @@
+"""Secret references (D10). A connector holds `env:NAME` or `vault:NAME`, never a value.
+
+Phase 1 resolves `env:` from the process environment. `vault:` is reserved for the
+Phase 4 vault and refused until then. Errors name the reference, never a value.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+
+SECRET_REF_PATTERN = r"^(env|vault):[A-Z_][A-Z0-9_]*$"
+
+
+class SecretRefError(ValueError):
+    pass
+
+
+# A key pasted (and upper-cased) into a "variable name" field still has its key shape (D16).
+_KEY_SHAPED_NAME = re.compile(
+    r"^(?:(?:SK|RK)_(?:LIVE|TEST)_|RZP_(?:LIVE|TEST)_|AIZA[0-9A-Z_]{16,}|SK_ANT_)", re.IGNORECASE)
+
+
+def check_ref(ref: str | None) -> str | None:
+    """Pydantic-friendly: refuse a reference whose variable name is really a key."""
+    if ref is not None and _KEY_SHAPED_NAME.match(ref.split(":", 1)[-1]):
+        raise ValueError("that variable name looks like a key; put the key in .env and reference it by name")
+    return ref
+
+
+def resolve_secret(ref: str | None) -> str | None:
+    """The secret value for `ref`, or None when no secret is configured."""
+    if ref is None:
+        return None
+    scheme, _, name = ref.partition(":")
+    if scheme == "vault":
+        raise SecretRefError(f"secret '{ref}' needs the vault, which arrives in Phase 4; use env:{name} for now")
+    if scheme != "env" or not name:
+        raise SecretRefError(f"unsupported secret reference '{ref}'")
+    value = os.environ.get(name)
+    if not value:
+        raise SecretRefError(f"{name} is not set (referenced as '{ref}'); add it to .env")
+    return value
+
+
+def resolve_env(env: dict[str, str], env_refs: dict[str, str]) -> dict[str, str]:
+    """Plain env values plus resolved secret env values (D12, D13)."""
+    resolved = dict(env)
+    for var, ref in env_refs.items():
+        value = resolve_secret(ref)
+        if value is not None:
+            resolved[var] = value
+    return resolved