toolfunnel 0.3.0 → 0.5.0

package/src/core/server-config.js

@@ -0,0 +1,77 @@
+'use strict';
+
+/**
+ * server-config.js — OPTIONAL gateway-level configuration: identity + default ports.
+ *
+ * Reads `toolfunnel.json` at the gateway root (absent by default). This is what lets a
+ * third party ship an MCP built on toolfunnel that introduces itself as ITSELF: the
+ * `initialize` handshake and `/health` report THEIR serverName/serverVersion, and their
+ * pack can carry its own default ports so `npx their-mcp` never collides with another
+ * gateway on the user's machine. CLI flags still win: precedence is flag > config > default.
+ *
+ * Shape (every field optional — an empty/absent file IS the default identity):
+ *   {
+ *     "serverName":    "my-mcp",     // initialize serverInfo.name          (default "toolfunnel")
+ *     "serverVersion": "1.0.0",      // initialize serverInfo.version       (default: package.json)
+ *     "httpPort":      9998,         // --http default port                 (default 9998)
+ *     "uiPort":        9777          // --ui   default port                 (default 9777)
+ *   }
+ *
+ * TOLERANT by design, field by field: a missing file, bad JSON, or an invalid FIELD falls
+ * back to that field's default — a broken identity file must never stop the gateway from
+ * starting as plain "toolfunnel" (the same fail-open-to-default posture as log.config.json;
+ * contrast auth config, which fails CLOSED because it guards a boundary). No new deps.
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+/** The compiled-in defaults. version comes from package.json (single source of truth). */
+function defaults() {
+  let version = '0.0.0';
+  try {
+    version = require('../../package.json').version || version;
+  } catch (_e) {
+    /* unreadable package.json → a placeholder beats a crash */
+  }
+  return { serverName: 'toolfunnel', serverVersion: version, httpPort: 9998, uiPort: 9777 };
+}
+
+/** A valid TCP port for a bind default: an integer 1..65535 (0 = OS-assigned stays flag-only —
+ *  an ephemeral DEFAULT would make the gateway un-findable across restarts). */
+function isPort(v) {
+  return Number.isInteger(v) && v >= 1 && v <= 65535;
+}
+
+/**
+ * Load the gateway config, merged over the defaults.
+ * @param {string} rootDir  the gateway root (the directory that holds toolfunnel.json)
+ * @returns {{serverName:string, serverVersion:string, httpPort:number, uiPort:number}}
+ */
+function loadServerConfig(rootDir) {
+  const out = defaults();
+  let raw;
+  try {
+    raw = fs.readFileSync(path.join(rootDir, 'toolfunnel.json'), 'utf8');
+  } catch (_e) {
+    return out; // absent — the normal case
+  }
+  let cfg;
+  try {
+    cfg = JSON.parse(raw);
+  } catch (_e) {
+    return out; // bad JSON — start as the default identity rather than not at all
+  }
+  if (!cfg || typeof cfg !== 'object' || Array.isArray(cfg)) return out;
+  if (typeof cfg.serverName === 'string' && cfg.serverName.trim().length > 0) {
+    out.serverName = cfg.serverName.trim();
+  }
+  if (typeof cfg.serverVersion === 'string' && cfg.serverVersion.trim().length > 0) {
+    out.serverVersion = cfg.serverVersion.trim();
+  }
+  if (isPort(cfg.httpPort)) out.httpPort = cfg.httpPort;
+  if (isPort(cfg.uiPort)) out.uiPort = cfg.uiPort;
+  return out;
+}
+
+module.exports = { loadServerConfig };

package/src/extend/create-tool.md

@@ -2,8 +2,9 @@
 
 This is the instruction served by `toolfunnel_howto({ topic: "create-tool" })`. It tells you — the
 model — how to add a new tool to the register so it becomes callable through `toolfunnel_run_tool`
-(and listable via `toolfunnel_list_tools` / `toolfunnel_tool_instructions`). No restart is needed: the register
-is read live every time the meta-tools are called.
+(and listable via `toolfunnel_list_tools` / `toolfunnel_tool_instructions`). No restart is needed: a
+running gateway watches `tools.register.json` and hot-reloads its register on change (debounced
+~150 ms), the same way `mcp/expose.json` and the hook files reload.
 
 A tool is exactly two things:
 1. **A register entry** in `tools/tools.register.json` (structured metadata + how to invoke it).
@@ -23,6 +24,12 @@ A tool is exactly two things:
   "summary": "Uppercases the supplied text and echoes it back.",
   "category": "demo",
   "instructions": "Pass { text: string }. Returns the text uppercased. Use this only as a sanity check of the run path.",
+  "inputSchema": {
+    "type": "object",
+    "properties": { "text": { "type": "string", "description": "The text to uppercase." } },
+    "required": ["text"],
+    "additionalProperties": false
+  },
   "invoke": { "type": "script", "path": "scripts/echo-upper.sh" }
 }
 ```
@@ -36,6 +43,7 @@ Field-by-field — these names are fixed (registry.js reads exactly these):
 | `summary` | string | One line. This is ALL that `toolfunnel_list_tools` returns for the tool — keep it tight and discriminating so the model can pick the right tool from the brief alone. |
 | `category` | string | Grouping label (e.g. `demo`, `memory`, `messaging`, `media`). Used by `toolfunnel_list_tools({ category })` to filter. |
 | `instructions` | string | Full usage doc returned by `toolfunnel_tool_instructions({ name })`. Describe the args object, what comes back, and any safety caveats. This is the long-tail payload — be complete here, terse in `summary`. |
+| `inputSchema` | object (optional) | A JSON Schema for the tool's args. **This is what makes the tool a first-class MCP citizen when promoted hot:** the top-level `tools/list` advertises it VERBATIM, so a connected client knows the exact arg shape without reading `instructions`. Omit it and a hot-promoted tool falls back to a free-form `{ "type": "object" }` (callable, but the client gets no arg hints). Must be an object — the register rejects any other shape at authoring time. |
 | `invoke` | object | How `run` executes the tool. See below. |
 
 ### The `invoke` object — two forms
@@ -128,5 +136,17 @@ matches its `tool_name` (see `toolfunnel_howto({ topic: "add-hook" })`), not omi
 5. Test with `toolfunnel_run_tool({ name, args })`. Verify the gate behaves: a benign call returns
    `{ ok:true, output }`; if a matching PreToolUse hook should deny it, confirm it does.
 
-No reconnect is needed — the register is dynamic. (Only *curated-direct* tools negotiated at MCP
-connect need a reconnect; register tools reached via the meta-tools are always live.)
+No reconnect is needed — the register is dynamic: the running gateway hot-reloads
+`tools.register.json` on change and emits `notifications/tools/list_changed`, exactly like an
+`expose.json` or hook edit.
+
+---
+
+## 5. Promoting your tool to the top-level surface (the "instant MCP" recipe)
+
+By default your tool lives behind the lean meta-tools (discovered via `toolfunnel_list_tools`).
+Set `hot: true` for its id in `tools/tools.state.json` (or via the UI / `tf_tool_set`) and it is
+advertised DIRECTLY in the MCP `tools/list` — under its own `id`, with your `summary` as the
+description and your `inputSchema` as the advertised schema. Author a real `inputSchema` before
+promoting: that schema is the difference between a client calling your tool correctly first time
+and it guessing at a free-form object.

package/src/extend/package.md

@@ -5,6 +5,39 @@ tools, hooks, and MCP configs into a **shareable package** and how import/export
 **byproduct** of the gateway design: because every unit is already structured and
 self-documented, a package is just a folder that lists its units and carries their files.
 
+---
+
+## 0. The WHOLE-SETUP path — `tf_pack` (start here)
+
+Since 0.4.0 the fastest way to ship is not per-unit packaging but the **whole setup**: your entire
+config home (tools + scripts, hot/hidden curation, upstream references, policy hooks, identity) is
+already the product. One call packages it into a deployment-ready artifact under `<home>/dist/`:
+
+- `tf_pack { format: "home" }` → a **portable config home**. Zip it, git-init it, or run it
+  directly: `toolfunnel --config-dir <dir>`. The receiving gateway seeds nothing — the pack IS the
+  config, including the hooks: **the gate travels**, enforcing your policy on the recipient's
+  machine regardless of which MCP client they use.
+- `tf_pack { format: "npm", name: "my-mcp" }` → a **publishable npm package**: a generated
+  `package.json` that depends on `toolfunnel` (caret range — DEPEND, never copy the engine), a
+  2-line bin launcher pointing `--config-dir` at the bundled home, and a README stub.
+  `cd dist/my-mcp && npm publish` → your users run `npx my-mcp` and get YOUR server, under YOUR
+  name (the bundled `toolfunnel.json` identity), with YOUR gate.
+
+Declare the runtimes your tools need in the home's `toolfunnel.json` — they travel with the pack
+and the receiving gateway warns at startup about anything missing:
+
+```json
+"requires": [ { "command": "python", "min": "3.10", "why": "the pdf tools" } ]
+```
+
+Audit honesty (both directions): packs spawn commands. Tell your users to read your pack's
+`expose.json`, `tools.register.json`, AND `toolfunnel.json` (its `requires` probes execute at
+gateway startup — bare program names only, the loader rejects anything shell-shaped) — and read
+all three yourself in anything you install.
+
+The per-UNIT packaging below remains the right shape for sharing a single tool/hook/MCP selection
+rather than a whole gateway.
+
 > Phase note: packages are Phase 2 (`packages/` directory). The structure below is what the importer
 > and exporter target; defining it now means the units authored today are already package-ready.
 

package/src/mcp/aggregator.js

@@ -49,10 +49,12 @@ const path = require('node:path');
 const { McpClient } = require('./mcp-client');
 const logger = require('../core/logger');
 
-// The sandbox root, resolved exactly the way server.js does (this file lives in
-// src/mcp/, so two levels up is the gateway root). Used as the isolation boundary the
-// default clientFactory enforces. A caller may override via opts.v3Root for tests.
-const DEFAULT_ROOT = path.resolve(__dirname, '..', '..');
+// The sandbox root = the CONFIG HOME (TOOLFUNNEL_HOME / --config-dir; defaults to the package
+// root — see src/core/config-home.js). Used as the isolation boundary the default clientFactory
+// enforces: a pack's path-shaped upstream args live under the home, so the home is the boundary.
+// A caller may override via opts.v3Root for tests.
+const { resolveConfigHome } = require('../core/config-home');
+const DEFAULT_ROOT = resolveConfigHome();
 
 /** True for a non-empty string. */
 function isNonEmptyString(v) {
@@ -522,9 +524,6 @@ class Aggregator {
     const upstream = this._store.getUpstream(upstreamId);
     if (!upstream || upstream.enabled !== true) return; // detached/disabled — stop trying
     const MAX_FAST_ATTEMPTS = 6; // exponential phase; beyond it, a slow keepalive at the 30s cap
-    if (attempt === MAX_FAST_ATTEMPTS) {
-      logger.log({ type: 'mcp', event: 'reconnect_slow', upstream: upstreamId, attempts: attempt });
-    }
     const delay = Math.min(1000 * Math.pow(2, attempt), 30000);
     const timer = setTimeout(async () => {
       this._reconnectTimers.delete(upstreamId);
@@ -537,6 +536,12 @@ class Aggregator {
         this._signalToolsChanged(upstreamId);
       } catch (err) {
         logger.log({ type: 'mcp', event: 'reconnect_failed', upstream: upstreamId, attempt, error: errMsg(err) });
+        // Log the fast→slow transition exactly ONCE — when the counter FIRST reaches the cap. The
+        // old guard sat on the re-entry side, where the capped counter re-satisfied it every 30s
+        // forever against a permanently-dead upstream (log spam, not information).
+        if (attempt + 1 === MAX_FAST_ATTEMPTS) {
+          logger.log({ type: 'mcp', event: 'reconnect_slow', upstream: upstreamId, attempts: MAX_FAST_ATTEMPTS });
+        }
         // Keep escalating to the cap, then keep retrying slowly (cap the counter so the delay stays
         // at 30s) — never permanently give up while the upstream stays enabled.
         this._scheduleReconnect(upstreamId, Math.min(attempt + 1, MAX_FAST_ATTEMPTS));

package/src/mcp/http-transport.js

@@ -145,7 +145,14 @@ function isLoopbackHost(hostHeader) {
   return host === '127.0.0.1' || host === 'localhost' || host === '::1' || host === '0:0:0:0:0:0:0:1';
 }
 
-/** Read a request body up to MAX_BODY_BYTES. Resolves the buffered string, or rejects 'too large'. */
+/**
+ * Read a request body up to MAX_BODY_BYTES. Resolves the buffered string, or rejects 'too large'.
+ * On over-cap it does NOT destroy the request — destroying here tears down the shared req/res
+ * socket before the caller's -32700 reply can flush (the client saw ECONNRESET instead of the clean
+ * parse error the declared-Content-Length path produces). Instead: stop BUFFERING (later chunks are
+ * discarded, so memory stays bounded) and reject; the caller replies via sendJsonAndClose, which
+ * flushes the error body first and destroys the request only in the res.end callback.
+ */
 function readBody(req) {
   return new Promise((resolve, reject) => {
     let size = 0;
@@ -157,15 +164,11 @@ function readBody(req) {
       fn(val);
     };
     req.on('data', (chunk) => {
+      if (done) return; // over-cap already signalled — discard, don't buffer
       size += chunk.length;
       if (size > MAX_BODY_BYTES) {
-        // Stop reading; the caller turns this into a parse-error response.
+        // Stop buffering; the caller flush-then-closes with the same -32700 as the declared-CL path.
         finish(reject, new Error('request body too large'));
-        try {
-          req.destroy();
-        } catch (_e) {
-          /* ignore */
-        }
         return;
       }
       chunks.push(chunk);
@@ -193,16 +196,19 @@ function sendJson(res, status, obj) {
 
 /**
  * Send a JSON response and then CLOSE the connection, draining the unread request body WITHOUT
- * buffering it. Used by the Content-Length pre-check (handleMcpPost): when a client announces an
- * oversized body we must reply cleanly (the documented -32700) *without* reading the multi-MiB
- * payload off the socket. The naive fix — `req.destroy()` straight after `res.end()` — resets the
- * socket before the response bytes flush, surfacing to the client as ECONNRESET instead of the
- * clean parse-error body. The robust sequence is:
+ * buffering it. Used by both over-cap paths (the Content-Length pre-check and readBody's streaming
+ * cap): the client must receive the documented -32700 cleanly while the oversized payload never
+ * touches memory. The naive close — `req.destroy()`, even deferred to the res.end flush callback —
+ * is NOT safe: destroy() sends a TCP RST whenever request bytes are still unread/in-flight, and on
+ * RST Windows discards the peer's receive queue, eating the response we just flushed (the client
+ * saw ECONNRESET instead of the -32700; proven by test 5 in http.test.js). The robust sequence:
  *   1. announce `Connection: close` so the client/keep-alive agent expects the socket to end here,
- *   2. write the full response body, and
- *   3. ONLY in the res.end(...) flush callback destroy the request — by then the response has been
- *      handed to the kernel, so the client receives the -32700 body, THEN sees the close. The
- *      unread request body is discarded (never buffered → no DoS) rather than politely resumed.
+ *   2. write the full response body, then in the res.end(...) flush callback:
+ *   3. resume() the request so any still-arriving body is DISCARDED (never buffered → no DoS),
+ *   4. HALF-CLOSE the socket (socket.end()) — the FIN travels in-order AFTER the response bytes,
+ *      so the client always reads the -32700 first, and
+ *   5. arm an unref'd hard-destroy timer as the backstop: a client that never stops sending and
+ *      never closes gets its socket torn down after a grace window (bounded lifetime, no leak).
  * Defensive: like sendJson, a serialise failure degrades to a 500-shaped error body; the whole
  * function is wrapped so a write-after-headers race can never throw out of handleMcpPost.
  * @param {import('node:http').IncomingMessage} req
@@ -233,9 +239,20 @@ function sendJsonAndClose(req, res, status, obj) {
       // pipelined follow-up (whose body would collide with the one we are about to discard).
       Connection: 'close',
     });
-    // Destroy the request ONLY after the response body has flushed to the kernel, so the client
-    // receives the clean -32700 first and the unread oversized body is dropped, not buffered.
-    res.end(body, destroyReq);
+    res.end(body, () => {
+      // The response has flushed to the kernel. Close down WITHOUT an RST (see the doc above):
+      try {
+        req.resume(); // discard (never buffer) whatever request body is still arriving
+        if (req.socket && !req.socket.destroyed) req.socket.end(); // FIN, in-order after the body
+      } catch (_e) {
+        return destroyReq(); // a torn socket at this point → plain destroy is all that is left
+      }
+      // Backstop: bound the socket's lifetime if the client neither finishes nor closes. The
+      // timer is unref'd so it never keeps the process alive; a normal close clears it.
+      const t = setTimeout(destroyReq, 10000);
+      if (t && typeof t.unref === 'function') t.unref();
+      if (req.socket) req.socket.once('close', () => clearTimeout(t));
+    });
   } catch (_e) {
     // writeHead/end can throw if headers were already sent by a racing path — fall back to a
     // straight destroy so the socket can never be left half-open.
@@ -283,6 +300,8 @@ function createHttpMcpServer(opts = {}) {
   /** @type {boolean} latch so stop() is idempotent and start() can't double-bind. */
   let started = false;
   let stopping = false;
+  /** @type {(() => void)|null} teardown for the config hot-reload watchers (armed in start()). */
+  let configWatchersStop = null;
   /**
    * The result of the LAST connectAll() (from start() or reload()), surfaced by health() so a
    * silent upstream connect-failure becomes VISIBLE (otherwise it only reaches stderr via log()).
@@ -397,6 +416,16 @@ function createHttpMcpServer(opts = {}) {
     return pushToSse({ jsonrpc: JSONRPC, method: 'notifications/tools/list_changed' });
   }
 
+  /**
+   * Wire the CURRENT build's aggregator "tools changed" signal (a background reconnect finally
+   * winning an upstream) to the SSE broadcast, so an HTTP client honouring listChanged re-fetches.
+   * Re-called after every build (re)creation because a fresh aggregator has no listener — the stdio
+   * path does the equivalent in main(). NEVER throws.
+   */
+  function wireAggregatorNotify() {
+    if (build && build.aggregator) build.aggregator.onToolsChanged = () => broadcastToolsListChanged();
+  }
+
   // ── OAuth 2.1 resource-server gate (OPTIONAL; default OFF) ──────────────────────────────────
   // The validator is memoised and rebuilt ONLY when the relevant auth config changes — so a live
   // enable/disable/edit takes effect on the next request WITHOUT discarding jose's JWKS cache in
@@ -501,9 +530,11 @@ function createHttpMcpServer(opts = {}) {
     try {
       raw = await readBody(req);
     } catch (err) {
-      // Oversized / aborted body → a clean parse error, server stays alive.
+      // Oversized / aborted body → the SAME flush-then-close -32700 as the Content-Length pre-check
+      // (readBody no longer destroys the request, so the reply can actually reach the client; a
+      // genuinely-dead socket is fine too — sendJsonAndClose's writes are defensively wrapped).
       log('POST /mcp body read failed:', (err && err.message) || String(err));
-      return sendJson(res, 200, makeError(ERR.PARSE, 'Parse error: ' + ((err && err.message) || 'bad body')));
+      return sendJsonAndClose(req, res, 200, makeError(ERR.PARSE, 'Parse error: ' + ((err && err.message) || 'bad body')));
     }
 
     let msg;
@@ -763,6 +794,7 @@ function createHttpMcpServer(opts = {}) {
     // the very first request. connectAll() NEVER throws (failures land in failed[]); with the
     // default EMPTY expose.json it is an instant no-op.
     build = buildFactory();
+    wireAggregatorNotify();
     if (build && build.aggregator && typeof build.aggregator.connectAll === 'function') {
       try {
         const res = await build.aggregator.connectAll();
@@ -810,6 +842,15 @@ function createHttpMcpServer(opts = {}) {
         boundPort = addr && typeof addr === 'object' ? addr.port : requestedPort;
         started = true;
         stopping = false;
+        // Arm config hot-reload now the build is live + bound. Pass a GETTER so the reloaders
+        // always target the CURRENT build (reload() rebuilds it); send = pushToSse so a
+        // hook/register/expose change broadcasts notifications/tools/list_changed to SSE clients.
+        // This is the wiring the HTTP host was missing — stdio's main() has always done it.
+        try {
+          configWatchersStop = require('./server').startConfigWatchers(() => build, (n) => pushToSse(n));
+        } catch (e) {
+          log('config watchers failed to arm (host runs, without hot-reload):', (e && e.message) || String(e));
+        }
         // Swap the one-shot bind-error handler for a steady-state one so a later socket error
         // (a client RST, etc.) is logged, not thrown.
         httpServer.on('error', (e) => log('http server error:', (e && e.message) || String(e)));
@@ -836,6 +877,14 @@ function createHttpMcpServer(opts = {}) {
     }
     stopping = true;
 
+    // 0. Disarm the config watchers FIRST, so no debounced reload can fire against the build we're
+    //    about to tear down (a reload mutating a half-closed aggregator would be a benign no-op, but
+    //    disarming first is the clean order).
+    if (configWatchersStop) {
+      try { configWatchersStop(); } catch (_e) { /* never throw */ }
+      configWatchersStop = null;
+    }
+
     // 1. Tear down SSE clients: clear their keep-alive timers and end the responses so their
     //    sockets close (an open SSE socket would otherwise keep the server from closing).
     for (const res of Array.from(sseClients)) {
@@ -915,6 +964,7 @@ function createHttpMcpServer(opts = {}) {
     await closeAggregatorSafely();
     // 2. Rebuild from the edited config + reconnect.
     build = buildFactory();
+    wireAggregatorNotify();
     if (build && build.aggregator && typeof build.aggregator.connectAll === 'function') {
       try {
         const res = await build.aggregator.connectAll();

package/src/mcp/mcp-client.js

@@ -49,9 +49,15 @@ const CLIENT_INFO = Object.freeze({ name: 'toolfunnel', version: '3.0.0-client'
  * and an explicit `.cmd` throws EINVAL (the CVE-2024-27980 mitigation). Most MCP servers launch via
  * exactly such shims (`npx`, `npm`, `uvx`, `pnpm`, …), so on Windows route anything that isn't already
  * a concrete `.exe`/`.com` through cmd.exe, which resolves PATHEXT shims. The args stay DISCRETE
- * (Node escapes each one under shell:false) so nothing is shell-concatenated — no injection surface on
- * this privileged path, and none of the `shell:true` DEP0190 hazard. close()'s taskkill /T still reaps
- * the whole cmd → shim → server process tree.
+ * (never shell-concatenated into one string) and there is none of the `shell:true` DEP0190 hazard.
+ * HONEST CAVEAT: because the spawned binary here is cmd.exe itself (an .exe), Node applies its
+ * STANDARD argv quoting — not cmd.exe ^-metacharacter escaping (that special-casing only fires when
+ * the command IS the .cmd/.bat). An arg containing cmd metacharacters (& | > < ") outside quotes can
+ * therefore still be reinterpreted by cmd.exe. These args come from the operator's own expose.json,
+ * so this is a correctness caveat on operator-authored config, not a privilege boundary; the fully
+ * escaped alternative (resolve the real interpreter à la auth/install.js's `node + npm-cli.js` and
+ * spawn it directly) is the upgrade path if it ever bites. close()'s taskkill /T still reaps the
+ * whole cmd → shim → server process tree.
  *
  * @param {string} command  the configured upstream command (e.g. "npx", "node", an abs path)
  * @param {string[]} args   the configured argv