toolfunnel 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -6
- package/bin/toolfunnel.js +39 -4
- package/docs/KNOWN_BUGS.md +225 -0
- package/docs/packaging.md +115 -0
- package/docs/releasing.md +31 -0
- package/docs/setup.md +1 -1
- package/package.json +3 -1
- package/src/auth/config.js +4 -2
- package/src/auth/resource-server.js +28 -3
- package/src/core/config-home.js +134 -0
- package/src/core/logger.js +4 -2
- package/src/core/requires.js +138 -0
- package/src/core/server-config.js +77 -0
- package/src/extend/create-tool.md +24 -4
- package/src/extend/package.md +33 -0
- package/src/mcp/aggregator.js +12 -7
- package/src/mcp/http-transport.js +40 -21
- package/src/mcp/mcp-client.js +9 -3
- package/src/mcp/server.js +84 -2
- package/src/packages/packager.js +5 -3
- package/src/tools/registry.js +43 -0
- package/src/ui/server.js +49 -3
- package/tools/scripts/tf-env.js +43 -0
- package/tools/scripts/tf-hook-add.js +4 -5
- package/tools/scripts/tf-hook-set.js +4 -5
- package/tools/scripts/tf-list.js +9 -12
- package/tools/scripts/tf-log.js +4 -1
- package/tools/scripts/tf-mcp-add.js +4 -6
- package/tools/scripts/tf-mcp-set.js +4 -6
- package/tools/scripts/tf-pack.js +237 -0
- package/tools/scripts/tf-tool-add.js +16 -16
- package/tools/scripts/tf-tool-set.js +5 -11
- package/tools/tools.register.json +46 -1
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* config-home.js — WHERE the gateway's mutable config lives (board item 9, the packaging enabler).
|
|
5
|
+
*
|
|
6
|
+
* The problem this solves: an npm/npx-installed gateway keeps its config INSIDE the package
|
|
7
|
+
* directory, so `npm update toolfunnel` replaces the package tree and EATS the user's tools,
|
|
8
|
+
* hooks, curation, and identity. Config must be able to live in a home of its own:
|
|
9
|
+
*
|
|
10
|
+
* resolution precedence: --config-dir flag > TOOLFUNNEL_HOME env > the package root
|
|
11
|
+
*
|
|
12
|
+
* The default is UNCHANGED: a git clone keeps everything in the repo root exactly as before
|
|
13
|
+
* (home === package root, no seeding, byte-identical behaviour). Pointing TOOLFUNNEL_HOME at an
|
|
14
|
+
* empty directory makes the gateway SEED it on first use — the shipped register + its scripts, an
|
|
15
|
+
* empty expose.json, the default hooks manifest — and every mutable path (tools/ mcp/ hooks/
|
|
16
|
+
* auth/ logs/ toolfunnel.json packages/) anchors there from then on. Seeding never overwrites an
|
|
17
|
+
* existing file, so a home survives package updates by construction.
|
|
18
|
+
*
|
|
19
|
+
* Two env vars are the process-tree contract (children inherit them via defaultRunScript's
|
|
20
|
+
* spawn env):
|
|
21
|
+
* TOOLFUNNEL_HOME — the resolved ABSOLUTE config home (normalised at init so a relative value
|
|
22
|
+
* can't skew between processes with different cwds).
|
|
23
|
+
* TOOLFUNNEL_PKG — the package root of the gateway that spawned the tool, so a management
|
|
24
|
+
* script SEEDED INTO an external home can still require the engine code
|
|
25
|
+
* (src/…) it was copied away from. A script always prefers its own
|
|
26
|
+
* __dirname-derived root when that root contains the engine (the git-clone
|
|
27
|
+
* layout), so a stale inherited value can never shadow a real local install.
|
|
28
|
+
*
|
|
29
|
+
* Zero new dependencies. Node built-ins only.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
const fs = require('node:fs');
|
|
33
|
+
const path = require('node:path');
|
|
34
|
+
|
|
35
|
+
/** The package root: this file lives at <pkg>/src/core/config-home.js. */
|
|
36
|
+
const PKG_ROOT = path.resolve(__dirname, '..', '..');
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Resolve the config home. Pure — no filesystem writes, no env mutation.
|
|
40
|
+
* @param {{dir?: string}} [opts] dir = an explicit --config-dir value (wins over the env)
|
|
41
|
+
* @returns {string} absolute config-home path
|
|
42
|
+
*/
|
|
43
|
+
function resolveConfigHome(opts) {
|
|
44
|
+
const o = opts || {};
|
|
45
|
+
const fromFlag = typeof o.dir === 'string' && o.dir.trim().length > 0 ? o.dir.trim() : null;
|
|
46
|
+
const fromEnv = typeof process.env.TOOLFUNNEL_HOME === 'string' && process.env.TOOLFUNNEL_HOME.trim().length > 0
|
|
47
|
+
? process.env.TOOLFUNNEL_HOME.trim()
|
|
48
|
+
: null;
|
|
49
|
+
const chosen = fromFlag || fromEnv;
|
|
50
|
+
return chosen ? path.resolve(chosen) : PKG_ROOT;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Recursively copy a directory, skipping any destination file that already exists. */
|
|
54
|
+
function copyTreeIfAbsent(srcDir, destDir, seeded) {
|
|
55
|
+
let entries;
|
|
56
|
+
try {
|
|
57
|
+
entries = fs.readdirSync(srcDir, { withFileTypes: true });
|
|
58
|
+
} catch (_e) {
|
|
59
|
+
return; // a missing source dir seeds nothing (defensive)
|
|
60
|
+
}
|
|
61
|
+
fs.mkdirSync(destDir, { recursive: true });
|
|
62
|
+
for (const e of entries) {
|
|
63
|
+
const from = path.join(srcDir, e.name);
|
|
64
|
+
const to = path.join(destDir, e.name);
|
|
65
|
+
if (e.isDirectory()) {
|
|
66
|
+
copyTreeIfAbsent(from, to, seeded);
|
|
67
|
+
} else if (e.isFile() && !fs.existsSync(to)) {
|
|
68
|
+
fs.copyFileSync(from, to);
|
|
69
|
+
seeded.push(to);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Make `home` a working config home, seeding whatever is ABSENT from the shipped defaults.
|
|
76
|
+
* Idempotent and never destructive: an existing file is NEVER overwritten (a home must survive
|
|
77
|
+
* package updates — that is the whole point). home === package root → a no-op (the git-clone
|
|
78
|
+
* layout is already complete). Throws only if the home cannot be created at all.
|
|
79
|
+
*
|
|
80
|
+
* Deliberately NOT seeded: tools.state.json (absent = default visibility), auth/ (absent = auth
|
|
81
|
+
* off), logs/ (absent = logging off), toolfunnel.json (absent = the default identity — an
|
|
82
|
+
* npm-wrap ships its OWN copy inside its bundled home).
|
|
83
|
+
*
|
|
84
|
+
* @param {string} home absolute config-home path (from resolveConfigHome)
|
|
85
|
+
* @returns {{home: string, seeded: string[]}} the files seeded on THIS call (empty = was complete)
|
|
86
|
+
*/
|
|
87
|
+
function ensureConfigHome(home) {
|
|
88
|
+
const seeded = [];
|
|
89
|
+
if (path.resolve(home) === PKG_ROOT) return { home: PKG_ROOT, seeded };
|
|
90
|
+
fs.mkdirSync(home, { recursive: true });
|
|
91
|
+
|
|
92
|
+
// The register + every shipped script (management tf_* + demo tools). The management scripts
|
|
93
|
+
// are config-tree citizens (the register's invoke paths point at them), so they travel.
|
|
94
|
+
copyTreeIfAbsent(path.join(PKG_ROOT, 'tools'), path.join(home, 'tools'), seeded);
|
|
95
|
+
// The state overlay is user-state, not a shipped default — drop a copied one if the package
|
|
96
|
+
// tree happened to carry local toggles (dev clone); a FRESH home starts with default visibility.
|
|
97
|
+
const strayState = path.join(home, 'tools', 'tools.state.json');
|
|
98
|
+
if (seeded.includes(strayState)) {
|
|
99
|
+
try { fs.unlinkSync(strayState); } catch (_e) { /* best-effort */ }
|
|
100
|
+
seeded.splice(seeded.indexOf(strayState), 1);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// The upstream/curation store and the hooks tree. NOTE the npm tarball ships
|
|
104
|
+
// mcp/expose.example.json but NOT mcp/expose.json (the live store is user-state), so after the
|
|
105
|
+
// tree copy the live store is synthesised EMPTY when absent. Never seed it from the example —
|
|
106
|
+
// the example is a POPULATED demo (it attaches the mock upstream), and a fresh home must start
|
|
107
|
+
// connected to nothing, exactly like a fresh clone.
|
|
108
|
+
copyTreeIfAbsent(path.join(PKG_ROOT, 'mcp'), path.join(home, 'mcp'), seeded);
|
|
109
|
+
const exposePath = path.join(home, 'mcp', 'expose.json');
|
|
110
|
+
if (!fs.existsSync(exposePath)) {
|
|
111
|
+
fs.mkdirSync(path.dirname(exposePath), { recursive: true });
|
|
112
|
+
fs.writeFileSync(exposePath, JSON.stringify({ version: 1, upstreams: [], expose: [] }, null, 2) + '\n');
|
|
113
|
+
seeded.push(exposePath);
|
|
114
|
+
}
|
|
115
|
+
copyTreeIfAbsent(path.join(PKG_ROOT, 'hooks'), path.join(home, 'hooks'), seeded);
|
|
116
|
+
return { home, seeded };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* The one-call init used by bin/toolfunnel.js for every mode: resolve, seed, and write the
|
|
121
|
+
* process-tree contract back into the env (see the header) so every module loaded AFTER this —
|
|
122
|
+
* and every child tool spawned later — reads the same resolved home.
|
|
123
|
+
* @param {{dir?: string}} [opts]
|
|
124
|
+
* @returns {{home: string, seeded: string[]}}
|
|
125
|
+
*/
|
|
126
|
+
function initConfigHome(opts) {
|
|
127
|
+
const home = resolveConfigHome(opts);
|
|
128
|
+
const res = ensureConfigHome(home);
|
|
129
|
+
process.env.TOOLFUNNEL_HOME = res.home;
|
|
130
|
+
process.env.TOOLFUNNEL_PKG = PKG_ROOT;
|
|
131
|
+
return res;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
module.exports = { PKG_ROOT, resolveConfigHome, ensureConfigHome, initConfigHome };
|
package/src/core/logger.js
CHANGED
|
@@ -27,8 +27,10 @@
|
|
|
27
27
|
const fs = require('node:fs');
|
|
28
28
|
const path = require('node:path');
|
|
29
29
|
|
|
30
|
-
/**
|
|
31
|
-
|
|
30
|
+
/** The CONFIG HOME (TOOLFUNNEL_HOME / --config-dir; defaults to the package root — see
|
|
31
|
+
* config-home.js). Logs + their toggle are user-state, so they live with the home. */
|
|
32
|
+
const { resolveConfigHome } = require('./config-home');
|
|
33
|
+
const ROOT = resolveConfigHome();
|
|
32
34
|
|
|
33
35
|
/** The toggle/config file. NOT created until setConfig() writes it. */
|
|
34
36
|
const CONFIG_PATH = path.join(ROOT, 'logs', 'log.config.json');
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* requires.js — runtime-requirement preflight (the packaging story's honesty layer).
|
|
5
|
+
*
|
|
6
|
+
* A shipped pack's tools may need runtimes the gateway itself does not (python for a PDF tool,
|
|
7
|
+
* git for a repo tool). The pack DECLARES them in its toolfunnel.json — the identity file that
|
|
8
|
+
* already travels with every config home — and the gateway probes them ONCE at startup:
|
|
9
|
+
*
|
|
10
|
+
* "requires": [
|
|
11
|
+
* { "command": "python", "min": "3.10", "why": "the pdf-extract tools" },
|
|
12
|
+
* { "command": "git" }
|
|
13
|
+
* ]
|
|
14
|
+
*
|
|
15
|
+
* Probes are ADVISORY, never fatal: a missing runtime breaks only the tools that need it, so the
|
|
16
|
+
* gateway starts and the operator gets a clear stderr line naming what is missing, the version
|
|
17
|
+
* found vs needed, and the declared "why". Fields per entry (only `command` is required):
|
|
18
|
+
* command the executable to probe (e.g. "python", "node", "git")
|
|
19
|
+
* versionArg the flag that prints a version (default "--version")
|
|
20
|
+
* min minimum version, dotted numerics (e.g. "3.10" / "18" / "3.10.2")
|
|
21
|
+
* why shown in the warning so the operator knows which tools care
|
|
22
|
+
*
|
|
23
|
+
* Zero new dependencies: child_process probes + a hand-rolled dotted-numeric compare. NO semver
|
|
24
|
+
* library — by the packaging hard constraint, and dotted numerics cover real runtime versions.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
const fs = require('node:fs');
|
|
28
|
+
const path = require('node:path');
|
|
29
|
+
const { spawnSync } = require('node:child_process');
|
|
30
|
+
|
|
31
|
+
/** Parse the FIRST dotted-numeric version out of a probe's output ("Python 3.12.1" → [3,12,1]).
|
|
32
|
+
* Returns null when no version-shaped token is present. */
|
|
33
|
+
function parseVersion(text) {
|
|
34
|
+
const m = /(\d+)\.(\d+)(?:\.(\d+))?/.exec(String(text || ''));
|
|
35
|
+
if (!m) {
|
|
36
|
+
const single = /\b(\d+)\b/.exec(String(text || ''));
|
|
37
|
+
return single ? [Number(single[1]), 0, 0] : null;
|
|
38
|
+
}
|
|
39
|
+
return [Number(m[1]), Number(m[2]), Number(m[3] || 0)];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Hand-rolled dotted-numeric compare: -1 | 0 | 1 for a<b | a==b | a>b. */
|
|
43
|
+
function compareVersions(a, b) {
|
|
44
|
+
for (let i = 0; i < 3; i += 1) {
|
|
45
|
+
const x = a[i] || 0;
|
|
46
|
+
const y = b[i] || 0;
|
|
47
|
+
if (x < y) return -1;
|
|
48
|
+
if (x > y) return 1;
|
|
49
|
+
}
|
|
50
|
+
return 0;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Probe one command for its version string. On Windows a PATHEXT shim (a .cmd like npx) cannot be
|
|
55
|
+
* spawned shell:false (ENOENT/EINVAL — the CVE-2024-27980 mitigation), so a failed direct probe is
|
|
56
|
+
* retried through cmd.exe, mirroring mcp-client's winLaunch.
|
|
57
|
+
* @param {string} command
|
|
58
|
+
* @param {string} versionArg
|
|
59
|
+
* @returns {{found: boolean, version: (number[]|null), raw: string}}
|
|
60
|
+
*/
|
|
61
|
+
function probe(command, versionArg) {
|
|
62
|
+
const runOnce = (cmd, args) => spawnSync(cmd, args, { encoding: 'utf8', windowsHide: true, timeout: 5000 });
|
|
63
|
+
let res = runOnce(command, [versionArg]);
|
|
64
|
+
if (res.error && process.platform === 'win32') {
|
|
65
|
+
const comspec = process.env.ComSpec || 'cmd.exe';
|
|
66
|
+
res = runOnce(comspec, ['/c', command, versionArg]);
|
|
67
|
+
}
|
|
68
|
+
if (res.error || res.status == null) return { found: false, version: null, raw: '' };
|
|
69
|
+
const raw = `${res.stdout || ''} ${res.stderr || ''}`.trim();
|
|
70
|
+
const version = parseVersion(raw);
|
|
71
|
+
// "Spawned" is not "exists": the cmd.exe fallback launches fine for a NONEXISTENT command and
|
|
72
|
+
// exits non-zero ("is not recognized…"). Existence = a clean exit OR a version-shaped answer.
|
|
73
|
+
const found = res.status === 0 || version !== null;
|
|
74
|
+
return { found, version: found ? version : null, raw };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Check the `requires` declared in `<home>/toolfunnel.json`. Returns the problem list — the
|
|
79
|
+
* caller decides how to surface it (the gateway logs each to stderr and starts anyway). NEVER
|
|
80
|
+
* throws; a malformed requires entry is itself reported as a problem rather than crashing.
|
|
81
|
+
* @param {string} home the resolved config home
|
|
82
|
+
* @returns {Array<{command:string, problem:string}>}
|
|
83
|
+
*/
|
|
84
|
+
function checkRequires(home) {
|
|
85
|
+
let cfg;
|
|
86
|
+
try {
|
|
87
|
+
cfg = JSON.parse(fs.readFileSync(path.join(home, 'toolfunnel.json'), 'utf8'));
|
|
88
|
+
} catch (_e) {
|
|
89
|
+
return []; // absent/bad identity file → nothing declared
|
|
90
|
+
}
|
|
91
|
+
const list = cfg && Array.isArray(cfg.requires) ? cfg.requires : [];
|
|
92
|
+
const problems = [];
|
|
93
|
+
for (const entry of list) {
|
|
94
|
+
if (!entry || typeof entry !== 'object' || typeof entry.command !== 'string' || entry.command.trim() === '') {
|
|
95
|
+
problems.push({ command: String(entry && entry.command), problem: 'malformed requires entry (needs a "command" string)' });
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
const command = entry.command.trim();
|
|
99
|
+
// A requires command is a bare PROGRAM NAME, never a shell string. This is load-bearing on
|
|
100
|
+
// Windows: the cmd.exe shim fallback below hands the token to cmd, where an unquoted
|
|
101
|
+
// metacharacter ("x&evil") would CHAIN commands — a probe must never be able to run anything
|
|
102
|
+
// beyond `<program> <versionArg>`. Reject anything shell-shaped up front (also catches paths:
|
|
103
|
+
// declare tools by PATH name, which is the only portable claim a pack can make anyway).
|
|
104
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._+-]*$/.test(command)) {
|
|
105
|
+
problems.push({ command, problem: `requires.command "${command}" is not a bare program name (no paths, spaces, or shell characters)` });
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
const why = typeof entry.why === 'string' && entry.why ? ` (needed for: ${entry.why})` : '';
|
|
109
|
+
const versionArg = typeof entry.versionArg === 'string' && /^--?[A-Za-z][A-Za-z-]*$/.test(entry.versionArg)
|
|
110
|
+
? entry.versionArg
|
|
111
|
+
: '--version'; // same rule: a version FLAG, never a shell fragment
|
|
112
|
+
const p = probe(command, versionArg);
|
|
113
|
+
if (!p.found) {
|
|
114
|
+
problems.push({ command, problem: `"${command}" was not found on PATH${why}` });
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
if (typeof entry.min === 'string' && entry.min.trim() !== '') {
|
|
118
|
+
const min = parseVersion(entry.min);
|
|
119
|
+
if (!min) {
|
|
120
|
+
problems.push({ command, problem: `requires.min "${entry.min}" is not a dotted numeric version` });
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
if (!p.version) {
|
|
124
|
+
problems.push({ command, problem: `"${command}" answered but printed no recognisable version (wanted >= ${entry.min})${why}` });
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (compareVersions(p.version, min) < 0) {
|
|
128
|
+
problems.push({
|
|
129
|
+
command,
|
|
130
|
+
problem: `"${command}" is ${p.version.join('.')} but this setup declares >= ${entry.min}${why}`,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return problems;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
module.exports = { checkRequires, parseVersion, compareVersions, probe };
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* server-config.js — OPTIONAL gateway-level configuration: identity + default ports.
|
|
5
|
+
*
|
|
6
|
+
* Reads `toolfunnel.json` at the gateway root (absent by default). This is what lets a
|
|
7
|
+
* third party ship an MCP built on toolfunnel that introduces itself as ITSELF: the
|
|
8
|
+
* `initialize` handshake and `/health` report THEIR serverName/serverVersion, and their
|
|
9
|
+
* pack can carry its own default ports so `npx their-mcp` never collides with another
|
|
10
|
+
* gateway on the user's machine. CLI flags still win: precedence is flag > config > default.
|
|
11
|
+
*
|
|
12
|
+
* Shape (every field optional — an empty/absent file IS the default identity):
|
|
13
|
+
* {
|
|
14
|
+
* "serverName": "my-mcp", // initialize serverInfo.name (default "toolfunnel")
|
|
15
|
+
* "serverVersion": "1.0.0", // initialize serverInfo.version (default: package.json)
|
|
16
|
+
* "httpPort": 9998, // --http default port (default 9998)
|
|
17
|
+
* "uiPort": 9777 // --ui default port (default 9777)
|
|
18
|
+
* }
|
|
19
|
+
*
|
|
20
|
+
* TOLERANT by design, field by field: a missing file, bad JSON, or an invalid FIELD falls
|
|
21
|
+
* back to that field's default — a broken identity file must never stop the gateway from
|
|
22
|
+
* starting as plain "toolfunnel" (the same fail-open-to-default posture as log.config.json;
|
|
23
|
+
* contrast auth config, which fails CLOSED because it guards a boundary). No new deps.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
const fs = require('node:fs');
|
|
27
|
+
const path = require('node:path');
|
|
28
|
+
|
|
29
|
+
/** The compiled-in defaults. version comes from package.json (single source of truth). */
|
|
30
|
+
function defaults() {
|
|
31
|
+
let version = '0.0.0';
|
|
32
|
+
try {
|
|
33
|
+
version = require('../../package.json').version || version;
|
|
34
|
+
} catch (_e) {
|
|
35
|
+
/* unreadable package.json → a placeholder beats a crash */
|
|
36
|
+
}
|
|
37
|
+
return { serverName: 'toolfunnel', serverVersion: version, httpPort: 9998, uiPort: 9777 };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** A valid TCP port for a bind default: an integer 1..65535 (0 = OS-assigned stays flag-only —
|
|
41
|
+
* an ephemeral DEFAULT would make the gateway un-findable across restarts). */
|
|
42
|
+
function isPort(v) {
|
|
43
|
+
return Number.isInteger(v) && v >= 1 && v <= 65535;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Load the gateway config, merged over the defaults.
|
|
48
|
+
* @param {string} rootDir the gateway root (the directory that holds toolfunnel.json)
|
|
49
|
+
* @returns {{serverName:string, serverVersion:string, httpPort:number, uiPort:number}}
|
|
50
|
+
*/
|
|
51
|
+
function loadServerConfig(rootDir) {
|
|
52
|
+
const out = defaults();
|
|
53
|
+
let raw;
|
|
54
|
+
try {
|
|
55
|
+
raw = fs.readFileSync(path.join(rootDir, 'toolfunnel.json'), 'utf8');
|
|
56
|
+
} catch (_e) {
|
|
57
|
+
return out; // absent — the normal case
|
|
58
|
+
}
|
|
59
|
+
let cfg;
|
|
60
|
+
try {
|
|
61
|
+
cfg = JSON.parse(raw);
|
|
62
|
+
} catch (_e) {
|
|
63
|
+
return out; // bad JSON — start as the default identity rather than not at all
|
|
64
|
+
}
|
|
65
|
+
if (!cfg || typeof cfg !== 'object' || Array.isArray(cfg)) return out;
|
|
66
|
+
if (typeof cfg.serverName === 'string' && cfg.serverName.trim().length > 0) {
|
|
67
|
+
out.serverName = cfg.serverName.trim();
|
|
68
|
+
}
|
|
69
|
+
if (typeof cfg.serverVersion === 'string' && cfg.serverVersion.trim().length > 0) {
|
|
70
|
+
out.serverVersion = cfg.serverVersion.trim();
|
|
71
|
+
}
|
|
72
|
+
if (isPort(cfg.httpPort)) out.httpPort = cfg.httpPort;
|
|
73
|
+
if (isPort(cfg.uiPort)) out.uiPort = cfg.uiPort;
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
module.exports = { loadServerConfig };
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
This is the instruction served by `toolfunnel_howto({ topic: "create-tool" })`. It tells you — the
|
|
4
4
|
model — how to add a new tool to the register so it becomes callable through `toolfunnel_run_tool`
|
|
5
|
-
(and listable via `toolfunnel_list_tools` / `toolfunnel_tool_instructions`). No restart is needed:
|
|
6
|
-
|
|
5
|
+
(and listable via `toolfunnel_list_tools` / `toolfunnel_tool_instructions`). No restart is needed: a
|
|
6
|
+
running gateway watches `tools.register.json` and hot-reloads its register on change (debounced
|
|
7
|
+
~150 ms), the same way `mcp/expose.json` and the hook files reload.
|
|
7
8
|
|
|
8
9
|
A tool is exactly two things:
|
|
9
10
|
1. **A register entry** in `tools/tools.register.json` (structured metadata + how to invoke it).
|
|
@@ -23,6 +24,12 @@ A tool is exactly two things:
|
|
|
23
24
|
"summary": "Uppercases the supplied text and echoes it back.",
|
|
24
25
|
"category": "demo",
|
|
25
26
|
"instructions": "Pass { text: string }. Returns the text uppercased. Use this only as a sanity check of the run path.",
|
|
27
|
+
"inputSchema": {
|
|
28
|
+
"type": "object",
|
|
29
|
+
"properties": { "text": { "type": "string", "description": "The text to uppercase." } },
|
|
30
|
+
"required": ["text"],
|
|
31
|
+
"additionalProperties": false
|
|
32
|
+
},
|
|
26
33
|
"invoke": { "type": "script", "path": "scripts/echo-upper.sh" }
|
|
27
34
|
}
|
|
28
35
|
```
|
|
@@ -36,6 +43,7 @@ Field-by-field — these names are fixed (registry.js reads exactly these):
|
|
|
36
43
|
| `summary` | string | One line. This is ALL that `toolfunnel_list_tools` returns for the tool — keep it tight and discriminating so the model can pick the right tool from the brief alone. |
|
|
37
44
|
| `category` | string | Grouping label (e.g. `demo`, `memory`, `messaging`, `media`). Used by `toolfunnel_list_tools({ category })` to filter. |
|
|
38
45
|
| `instructions` | string | Full usage doc returned by `toolfunnel_tool_instructions({ name })`. Describe the args object, what comes back, and any safety caveats. This is the long-tail payload — be complete here, terse in `summary`. |
|
|
46
|
+
| `inputSchema` | object (optional) | A JSON Schema for the tool's args. **This is what makes the tool a first-class MCP citizen when promoted hot:** the top-level `tools/list` advertises it VERBATIM, so a connected client knows the exact arg shape without reading `instructions`. Omit it and a hot-promoted tool falls back to a free-form `{ "type": "object" }` (callable, but the client gets no arg hints). Must be an object — the register rejects any other shape at authoring time. |
|
|
39
47
|
| `invoke` | object | How `run` executes the tool. See below. |
|
|
40
48
|
|
|
41
49
|
### The `invoke` object — two forms
|
|
@@ -128,5 +136,17 @@ matches its `tool_name` (see `toolfunnel_howto({ topic: "add-hook" })`), not omi
|
|
|
128
136
|
5. Test with `toolfunnel_run_tool({ name, args })`. Verify the gate behaves: a benign call returns
|
|
129
137
|
`{ ok:true, output }`; if a matching PreToolUse hook should deny it, confirm it does.
|
|
130
138
|
|
|
131
|
-
No reconnect is needed — the register is dynamic
|
|
132
|
-
|
|
139
|
+
No reconnect is needed — the register is dynamic: the running gateway hot-reloads
|
|
140
|
+
`tools.register.json` on change and emits `notifications/tools/list_changed`, exactly like an
|
|
141
|
+
`expose.json` or hook edit.
|
|
142
|
+
|
|
143
|
+
---
|
|
144
|
+
|
|
145
|
+
## 5. Promoting your tool to the top-level surface (the "instant MCP" recipe)
|
|
146
|
+
|
|
147
|
+
By default your tool lives behind the lean meta-tools (discovered via `toolfunnel_list_tools`).
|
|
148
|
+
Set `hot: true` for its id in `tools/tools.state.json` (or via the UI / `tf_tool_set`) and it is
|
|
149
|
+
advertised DIRECTLY in the MCP `tools/list` — under its own `id`, with your `summary` as the
|
|
150
|
+
description and your `inputSchema` as the advertised schema. Author a real `inputSchema` before
|
|
151
|
+
promoting: that schema is the difference between a client calling your tool correctly first time
|
|
152
|
+
and it guessing at a free-form object.
|
package/src/extend/package.md
CHANGED
|
@@ -5,6 +5,39 @@ tools, hooks, and MCP configs into a **shareable package** and how import/export
|
|
|
5
5
|
**byproduct** of the gateway design: because every unit is already structured and
|
|
6
6
|
self-documented, a package is just a folder that lists its units and carries their files.
|
|
7
7
|
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## 0. The WHOLE-SETUP path — `tf_pack` (start here)
|
|
11
|
+
|
|
12
|
+
Since 0.4.0 the fastest way to ship is not per-unit packaging but the **whole setup**: your entire
|
|
13
|
+
config home (tools + scripts, hot/hidden curation, upstream references, policy hooks, identity) is
|
|
14
|
+
already the product. One call packages it into a deployment-ready artifact under `<home>/dist/`:
|
|
15
|
+
|
|
16
|
+
- `tf_pack { format: "home" }` → a **portable config home**. Zip it, git-init it, or run it
|
|
17
|
+
directly: `toolfunnel --config-dir <dir>`. The receiving gateway seeds nothing — the pack IS the
|
|
18
|
+
config, including the hooks: **the gate travels**, enforcing your policy on the recipient's
|
|
19
|
+
machine regardless of which MCP client they use.
|
|
20
|
+
- `tf_pack { format: "npm", name: "my-mcp" }` → a **publishable npm package**: a generated
|
|
21
|
+
`package.json` that depends on `toolfunnel` (caret range — DEPEND, never copy the engine), a
|
|
22
|
+
2-line bin launcher pointing `--config-dir` at the bundled home, and a README stub.
|
|
23
|
+
`cd dist/my-mcp && npm publish` → your users run `npx my-mcp` and get YOUR server, under YOUR
|
|
24
|
+
name (the bundled `toolfunnel.json` identity), with YOUR gate.
|
|
25
|
+
|
|
26
|
+
Declare the runtimes your tools need in the home's `toolfunnel.json` — they travel with the pack
|
|
27
|
+
and the receiving gateway warns at startup about anything missing:
|
|
28
|
+
|
|
29
|
+
```json
|
|
30
|
+
"requires": [ { "command": "python", "min": "3.10", "why": "the pdf tools" } ]
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Audit honesty (both directions): packs spawn commands. Tell your users to read your pack's
|
|
34
|
+
`expose.json`, `tools.register.json`, AND `toolfunnel.json` (its `requires` probes execute at
|
|
35
|
+
gateway startup — bare program names only, the loader rejects anything shell-shaped) — and read
|
|
36
|
+
all three yourself in anything you install.
|
|
37
|
+
|
|
38
|
+
The per-UNIT packaging below remains the right shape for sharing a single tool/hook/MCP selection
|
|
39
|
+
rather than a whole gateway.
|
|
40
|
+
|
|
8
41
|
> Phase note: packages are Phase 2 (`packages/` directory). The structure below is what the importer
|
|
9
42
|
> and exporter target; defining it now means the units authored today are already package-ready.
|
|
10
43
|
|
package/src/mcp/aggregator.js
CHANGED
|
@@ -49,10 +49,12 @@ const path = require('node:path');
|
|
|
49
49
|
const { McpClient } = require('./mcp-client');
|
|
50
50
|
const logger = require('../core/logger');
|
|
51
51
|
|
|
52
|
-
// The sandbox root
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
|
|
52
|
+
// The sandbox root = the CONFIG HOME (TOOLFUNNEL_HOME / --config-dir; defaults to the package
|
|
53
|
+
// root — see src/core/config-home.js). Used as the isolation boundary the default clientFactory
|
|
54
|
+
// enforces: a pack's path-shaped upstream args live under the home, so the home is the boundary.
|
|
55
|
+
// A caller may override via opts.v3Root for tests.
|
|
56
|
+
const { resolveConfigHome } = require('../core/config-home');
|
|
57
|
+
const DEFAULT_ROOT = resolveConfigHome();
|
|
56
58
|
|
|
57
59
|
/** True for a non-empty string. */
|
|
58
60
|
function isNonEmptyString(v) {
|
|
@@ -522,9 +524,6 @@ class Aggregator {
|
|
|
522
524
|
const upstream = this._store.getUpstream(upstreamId);
|
|
523
525
|
if (!upstream || upstream.enabled !== true) return; // detached/disabled — stop trying
|
|
524
526
|
const MAX_FAST_ATTEMPTS = 6; // exponential phase; beyond it, a slow keepalive at the 30s cap
|
|
525
|
-
if (attempt === MAX_FAST_ATTEMPTS) {
|
|
526
|
-
logger.log({ type: 'mcp', event: 'reconnect_slow', upstream: upstreamId, attempts: attempt });
|
|
527
|
-
}
|
|
528
527
|
const delay = Math.min(1000 * Math.pow(2, attempt), 30000);
|
|
529
528
|
const timer = setTimeout(async () => {
|
|
530
529
|
this._reconnectTimers.delete(upstreamId);
|
|
@@ -537,6 +536,12 @@ class Aggregator {
|
|
|
537
536
|
this._signalToolsChanged(upstreamId);
|
|
538
537
|
} catch (err) {
|
|
539
538
|
logger.log({ type: 'mcp', event: 'reconnect_failed', upstream: upstreamId, attempt, error: errMsg(err) });
|
|
539
|
+
// Log the fast→slow transition exactly ONCE — when the counter FIRST reaches the cap. The
|
|
540
|
+
// old guard sat on the re-entry side, where the capped counter re-satisfied it every 30s
|
|
541
|
+
// forever against a permanently-dead upstream (log spam, not information).
|
|
542
|
+
if (attempt + 1 === MAX_FAST_ATTEMPTS) {
|
|
543
|
+
logger.log({ type: 'mcp', event: 'reconnect_slow', upstream: upstreamId, attempts: MAX_FAST_ATTEMPTS });
|
|
544
|
+
}
|
|
540
545
|
// Keep escalating to the cap, then keep retrying slowly (cap the counter so the delay stays
|
|
541
546
|
// at 30s) — never permanently give up while the upstream stays enabled.
|
|
542
547
|
this._scheduleReconnect(upstreamId, Math.min(attempt + 1, MAX_FAST_ATTEMPTS));
|
|
@@ -145,7 +145,14 @@ function isLoopbackHost(hostHeader) {
|
|
|
145
145
|
return host === '127.0.0.1' || host === 'localhost' || host === '::1' || host === '0:0:0:0:0:0:0:1';
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
-
/**
|
|
148
|
+
/**
|
|
149
|
+
* Read a request body up to MAX_BODY_BYTES. Resolves the buffered string, or rejects 'too large'.
|
|
150
|
+
* On over-cap it does NOT destroy the request — destroying here tears down the shared req/res
|
|
151
|
+
* socket before the caller's -32700 reply can flush (the client saw ECONNRESET instead of the clean
|
|
152
|
+
* parse error the declared-Content-Length path produces). Instead: stop BUFFERING (later chunks are
|
|
153
|
+
* discarded, so memory stays bounded) and reject; the caller replies via sendJsonAndClose, which
|
|
154
|
+
* flushes the error body first and destroys the request only in the res.end callback.
|
|
155
|
+
*/
|
|
149
156
|
function readBody(req) {
|
|
150
157
|
return new Promise((resolve, reject) => {
|
|
151
158
|
let size = 0;
|
|
@@ -157,15 +164,11 @@ function readBody(req) {
|
|
|
157
164
|
fn(val);
|
|
158
165
|
};
|
|
159
166
|
req.on('data', (chunk) => {
|
|
167
|
+
if (done) return; // over-cap already signalled — discard, don't buffer
|
|
160
168
|
size += chunk.length;
|
|
161
169
|
if (size > MAX_BODY_BYTES) {
|
|
162
|
-
// Stop
|
|
170
|
+
// Stop buffering; the caller flush-then-closes with the same -32700 as the declared-CL path.
|
|
163
171
|
finish(reject, new Error('request body too large'));
|
|
164
|
-
try {
|
|
165
|
-
req.destroy();
|
|
166
|
-
} catch (_e) {
|
|
167
|
-
/* ignore */
|
|
168
|
-
}
|
|
169
172
|
return;
|
|
170
173
|
}
|
|
171
174
|
chunks.push(chunk);
|
|
@@ -193,16 +196,19 @@ function sendJson(res, status, obj) {
|
|
|
193
196
|
|
|
194
197
|
/**
|
|
195
198
|
* Send a JSON response and then CLOSE the connection, draining the unread request body WITHOUT
|
|
196
|
-
* buffering it. Used by the Content-Length pre-check
|
|
197
|
-
*
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
*
|
|
199
|
+
* buffering it. Used by both over-cap paths (the Content-Length pre-check and readBody's streaming
|
|
200
|
+
* cap): the client must receive the documented -32700 cleanly while the oversized payload never
|
|
201
|
+
* touches memory. The naive close — `req.destroy()`, even deferred to the res.end flush callback —
|
|
202
|
+
* is NOT safe: destroy() sends a TCP RST whenever request bytes are still unread/in-flight, and on
|
|
203
|
+
* RST Windows discards the peer's receive queue, eating the response we just flushed (the client
|
|
204
|
+
* saw ECONNRESET instead of the -32700; proven by test 5 in http.test.js). The robust sequence:
|
|
201
205
|
* 1. announce `Connection: close` so the client/keep-alive agent expects the socket to end here,
|
|
202
|
-
* 2. write the full response body,
|
|
203
|
-
* 3.
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
+
* 2. write the full response body, then in the res.end(...) flush callback:
|
|
207
|
+
* 3. resume() the request so any still-arriving body is DISCARDED (never buffered → no DoS),
|
|
208
|
+
* 4. HALF-CLOSE the socket (socket.end()) — the FIN travels in-order AFTER the response bytes,
|
|
209
|
+
* so the client always reads the -32700 first, and
|
|
210
|
+
* 5. arm an unref'd hard-destroy timer as the backstop: a client that never stops sending and
|
|
211
|
+
* never closes gets its socket torn down after a grace window (bounded lifetime, no leak).
|
|
206
212
|
* Defensive: like sendJson, a serialise failure degrades to a 500-shaped error body; the whole
|
|
207
213
|
* function is wrapped so a write-after-headers race can never throw out of handleMcpPost.
|
|
208
214
|
* @param {import('node:http').IncomingMessage} req
|
|
@@ -233,9 +239,20 @@ function sendJsonAndClose(req, res, status, obj) {
|
|
|
233
239
|
// pipelined follow-up (whose body would collide with the one we are about to discard).
|
|
234
240
|
Connection: 'close',
|
|
235
241
|
});
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
242
|
+
res.end(body, () => {
|
|
243
|
+
// The response has flushed to the kernel. Close down WITHOUT an RST (see the doc above):
|
|
244
|
+
try {
|
|
245
|
+
req.resume(); // discard (never buffer) whatever request body is still arriving
|
|
246
|
+
if (req.socket && !req.socket.destroyed) req.socket.end(); // FIN, in-order after the body
|
|
247
|
+
} catch (_e) {
|
|
248
|
+
return destroyReq(); // a torn socket at this point → plain destroy is all that is left
|
|
249
|
+
}
|
|
250
|
+
// Backstop: bound the socket's lifetime if the client neither finishes nor closes. The
|
|
251
|
+
// timer is unref'd so it never keeps the process alive; a normal close clears it.
|
|
252
|
+
const t = setTimeout(destroyReq, 10000);
|
|
253
|
+
if (t && typeof t.unref === 'function') t.unref();
|
|
254
|
+
if (req.socket) req.socket.once('close', () => clearTimeout(t));
|
|
255
|
+
});
|
|
239
256
|
} catch (_e) {
|
|
240
257
|
// writeHead/end can throw if headers were already sent by a racing path — fall back to a
|
|
241
258
|
// straight destroy so the socket can never be left half-open.
|
|
@@ -501,9 +518,11 @@ function createHttpMcpServer(opts = {}) {
|
|
|
501
518
|
try {
|
|
502
519
|
raw = await readBody(req);
|
|
503
520
|
} catch (err) {
|
|
504
|
-
// Oversized / aborted body →
|
|
521
|
+
// Oversized / aborted body → the SAME flush-then-close -32700 as the Content-Length pre-check
|
|
522
|
+
// (readBody no longer destroys the request, so the reply can actually reach the client; a
|
|
523
|
+
// genuinely-dead socket is fine too — sendJsonAndClose's writes are defensively wrapped).
|
|
505
524
|
log('POST /mcp body read failed:', (err && err.message) || String(err));
|
|
506
|
-
return
|
|
525
|
+
return sendJsonAndClose(req, res, 200, makeError(ERR.PARSE, 'Parse error: ' + ((err && err.message) || 'bad body')));
|
|
507
526
|
}
|
|
508
527
|
|
|
509
528
|
let msg;
|
package/src/mcp/mcp-client.js
CHANGED
|
@@ -49,9 +49,15 @@ const CLIENT_INFO = Object.freeze({ name: 'toolfunnel', version: '3.0.0-client'
|
|
|
49
49
|
* and an explicit `.cmd` throws EINVAL (the CVE-2024-27980 mitigation). Most MCP servers launch via
|
|
50
50
|
* exactly such shims (`npx`, `npm`, `uvx`, `pnpm`, …), so on Windows route anything that isn't already
|
|
51
51
|
* a concrete `.exe`/`.com` through cmd.exe, which resolves PATHEXT shims. The args stay DISCRETE
|
|
52
|
-
* (
|
|
53
|
-
*
|
|
54
|
-
*
|
|
52
|
+
* (never shell-concatenated into one string) and there is none of the `shell:true` DEP0190 hazard.
|
|
53
|
+
* HONEST CAVEAT: because the spawned binary here is cmd.exe itself (an .exe), Node applies its
|
|
54
|
+
* STANDARD argv quoting — not cmd.exe ^-metacharacter escaping (that special-casing only fires when
|
|
55
|
+
* the command IS the .cmd/.bat). An arg containing cmd metacharacters (& | > < ") outside quotes can
|
|
56
|
+
* therefore still be reinterpreted by cmd.exe. These args come from the operator's own expose.json,
|
|
57
|
+
* so this is a correctness caveat on operator-authored config, not a privilege boundary; the fully
|
|
58
|
+
* escaped alternative (resolve the real interpreter à la auth/install.js's `node + npm-cli.js` and
|
|
59
|
+
* spawn it directly) is the upgrade path if it ever bites. close()'s taskkill /T still reaps the
|
|
60
|
+
* whole cmd → shim → server process tree.
|
|
55
61
|
*
|
|
56
62
|
* @param {string} command the configured upstream command (e.g. "npx", "node", an abs path)
|
|
57
63
|
* @param {string[]} args the configured argv
|