toolnexus 0.7.0 → 0.9.0

package/dist/skill.js

@@ -3,6 +3,11 @@
  * discover **\/SKILL.md, parse frontmatter, and expose ONE `skill` tool that
  * loads a skill's instructions + sampled resources on demand (progressive
  * disclosure).
+ *
+ * Beyond on-disk discovery the source also accepts skills supplied as data
+ * (`SkillDef`) — SPEC.md §3. Directory-sourced skills keep the exact `file://`
+ * base + on-disk sibling sampling (byte-identical); data-sourced skills use a
+ * logical `skill://name/` base + a supplied resource list and never touch disk.
  */
 import { readFileSync, readdirSync, realpathSync, statSync } from "node:fs";
 import path from "node:path";
@@ -23,18 +28,25 @@ export const SKILLS_PROMPT_PREAMBLE = "Skills provide specialized instructions a
  * Parse YAML frontmatter (between the leading `---` fences) with a real YAML
  * parser, so folded (`>`)/literal (`|`) block scalars, quoting, and multi-line
  * values all resolve correctly. Scalar values are coerced to strings.
+ *
+ * `malformed` is true only when fences are present but the YAML fails to parse —
+ * distinguishing a malformed header from a body with no frontmatter, so the
+ * inventory (S3) can report the right skip reason. loadSkills' behavior is
+ * unchanged: a malformed header yields no name and the skill is skipped.
  */
 function parseFrontmatter(text) {
     const match = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/.exec(text);
     if (!match)
-        return { data: {}, content: text };
+        return { data: {}, content: text, malformed: false };
     const data = {};
     let parsed;
+    let malformed = false;
     try {
         parsed = parseYaml(match[1]);
     }
     catch {
         parsed = null;
+        malformed = true;
     }
     if (parsed && typeof parsed === "object") {
         for (const [key, value] of Object.entries(parsed)) {
@@ -45,7 +57,7 @@ function parseFrontmatter(text) {
             }
         }
     }
-    return { data, content: match[2] };
+    return { data, content: match[2], malformed };
 }
 function walkSkillFiles(root) {
     const out = [];
@@ -150,45 +162,130 @@ function sampleSiblingFiles(dir, limit = 10) {
     }
     return out;
 }
-/** Discover skills under one or more roots and build the `skill` loader tool. */
-export function loadSkills(dirs) {
-    const roots = Array.isArray(dirs) ? dirs : [dirs];
-    const skills = {};
-    for (const root of roots) {
-        let isDir = false;
+/** Discover candidates under one on-disk root (order = discovery order). */
+function candidatesFromDir(root) {
+    const out = [];
+    let isDir = false;
+    try {
+        isDir = statSync(root).isDirectory();
+    }
+    catch {
+        isDir = false;
+    }
+    if (!isDir) {
+        console.warn(`[toolnexus] skills dir not found: ${root}`);
+        return out;
+    }
+    for (const file of walkSkillFiles(root)) {
+        let text;
         try {
-            isDir = statSync(root).isDirectory();
+            text = readFileSync(file, "utf8");
         }
         catch {
-            isDir = false;
+            out.push({ skip: { location: file, reason: "unreadable" } });
+            continue;
         }
-        if (!isDir) {
-            console.warn(`[toolnexus] skills dir not found: ${root}`);
+        const { data, content, malformed } = parseFrontmatter(text);
+        if (malformed) {
+            out.push({ skip: { location: file, reason: "malformed-frontmatter" } });
             continue;
         }
-        for (const file of walkSkillFiles(root)) {
-            let text;
-            try {
-                text = readFileSync(file, "utf8");
-            }
-            catch {
-                continue;
-            }
-            const { data, content } = parseFrontmatter(text);
-            if (!data.name)
-                continue;
-            if (skills[data.name]) {
-                console.warn(`[toolnexus] duplicate skill name "${data.name}" (${file}) — keeping first`);
-                continue;
-            }
-            skills[data.name] = {
-                name: data.name,
-                description: data.description,
-                location: file,
-                content,
-            };
+        if (!data.name) {
+            out.push({ skip: { location: file, reason: "missing-name" } });
+            continue;
         }
+        out.push({ info: { name: data.name, description: data.description, location: file, content, origin: "fs" } });
     }
+    return out;
+}
+/** Turn data-supplied skill defs into candidates (logical origin). */
+function candidatesFromDefs(defs) {
+    return defs.map((d) => {
+        if (!d.name)
+            return { skip: { location: d.base ?? "skill://", reason: "missing-name" } };
+        const base = d.base && d.base.length > 0 ? d.base : `skill://${d.name}/`;
+        return {
+            info: {
+                name: d.name,
+                description: d.description,
+                location: base,
+                content: d.content ?? "",
+                origin: "logical",
+                resources: d.resources ?? [],
+                base,
+            },
+        };
+    });
+}
+/** Collect ordered candidates from dirs then data defs. */
+function collectCandidates(opts) {
+    const roots = opts.dirs === undefined ? [] : Array.isArray(opts.dirs) ? opts.dirs : [opts.dirs];
+    const cands = [];
+    for (const root of roots)
+        cands.push(...candidatesFromDir(root));
+    if (opts.skills)
+        cands.push(...candidatesFromDefs(opts.skills));
+    return cands;
+}
+/** Dedupe candidates by name (first-wins); later duplicates become skips. */
+function mergeCandidates(cands) {
+    const skills = {};
+    const skipped = [];
+    for (const c of cands) {
+        if (c.skip) {
+            skipped.push(c.skip);
+            continue;
+        }
+        const info = c.info;
+        if (skills[info.name]) {
+            console.warn(`[toolnexus] duplicate skill name "${info.name}" (${info.location}) — keeping first`);
+            skipped.push({ location: info.location, reason: "duplicate-name" });
+            continue;
+        }
+        skills[info.name] = info;
+    }
+    return { skills, skipped };
+}
+/**
+ * Apply the per-agent skill allowlist (S2), semantics identical to the MCP
+ * per-server tools filter and builtins: nil/empty ⇒ all; ≥1 true ⇒ allowlist
+ * (only true-mapped names); only-false ⇒ drop-list over all-on; unknown names
+ * are ignored and warned once.
+ */
+function applySkillsFilter(skills, filter) {
+    if (!filter)
+        return skills;
+    const keys = Object.keys(filter);
+    if (keys.length === 0)
+        return skills;
+    const hasTrue = keys.some((k) => filter[k] === true);
+    for (const k of keys) {
+        if (!(k in skills))
+            console.warn(`[toolnexus] skill filter name "${k}" matched no skill`);
+    }
+    const out = {};
+    for (const [name, info] of Object.entries(skills)) {
+        if (hasTrue ? filter[name] === true : filter[name] !== false)
+            out[name] = info;
+    }
+    return out;
+}
+/**
+ * Discover + validate skills from the same sources loadSkills accepts, returning
+ * parsed skills plus typed skip reasons — no toolkit wired (SPEC.md §3, S3). The
+ * inventory is UNFILTERED (it exists to author/validate the allowlist).
+ */
+export function listSkills(input) {
+    const opts = typeof input === "string" || Array.isArray(input) ? { dirs: input } : input;
+    const merged = mergeCandidates(collectCandidates(opts));
+    return { skills: Object.values(merged.skills), skipped: merged.skipped };
+}
+/** Discover skills (dirs and/or data) and build the `skill` loader tool. */
+export function loadSkills(input) {
+    const opts = typeof input === "string" || Array.isArray(input) ? { dirs: input } : input;
+    const sampleLimit = opts.sampleLimit ?? 0;
+    const merged = mergeCandidates(collectCandidates(opts));
+    const skills = applySkillsFilter(merged.skills, opts.filter);
     const tool = {
         name: "skill",
         description: SKILL_TOOL_DESCRIPTION,
@@ -208,10 +305,30 @@ export function loadSkills(dirs) {
                     isError: true,
                 };
             }
-            const dir = path.dirname(info.location);
-            const base = pathToFileURL(dir).href;
-            const files = sampleSiblingFiles(dir);
-            const output = [
+            // effLimit: 0 ⇒ default 10 (byte-identical to today), n>0 ⇒ cap, -1 ⇒ omit.
+            const effLimit = sampleLimit === 0 ? 10 : sampleLimit;
+            let base;
+            let files;
+            let metaDir;
+            // Emit the <skill_files> block: always for fs skills (byte-identical, even
+            // when empty), only when a data skill actually carries resources (an
+            // instruction-only data skill omits it). Never when sampling is disabled.
+            let emitFiles = effLimit !== -1;
+            if (info.origin === "logical") {
+                base = info.base ?? `skill://${info.name}/`;
+                const res = info.resources ?? [];
+                if (res.length === 0)
+                    emitFiles = false;
+                files = effLimit > 0 ? res.slice(0, effLimit) : res;
+                metaDir = base;
+            }
+            else {
+                const dir = path.dirname(info.location);
+                base = pathToFileURL(dir).href;
+                files = effLimit === -1 ? [] : sampleSiblingFiles(dir, effLimit);
+                metaDir = dir;
+            }
+            const lines = [
                 `<skill_content name="${info.name}">`,
                 `# Skill: ${info.name}`,
                 "",
@@ -219,14 +336,12 @@ export function loadSkills(dirs) {
                 "",
                 `Base directory for this skill: ${base}`,
                 "Relative paths in this skill (e.g., scripts/, reference/) are relative to this base directory.",
-                "Note: file list is sampled.",
-                "",
-                "<skill_files>",
-                files.map((f) => `<file>${f}</file>`).join("\n"),
-                "</skill_files>",
-                "</skill_content>",
-            ].join("\n");
-            return { output, isError: false, metadata: { name: info.name, dir } };
+            ];
+            if (emitFiles) {
+                lines.push("Note: file list is sampled.", "", "<skill_files>", files.map((f) => `<file>${f}</file>`).join("\n"), "</skill_files>");
+            }
+            lines.push("</skill_content>");
+            return { output: lines.join("\n"), isError: false, metadata: { name: info.name, dir: metaDir } };
         },
     };
     return {

package/dist/toolkit.d.ts

@@ -3,6 +3,7 @@
  * single uniform list, with provider adapters and a single execute() router.
  */
 import { type Tool, type ToolContext, type ToolResult, type McpStatus } from "./types.js";
+import { type SkillDef } from "./skill.js";
 import { type BuiltinsConfig } from "./builtin.js";
 import { type Agent } from "./a2a.js";
 import { type A2AConfig, type OnTask, type ServeHandle } from "./serve.js";
@@ -11,6 +12,14 @@ import type { Client } from "./client.js";
 export interface ToolkitOptions {
     mcpConfig?: string | object;
     skillsDir?: string | string[];
+    /** Skills supplied as data, bypassing the filesystem (§3, S1). */
+    skills?: SkillDef[];
+    /** Lazy provider of data-supplied skills, resolved once at toolkit build (§3, S1). */
+    skillProvider?: () => SkillDef[] | Promise<SkillDef[]>;
+    /** Per-agent skill allowlist keyed on name; same semantics as the MCP tools filter (§3, S2). */
+    skillsFilter?: Record<string, boolean>;
+    /** Sibling-file sample cap: 0 ⇒ default 10, n>0 ⇒ cap, -1 ⇒ omit <skill_files> (§3, S5). */
+    skillSampleLimit?: number;
     extraTools?: Tool[];
     /** Built-in tools (§4A). On by default. false | { disabled:true } | { enabled:false } ⇒ off. */
     builtins?: BuiltinsConfig;
@@ -23,6 +32,9 @@ export interface ToolkitOptions {
      * elicitation is not advertised.
      */
     waitFor?: (request: import("./types.js").Request) => Promise<import("./types.js").Answer>;
+    /** §2 Gap 3. Bounds the MCP load: each connect/list is capped by the server timeout and the
+     * whole load aborts promptly if this signal fires. Omit ⇒ unbounded as before. */
+    signal?: AbortSignal;
 }
 export declare class Toolkit {
     private readonly mcp;

package/dist/toolkit.js

@@ -34,8 +34,30 @@ export class Toolkit {
         }
     }
     static async create(opts) {
-        const mcp = opts.mcpConfig !== undefined ? await loadMcp(opts.mcpConfig, { waitFor: opts.waitFor }) : undefined;
-        const skill = opts.skillsDir !== undefined ? loadSkills(opts.skillsDir) : undefined;
+        const mcp = opts.mcpConfig !== undefined ? await loadMcp(opts.mcpConfig, { waitFor: opts.waitFor, signal: opts.signal }) : undefined;
+        // Skills come from directories, in-memory data, and/or a lazy provider (§3,
+        // S1). The provider is resolved here (create() is async) and merged with the
+        // data list; a provider failure is isolated so other sources still load.
+        let skill;
+        if (opts.skillsDir !== undefined || opts.skills !== undefined || opts.skillProvider !== undefined) {
+            let providerDefs = [];
+            if (opts.skillProvider) {
+                try {
+                    providerDefs = await opts.skillProvider();
+                }
+                catch (e) {
+                    console.warn(`[toolnexus] skill provider failed: ${e instanceof Error ? e.message : String(e)}`);
+                    providerDefs = [];
+                }
+            }
+            const dataDefs = [...(opts.skills ?? []), ...providerDefs];
+            skill = loadSkills({
+                dirs: opts.skillsDir,
+                skills: dataDefs.length > 0 ? dataDefs : undefined,
+                filter: opts.skillsFilter,
+                sampleLimit: opts.skillSampleLimit,
+            });
+        }
         // The toggle comes from the `builtins` option, or a top-level `builtins`
         // key on a parsed config object — same precedence as MCP's isEnabled.
         let builtinsCfg = opts.builtins;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "toolnexus",
-  "version": "0.7.0",
+  "version": "0.9.0",
   "description": "Dynamic MCP servers + agent skills as uniform tools for any LLM (opencode-style).",
   "type": "module",
   "main": "./dist/index.js",

package/src/client.ts

@@ -35,6 +35,18 @@ export interface ClientOptions {
    * (open a browser, message a channel, watch a file, forward to another agent). Omit for a durable
    * host: run() does not hang — it returns { status:"pending", pending:request } to resume later. */
   waitFor?: (request: Request) => Promise<Answer>
+  /** §8 Gap 1. Extra top-level keys shallow-merged into EVERY LLM body after the client builds
+   * its own — a requestParams key WINS on collision (e.g. max_tokens overrides the anthropic
+   * default). `messages`/`tools`/`stream` are forbidden here (stripped + warned) — rewrite
+   * messages via bodyTransform. Applied on all four paths. Omit ⇒ body byte-identical to today. */
+  requestParams?: Record<string, unknown>
+  /** §8 Gap 1. Receives the fully assembled body (after requestParams merge) just before marshal
+   * and returns the body to send — the escape hatch for provider-specific rewriting (e.g. tool-role
+   * message preprocessing) without a proxy. Returning a falsy value ⇒ unchanged. Runs on every call. */
+  bodyTransform?: (body: Record<string, any>) => Record<string, any> | undefined | void
+  /** §8 Gap 2. HTTP transport for LLM requests (retries included). Omit ⇒ global fetch. Scope is the
+   * LLM path only; MCP transports use the SDK's own client. */
+  fetch?: typeof fetch
 }
 
 /**
@@ -281,6 +293,61 @@ export class Client {
     this.store = opts.store ?? new InMemoryConversationStore()
   }
 
+  /**
+   * §8 Gap 4. The client's conversation store — the exact instance passed in `opts.store`, else
+   * the default in-memory store the client created. Read (`get`) and write (`save`) it directly to
+   * share state with `ask`; no consumer needs a shadow copy.
+   */
+  conversationStore(): ConversationStore {
+    return this.store
+  }
+
+  /**
+   * §8 Gap 1. Apply the request-shaping contract to an assembled body: strip forbidden keys from
+   * requestParams, shallow-merge requestParams (caller wins), then run bodyTransform last. Called at
+   * every LLM body-build site so the ordering contract holds identically on all four paths.
+   */
+  private finalizeBody(body: Record<string, any>): Record<string, any> {
+    const rp = this.opts.requestParams
+    if (rp) {
+      for (const [k, v] of Object.entries(rp)) {
+        if (k === "messages" || k === "tools" || k === "stream") {
+          console.warn(`[toolnexus] requestParams key "${k}" is not allowed — ignored (use bodyTransform)`)
+          continue
+        }
+        body[k] = v
+      }
+    }
+    if (this.opts.bodyTransform) {
+      const out = this.opts.bodyTransform(body)
+      if (out) return out
+    }
+    return body
+  }
+
+  /** §8 Gap 5. OpenAI-style body; omits tools/tool_choice when the tool list is empty. Key order
+   * matches the pre-change body so a no-options call is byte-identical. */
+  private openaiBody(messages: any[], tools: any[], stream: boolean): Record<string, any> {
+    const b: Record<string, any> = { model: this.opts.model, messages }
+    if (tools.length) {
+      b.tools = tools
+      b.tool_choice = "auto"
+    }
+    if (stream) {
+      b.stream = true
+      b.stream_options = { include_usage: true }
+    }
+    return this.finalizeBody(b)
+  }
+
+  /** §8 Gap 5. Anthropic-style body; omits tools when the tool list is empty. */
+  private anthropicBody(system: string, messages: any[], tools: any[], stream: boolean): Record<string, any> {
+    const b: Record<string, any> = { model: this.opts.model, max_tokens: 4096, system, messages }
+    if (tools.length) b.tools = tools
+    if (stream) b.stream = true
+    return this.finalizeBody(b)
+  }
+
   /** Prometheus text exposition of cumulative metrics (§8). Always valid, empty-but-valid before activity. */
   metrics(): string {
     return this.registry.render()
@@ -379,7 +446,7 @@ export class Client {
     let lastErr: unknown
     for (let attempt = 0; attempt <= retries; attempt++) {
       try {
-        const res = await fetch(url, { ...init, signal })
+        const res = await (this.opts.fetch ?? fetch)(url, { ...init, signal })
         if (res.ok || !RETRYABLE.has(res.status) || attempt === retries) return res
         const ra = Number(res.headers.get("retry-after"))
         await delay(ra ? ra * 1000 : base * 2 ** attempt + Math.random() * 100, signal)
@@ -470,7 +537,7 @@ export class Client {
         const data: any = await this.llmCallJson(`${this.opts.baseUrl.replace(/\/$/, "")}/chat/completions`, {
           method: "POST",
           headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json", ...this.opts.headers },
-          body: JSON.stringify({ model: this.opts.model, messages, tools, tool_choice: "auto" }),
+          body: JSON.stringify(this.openaiBody(messages, tools, false)),
         }, signal, "openai")
         addUsage(usage, data.usage, "openai")
         if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model: this.opts.model, turn })
@@ -570,7 +637,7 @@ export class Client {
             "Content-Type": "application/json",
             ...this.opts.headers,
           },
-          body: JSON.stringify({ model: this.opts.model, max_tokens: 4096, system, messages, tools }),
+          body: JSON.stringify(this.anthropicBody(system, messages, tools, false)),
         }, signal, "anthropic")
         addUsage(usage, data.usage, "anthropic")
         if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model: this.opts.model, turn })
@@ -646,7 +713,7 @@ export class Client {
           const res = await this.llmFetch(`${this.opts.baseUrl.replace(/\/$/, "")}/chat/completions`, {
             method: "POST",
             headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json", ...this.opts.headers },
-            body: JSON.stringify({ model: this.opts.model, messages, tools, tool_choice: "auto", stream: true, stream_options: { include_usage: true } }),
+            body: JSON.stringify(this.openaiBody(messages, tools, true)),
           }, signal)
           if (!res.ok || !res.body) throw new Error(`LLM ${res.status}: ${await res.text()}`)
           for await (const line of sseLines(res.body)) {
@@ -740,7 +807,7 @@ export class Client {
           const res = await this.llmFetch(endpoint, {
             method: "POST",
             headers: { "x-api-key": key, "anthropic-version": "2023-06-01", "Content-Type": "application/json", ...this.opts.headers },
-            body: JSON.stringify({ model: this.opts.model, max_tokens: 4096, system, messages, tools, stream: true }),
+            body: JSON.stringify(this.anthropicBody(system, messages, tools, true)),
           }, signal)
           if (!res.ok || !res.body) throw new Error(`LLM ${res.status}: ${await res.text()}`)
           for await (const line of sseLines(res.body)) {