toolgz 0.1.0

package/dist/render/index.js

@@ -0,0 +1,143 @@
+/** JSON Schema keys that carry no information the sampler can act on. */
+const DROP_KEYS = new Set([
+    "description",
+    "$schema",
+    "title",
+    "examples",
+    "default",
+    "additionalProperties",
+    "$id",
+    "$comment",
+]);
+/**
+ * Strip prose and boilerplate from a JSON Schema while preserving everything
+ * that constrains decoding: types, enums, required, item types, nesting.
+ *
+ * This is the single highest-leverage transform in the library — per-property
+ * `description` strings are the bulk of a real MCP tool definition.
+ */
+export function flattenSchema(schema) {
+    if (!schema || typeof schema !== "object")
+        return { type: "object" };
+    const out = {};
+    for (const [k, v] of Object.entries(schema)) {
+        if (DROP_KEYS.has(k))
+            continue;
+        if (k === "properties" && v && typeof v === "object") {
+            out.properties = Object.fromEntries(Object.entries(v).map(([pk, pv]) => [
+                pk,
+                flattenProperty(pv),
+            ]));
+        }
+        else if (k === "items") {
+            out.items = flattenProperty(v);
+        }
+        else {
+            out[k] = v;
+        }
+    }
+    if (!out.type)
+        out.type = "object";
+    return out;
+}
+function flattenProperty(p) {
+    if (!p || typeof p !== "object")
+        return p;
+    if (Array.isArray(p))
+        return p.map(flattenProperty);
+    const out = {};
+    for (const [k, v] of Object.entries(p)) {
+        if (DROP_KEYS.has(k))
+            continue;
+        if (k === "properties" && v && typeof v === "object") {
+            out.properties = Object.fromEntries(Object.entries(v).map(([pk, pv]) => [
+                pk,
+                flattenProperty(pv),
+            ]));
+        }
+        else if (k === "items") {
+            out.items = flattenProperty(v);
+        }
+        else if (k === "anyOf" || k === "oneOf" || k === "allOf") {
+            out[k] = v.map(flattenProperty);
+        }
+        else {
+            out[k] = v;
+        }
+    }
+    return out;
+}
+/**
+ * `name(required, optional?:a|b, list?:string[])`
+ *
+ * A model reads this natively. A 400-token JSON Schema becomes one line.
+ */
+export function signatureLine(tool, nameOverride) {
+    const schema = tool.schema ??
+        tool.inputSchema ??
+        tool.input_schema ??
+        {};
+    const props = schema.properties ?? {};
+    const required = new Set(schema.required ?? []);
+    const parts = Object.entries(props).map(([key, raw]) => {
+        const v = raw;
+        const opt = required.has(key) ? "" : "?";
+        let hint = "";
+        if (v?.enum)
+            hint = `:${v.enum.join("|")}`;
+        else if (v?.type === "array" && v.items?.type)
+            hint = `:${v.items.type}[]`;
+        return `${key}${opt}${hint}`;
+    });
+    return `${nameOverride ?? tool.name}(${parts.join(",")})`;
+}
+/**
+ * Aggressively shortened descriptor: first sentence, stop-words dropped,
+ * lowercased. Used by the `terse` map style, which trades legibility (and the
+ * real tool name) for a few tokens per line.
+ */
+export function terseDescriptor(s) {
+    return firstSentence(s)
+        .replace(/\b(the|a|an|to|of|for|in|with|and|that|this)\b ?/gi, "")
+        .replace(/[.!?]+$/, "")
+        .replace(/\s+/g, " ")
+        .trim()
+        .toLowerCase();
+}
+/** First sentence only — the rest is almost always restatement. */
+export function firstSentence(s) {
+    const m = s.match(/^(.*?[.!?])(\s|$)/s);
+    return (m ? m[1] : s).trim();
+}
+/** Cheap size proxy for stats. Not a token count — never used for billing. */
+export function countSchemaTokensApprox(x) {
+    return JSON.stringify(x ?? "").length;
+}
+/** Default namespace split: `github_create_issue` → github / create_issue. */
+export function defaultNamespaceOf(name) {
+    const i = name.search(/[_.]/);
+    if (i === -1)
+        return { ns: name, op: name };
+    return { ns: name.slice(0, i), op: name.slice(i + 1) };
+}
+export function normalize(tools, namespaceOf) {
+    const seen = new Set();
+    const out = tools.map((t) => {
+        if (!t?.name)
+            throw new Error("tool is missing a name");
+        if (seen.has(t.name))
+            throw new Error(`duplicate tool name: ${t.name}`);
+        seen.add(t.name);
+        const { ns, op } = namespaceOf(t.name);
+        return {
+            name: t.name,
+            description: t.description ?? "",
+            schema: t.inputSchema ?? t.input_schema ?? { type: "object" },
+            ns,
+            op,
+        };
+    });
+    // Deterministic ordering. Prefix stability is what makes prompt caching work;
+    // a caller iterating a Map or Set must not silently change the cache key.
+    return out.sort((a, b) => a.name.localeCompare(b.name));
+}

package/dist/runtime/similar.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Nearest-name matching for argument-error messages.
+ *
+ * Motivation, from measurement: across 360 benchmark runs the single most
+ * common failure was a model passing `query` to a parameter named `q` — 14 of
+ * 18 rejections. Each one burned a turn, and on a reasoning model a wasted turn
+ * is a wasted round of thinking, so this is a cost problem as much as a
+ * correctness one.
+ *
+ * We deliberately do NOT auto-remap. Silently moving a value from `query` to
+ * `q` would be guessing about caller intent, and a wrong guess dispatches bad
+ * data instead of raising. Instead we name the likely fix so the retry lands
+ * first time.
+ */
+/**
+ * Similarity in 0–100. Tuned so the cases we actually observed score high:
+ * `q`↔`query` (prefix), `pageSize`↔`per_page` (shared word), and unrelated
+ * names stay low enough to be filtered out.
+ */
+export declare function similarity(a: string, b: string): number;
+/**
+ * Closest candidate to `name`, or null when nothing is close enough to be worth
+ * suggesting. The floor matters: a bad suggestion is worse than none, because
+ * the model will act on it.
+ */
+export declare function nearest(name: string, candidates: readonly string[], floor?: number): string | null;

package/dist/runtime/similar.js

@@ -0,0 +1,86 @@
+/**
+ * Nearest-name matching for argument-error messages.
+ *
+ * Motivation, from measurement: across 360 benchmark runs the single most
+ * common failure was a model passing `query` to a parameter named `q` — 14 of
+ * 18 rejections. Each one burned a turn, and on a reasoning model a wasted turn
+ * is a wasted round of thinking, so this is a cost problem as much as a
+ * correctness one.
+ *
+ * We deliberately do NOT auto-remap. Silently moving a value from `query` to
+ * `q` would be guessing about caller intent, and a wrong guess dispatches bad
+ * data instead of raising. Instead we name the likely fix so the retry lands
+ * first time.
+ */
+/** Levenshtein distance, iterative single-row. */
+function editDistance(a, b) {
+    if (a === b)
+        return 0;
+    if (!a.length)
+        return b.length;
+    if (!b.length)
+        return a.length;
+    let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
+    for (let i = 1; i <= a.length; i++) {
+        const row = [i];
+        for (let j = 1; j <= b.length; j++) {
+            row[j] = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
+        }
+        prev = row;
+    }
+    return prev[b.length];
+}
+/** `per_page` / `perPage` / `PerPage` all normalise to `perpage`. */
+const norm = (s) => s.toLowerCase().replace(/[_\-\s]/g, "");
+/** Split camelCase and snake_case into lowercase word parts. */
+const words = (s) => s
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .split(/[_\-\s]+/)
+    .filter(Boolean)
+    .map((w) => w.toLowerCase());
+/**
+ * Similarity in 0–100. Tuned so the cases we actually observed score high:
+ * `q`↔`query` (prefix), `pageSize`↔`per_page` (shared word), and unrelated
+ * names stay low enough to be filtered out.
+ */
+export function similarity(a, b) {
+    if (!a || !b)
+        return 0;
+    const na = norm(a);
+    const nb = norm(b);
+    if (na === nb)
+        return 100;
+    // A short name that prefixes a longer one is the `q`/`query` case.
+    const [short, long] = na.length <= nb.length ? [na, nb] : [nb, na];
+    if (long.startsWith(short) || long.endsWith(short))
+        return 85;
+    const wa = new Set(words(a));
+    const wb = new Set(words(b));
+    const shared = [...wa].filter((w) => wb.has(w) && w.length > 2).length;
+    if (shared) {
+        const union = new Set([...wa, ...wb]).size;
+        return 50 + Math.round((shared / union) * 30);
+    }
+    if (long.includes(short) && short.length >= 3)
+        return 60;
+    const dist = editDistance(na, nb);
+    const ratio = 1 - dist / Math.max(na.length, nb.length);
+    return Math.max(0, Math.round(ratio * 55));
+}
+/**
+ * Closest candidate to `name`, or null when nothing is close enough to be worth
+ * suggesting. The floor matters: a bad suggestion is worse than none, because
+ * the model will act on it.
+ */
+export function nearest(name, candidates, floor = 55) {
+    let best = null;
+    let bestScore = 0;
+    for (const c of candidates) {
+        const s = similarity(name, c);
+        if (s > bestScore) {
+            bestScore = s;
+            best = c;
+        }
+    }
+    return bestScore >= floor ? best : null;
+}

package/dist/runtime/validate.d.ts

@@ -0,0 +1,11 @@
+import type { NormalizedTool } from "../types.js";
+/**
+ * Validate arguments against the *original* schema before dispatch.
+ *
+ * This matters most at levels 2 and 3: routing calls through a generic
+ * dispatcher moves argument checking out of the provider's constrained
+ * sampler and into here. Error strings are written for the model to read —
+ * they name the tool, the offending parameter, and where possible the exact
+ * fix, because a vague error costs another turn and another round of thinking.
+ */
+export declare function validateArgs(tool: NormalizedTool, args: Record<string, any>): string | null;

package/dist/runtime/validate.js

@@ -0,0 +1,87 @@
+import { nearest } from "./similar.js";
+/**
+ * Validate arguments against the *original* schema before dispatch.
+ *
+ * This matters most at levels 2 and 3: routing calls through a generic
+ * dispatcher moves argument checking out of the provider's constrained
+ * sampler and into here. Error strings are written for the model to read —
+ * they name the tool, the offending parameter, and where possible the exact
+ * fix, because a vague error costs another turn and another round of thinking.
+ */
+export function validateArgs(tool, args) {
+    const schema = tool.schema ?? {};
+    const props = schema.properties ?? {};
+    const required = schema.required ?? [];
+    const known = Object.keys(props);
+    const supplied = Object.keys(args);
+    // Keys the caller sent that the schema does not define. Computed first so a
+    // missing-required error can point at the likely rename.
+    const unknown = known.length ? supplied.filter((k) => !props[k]) : [];
+    for (const key of required) {
+        if (args[key] === undefined || args[key] === null || args[key] === "") {
+            // The `query` → `q` case: a required key is absent and an undefined key
+            // was supplied that looks like it. Suggest, never silently remap.
+            const guess = nearest(key, unknown);
+            const hint = guess
+                ? ` You passed "${guess}" — did you mean "${key}"? Rename it.`
+                : "";
+            return (`Missing required parameter "${key}" for ${tool.name}.` +
+                ` Required: ${required.join(", ")}.${hint}`);
+        }
+    }
+    if (unknown.length) {
+        const key = unknown[0];
+        const guess = nearest(key, known);
+        const hint = guess ? ` Did you mean "${guess}"?` : "";
+        return (`Unknown parameter "${key}" for ${tool.name}.` +
+            ` Accepted: ${known.join(", ")}.${hint}`);
+    }
+    for (const [key, value] of Object.entries(args)) {
+        const spec = props[key];
+        if (!spec)
+            continue;
+        const problem = checkType(tool.name, key, spec, value);
+        if (problem)
+            return problem;
+    }
+    return null;
+}
+function checkType(toolName, key, spec, value) {
+    if (spec.enum && !spec.enum.includes(value)) {
+        // Case drift ("approve" for "APPROVE") is common; show the exact spelling
+        // rather than only the list, so the retry is mechanical.
+        const ci = typeof value === "string"
+            ? spec.enum.find((e) => typeof e === "string" && e.toLowerCase() === value.toLowerCase())
+            : undefined;
+        const hint = ci ? ` Use exactly "${ci}".` : "";
+        return (`Invalid value for "${key}" on ${toolName}:` +
+            ` expected one of ${spec.enum.join(", ")}.${hint}`);
+    }
+    switch (spec.type) {
+        case "string":
+            if (typeof value !== "string")
+                return `Parameter "${key}" on ${toolName} must be a string.`;
+            break;
+        case "integer":
+            if (!Number.isInteger(value))
+                return `Parameter "${key}" on ${toolName} must be an integer.`;
+            break;
+        case "number":
+            if (typeof value !== "number")
+                return `Parameter "${key}" on ${toolName} must be a number.`;
+            break;
+        case "boolean":
+            if (typeof value !== "boolean")
+                return `Parameter "${key}" on ${toolName} must be a boolean.`;
+            break;
+        case "array":
+            if (!Array.isArray(value))
+                return `Parameter "${key}" on ${toolName} must be an array.`;
+            break;
+        case "object":
+            if (typeof value !== "object" || value === null || Array.isArray(value))
+                return `Parameter "${key}" on ${toolName} must be an object.`;
+            break;
+    }
+    return null;
+}

package/dist/types.d.ts

@@ -0,0 +1,112 @@
+/** Public types. Everything a consumer touches is declared here. */
+export type JsonSchema = {
+    type?: string;
+    properties?: Record<string, any>;
+    required?: string[];
+    items?: any;
+    enum?: any[];
+    [k: string]: any;
+};
+/**
+ * A tool as you already have it — the same shape MCP servers and every major
+ * SDK produce. `inputSchema` and `input_schema` are both accepted.
+ */
+export type Tool = {
+    name: string;
+    description?: string;
+    inputSchema?: JsonSchema;
+    input_schema?: JsonSchema;
+};
+/** Normalized internal form. */
+export type NormalizedTool = {
+    name: string;
+    description: string;
+    schema: JsonSchema;
+    ns: string;
+    op: string;
+};
+/**
+ * Compression levels. Each is a superset of the previous.
+ *
+ *  0  passthrough        — no change; useful as an A/B control in your own app
+ *  1  signature          — flatten JSON Schema, keep native tools + real names
+ *  2  namespace          — collapse related ops into one tool per namespace
+ *  3  minified           — single dispatcher + opaque codes
+ */
+export type Level = 0 | 1 | 2 | 3;
+/**
+ * How each line of the level-3 `<toolmap>` is rendered.
+ *
+ *   name           `a0 github_create_issue`
+ *   name+required  `a0 github_create_issue owner,repo,title`
+ *   signature      `a0 github_create_issue(owner,repo,title,body?,labels?)`
+ *   terse          `a0 create new issue in repository`
+ *
+ * `name` is the default and the smallest. `name+required` costs a few tokens
+ * per tool and exists to cut malformed arguments on models that fill the
+ * generic argument bag badly — the dispatcher levels give up provider-side
+ * constrained decoding, and this buys some of it back cheaply. `signature` also
+ * names the optional parameters, which removes most remaining `q()` lookups —
+ * a bigger cached map traded for fewer turns, which matters on models where
+ * every turn pays for a fresh round of reasoning. `terse` drops the real name
+ * entirely; it is the most aggressive and the least legible.
+ */
+export type MapStyle = "name" | "name+required" | "signature" | "terse";
+export type CompressOptions = {
+    level?: Level;
+    /** Level 3 only. Ignored at levels 0–2, which emit no map. Default "name". */
+    mapStyle?: MapStyle;
+    /**
+     * Group tools into namespaces. Default splits on the first `_` or `.`,
+     * which matches MCP naming convention (`github_create_issue`).
+     */
+    namespaceOf?: (toolName: string) => {
+        ns: string;
+        op: string;
+    };
+    /** Override the short alias used for a namespace at level 2. */
+    aliasOf?: (ns: string) => string;
+    /** Cap how many results a search/query meta-call returns. Default 8. */
+    searchLimit?: number;
+    /** Validate arguments against the original schema before dispatch. Default true. */
+    validate?: boolean;
+};
+export type Resolution = {
+    kind: "call";
+    name: string;
+    args: Record<string, any>;
+} | {
+    kind: "meta";
+    name: string;
+    result: string;
+} | {
+    kind: "error";
+    message: string;
+    recoverable: boolean;
+};
+export type CompressStats = {
+    level: Level;
+    toolCount: number;
+    wireToolCount: number;
+    originalChars: number;
+    compressedChars: number;
+    savedPct: number;
+};
+export type CompressResult = {
+    /** Tool definitions to send on the wire, in Anthropic shape. */
+    tools: unknown[];
+    /** Text to append to your system prompt. Empty string at levels 0–1. */
+    systemPreamble: string;
+    /** Whether the preamble should sit behind a cache breakpoint. */
+    cachePreamble: boolean;
+    /** Translate a raw model tool call back to a real one. */
+    resolve(rawName: string, rawArgs: Record<string, any>): Resolution;
+    /** Map a real tool name to its level-3 code. Throws below level 3. */
+    codeFor(toolName: string): string;
+    /** Build the raw call a model would emit for a real tool. Test/debug aid. */
+    encodeCallForTest(toolName: string, args: Record<string, any>): {
+        name: string;
+        args: Record<string, any>;
+    };
+    stats: CompressStats;
+};

package/dist/types.js

@@ -0,0 +1,2 @@
+/** Public types. Everything a consumer touches is declared here. */
+export {};

package/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "toolgz",
+  "version": "0.1.0",
+  "description": "Compress LLM tool definitions to reclaim context window. Measured across Anthropic, OpenAI, Google and xAI.",
+  "main": "dist/index.js",
+  "scripts": {
+    "test": "vitest run --exclude '**/integration/**'",
+    "brain": "tsx --disable-warning=ExperimentalWarning brain/brain.ts",
+    "bench": "tsx bench/harness/run.ts",
+    "build": "tsc",
+    "prepublishOnly": "npm run build && npm test",
+    "test:live": "vitest run tests/integration"
+  },
+  "keywords": [
+    "llm",
+    "mcp",
+    "tools",
+    "tool-calling",
+    "context-window",
+    "tokens",
+    "anthropic",
+    "claude",
+    "openai",
+    "gemini",
+    "grok",
+    "prompt-caching"
+  ],
+  "author": "",
+  "license": "Apache-2.0",
+  "type": "module",
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.115.0",
+    "@google/genai": "^2.13.0",
+    "dotenv": "^17.4.2",
+    "openai": "^6.49.0"
+  },
+  "devDependencies": {
+    "@types/node": "^26.1.1",
+    "tsx": "^4.23.1",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.10"
+  },
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./providers": {
+      "types": "./dist/providers/index.d.ts",
+      "import": "./dist/providers/index.js"
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/dperussina/toolgz.git"
+  },
+  "homepage": "https://github.com/dperussina/toolgz#readme",
+  "bugs": {
+    "url": "https://github.com/dperussina/toolgz/issues"
+  }
+}