thumbgate 1.29.1 → 1.30.0

package/scripts/thumbgate-bench.js

@@ -268,6 +268,13 @@ function withGateRuntime(options, callback) {
     'THUMBGATE_SECRET_SCAN_PROVIDER',
     'THUMBGATE_HARNESS',
     'THUMBGATE_HARNESS_CONFIG',
+    // Isolated bench must not inherit operator license / posture / daily-cap state.
+    // Without strict enforcement, deny expectations become warn-by-default and the
+    // published scorecard is unreproducible on a clean free-tier install.
+    'THUMBGATE_STRICT_ENFORCEMENT',
+    'THUMBGATE_ENFORCE_ENTITLEMENTS',
+    'THUMBGATE_PRO_LICENSE_KEY',
+    'THUMBGATE_LICENSE_KEY',
   ]);
   const runtimeDir = options.useRuntimeState
     ? null
@@ -289,6 +296,12 @@ function withGateRuntime(options, callback) {
       process.env.THUMBGATE_ATTRIBUTED_FEEDBACK = path.join(runtimeDir, 'attributed-feedback.jsonl');
       process.env.THUMBGATE_GUARDS_PATH = path.join(runtimeDir, 'pretool-guards.json');
       process.env.THUMBGATE_SECRET_SCAN_PROVIDER = 'heuristic';
+      // Pin enforcement posture for golden deny expectations (scorecard reproducibility).
+      process.env.THUMBGATE_STRICT_ENFORCEMENT = '1';
+      // Do not hard-fail free-tier installs; isolation already avoids daily-cap state.
+      delete process.env.THUMBGATE_ENFORCE_ENTITLEMENTS;
+      delete process.env.THUMBGATE_PRO_LICENSE_KEY;
+      delete process.env.THUMBGATE_LICENSE_KEY;
       fs.mkdirSync(process.env.THUMBGATE_FEEDBACK_DIR, { recursive: true });
       fs.writeFileSync(process.env.THUMBGATE_FEEDBACK_LOG, '');
       fs.writeFileSync(process.env.THUMBGATE_ATTRIBUTED_FEEDBACK, '');

package/scripts/tool-contract-validator.js

@@ -2,75 +2,303 @@
 
 /**
  * Tool Contract Validator
- * Validates tool arguments against the tool's inputSchema.
+ *
+ * A dependency-free JSON Schema subset for tool arguments and structured
+ * outputs. The validator intentionally fails closed for unsupported schema
+ * shapes used at an agent boundary instead of silently accepting them.
  */
+
+const FORMAT_CHECKS = Object.freeze({
+  'date-time': (value) => !Number.isNaN(Date.parse(value)) && /T/.test(value),
+  email: isEmail,
+  uri: (value) => {
+    try {
+      new URL(value);
+      return true;
+    } catch {
+      return false;
+    }
+  },
+});
+
 function validateToolContract(schema, args) {
+  if (!schema) return { valid: true, errors: [] };
   const errors = [];
-  if (!schema) return { valid: true, errors };
+  validateValue(schema, args, '', errors);
+  return { valid: errors.length === 0, errors };
+}
 
-  if (schema.type === 'object') {
-    if (typeof args !== 'object' || args === null || Array.isArray(args)) {
-      errors.push(`Expected object, got ${args === null ? 'null' : Array.isArray(args) ? 'array' : typeof args}`);
-      return { valid: false, errors };
+function validateStructuredOutput(output, schema) {
+  let value = output;
+  if (typeof output === 'string') {
+    try {
+      value = JSON.parse(output);
+    } catch (err) {
+      return {
+        valid: false,
+        errors: [`Structured output must be valid JSON: ${err.message}`],
+        value: null,
+      };
     }
+  }
+  const result = validateToolContract(schema, value);
+  return { ...result, value };
+}
+
+function validateConstant(schema, value, path, errors) {
+  if (schema.const !== undefined && !deepEqual(value, schema.const)) {
+    errors.push(`${label(path)} must equal ${display(schema.const)} (got ${display(value)})`);
+  }
+}
+
+function validateEnum(schema, value, path, errors) {
+  if (!Array.isArray(schema.enum) || schema.enum.some((entry) => deepEqual(entry, value))) return;
+  if (path && schema.type === 'string') {
+    errors.push(`Parameter '${path}' must be one of [${schema.enum.join(', ')}] (got '${value}')`);
+    return;
+  }
+  errors.push(`${label(path)} must be one of [${schema.enum.map(display).join(', ')}] (got ${display(value)})`);
+}
+
+function validateValue(schema, value, path, errors) {
+  if (schema === true || !schema) return;
+  if (schema === false) {
+    errors.push(`${label(path)} is disallowed by schema`);
+    return;
+  }
+
+  validateConstant(schema, value, path, errors);
+  validateEnum(schema, value, path, errors);
+
+  validateCombinators(schema, value, path, errors);
+
+  if (value === undefined) return;
+  const allowedTypes = normalizeTypes(schema.type);
+  if (allowedTypes.length > 0 && !allowedTypes.some((type) => matchesType(type, value))) {
+    errors.push(typeError(path, allowedTypes, value));
+    return;
+  }
+
+  if (value === null) return;
+  const actualType = jsonType(value);
+  if (actualType === 'object') validateObject(schema, value, path, errors);
+  if (actualType === 'array') validateArray(schema, value, path, errors);
+  if (actualType === 'string') validateString(schema, value, path, errors);
+  if (actualType === 'number' || actualType === 'integer') validateNumber(schema, value, path, errors);
+}
+
+function validateCombinators(schema, value, path, errors) {
+  if (Array.isArray(schema.allOf)) {
+    for (const child of schema.allOf) validateValue(child, value, path, errors);
+  }
+  if (Array.isArray(schema.anyOf)) {
+    const matches = schema.anyOf.filter((child) => validateChild(child, value).valid).length;
+    if (matches === 0) errors.push(`${label(path)} must match at least one anyOf schema`);
+  }
+  if (Array.isArray(schema.oneOf)) {
+    const matches = schema.oneOf.filter((child) => validateChild(child, value).valid).length;
+    if (matches !== 1) errors.push(`${label(path)} must match exactly one oneOf schema (matched ${matches})`);
+  }
+  if (schema.not && validateChild(schema.not, value).valid) {
+    errors.push(`${label(path)} matches a forbidden schema`);
+  }
+}
+
+function validateChild(schema, value) {
+  const errors = [];
+  validateValue(schema, value, '', errors);
+  return { valid: errors.length === 0, errors };
+}
+
+function validatePropertyCount(schema, keys, path, errors) {
+  if (Number.isFinite(schema.minProperties) && keys.length < schema.minProperties) {
+    errors.push(`${label(path)} must have at least ${schema.minProperties} properties`);
+  }
+  if (Number.isFinite(schema.maxProperties) && keys.length > schema.maxProperties) {
+    errors.push(`${label(path)} must have at most ${schema.maxProperties} properties`);
+  }
+}
+
+function validateRequiredProperties(schema, value, path, errors) {
+  for (const required of schema.required || []) {
+    if (value[required] === undefined || value[required] === null || value[required] === '') {
+      const requiredPath = joinPath(path, required);
+      errors.push(path
+        ? `Parameter '${path}': Missing required parameter: '${required}'`
+        : `Missing required parameter: '${required}'`);
+      if (path && requiredPath === path) break;
+    }
+  }
+}
+
+function validateKnownProperties(schema, value, path, errors) {
+  const properties = schema.properties || {};
+  for (const [key, childSchema] of Object.entries(properties)) {
+    if (value[key] === undefined) continue;
+    validateValue(childSchema, value[key], joinPath(path, key), errors);
+  }
+}
+
+function validateAdditionalProperties(schema, value, keys, path, errors) {
+  const properties = schema.properties || {};
+  for (const key of keys) {
+    if (Object.hasOwn(properties, key)) continue;
+    if (schema.additionalProperties === false) {
+      errors.push(`Unexpected parameter: '${joinPath(path, key)}'`);
+    } else if (schema.additionalProperties && typeof schema.additionalProperties === 'object') {
+      validateValue(schema.additionalProperties, value[key], joinPath(path, key), errors);
+    }
+  }
+}
+
+function validateObject(schema, value, path, errors) {
+  const keys = Object.keys(value);
+  validatePropertyCount(schema, keys, path, errors);
+  validateRequiredProperties(schema, value, path, errors);
+  validateKnownProperties(schema, value, path, errors);
+  validateAdditionalProperties(schema, value, keys, path, errors);
+}
 
-    // Check required fields
-    if (Array.isArray(schema.required)) {
-      for (const req of schema.required) {
-        if (args[req] === undefined || args[req] === null || args[req] === '') {
-          errors.push(`Missing required parameter: '${req}'`);
-        }
-      }
+function validateArray(schema, value, path, errors) {
+  if (Number.isFinite(schema.minItems) && value.length < schema.minItems) {
+    errors.push(`${label(path)} must contain at least ${schema.minItems} items`);
+  }
+  if (Number.isFinite(schema.maxItems) && value.length > schema.maxItems) {
+    errors.push(`${label(path)} must contain at most ${schema.maxItems} items`);
+  }
+  if (schema.uniqueItems === true) {
+    const serialized = value.map(stableStringify);
+    if (new Set(serialized).size !== serialized.length) {
+      errors.push(`${label(path)} must contain unique items`);
     }
+  }
+  if (schema.items) {
+    value.forEach((entry, index) => validateValue(schema.items, entry, `${path}[${index}]`, errors));
+  }
+}
 
-    // Check properties
-    if (schema.properties) {
-      for (const [key, propSchema] of Object.entries(schema.properties)) {
-        const value = args[key];
-        if (value === undefined || value === null) continue; // Optional field not provided
-
-        const valType = typeof value;
-        if (propSchema.type === 'string') {
-          if (valType !== 'string') {
-            errors.push(`Parameter '${key}' must be a string (got ${valType})`);
-          } else if (Array.isArray(propSchema.enum)) {
-            if (!propSchema.enum.includes(value)) {
-              errors.push(`Parameter '${key}' must be one of [${propSchema.enum.join(', ')}] (got '${value}')`);
-            }
-          }
-        } else if (propSchema.type === 'number') {
-          if (valType !== 'number' || isNaN(value)) {
-            errors.push(`Parameter '${key}' must be a number (got ${valType})`);
-          }
-        } else if (propSchema.type === 'boolean') {
-          if (valType !== 'boolean') {
-            errors.push(`Parameter '${key}' must be a boolean (got ${valType})`);
-          }
-        } else if (propSchema.type === 'array') {
-          if (!Array.isArray(value)) {
-            errors.push(`Parameter '${key}' must be an array (got ${valType})`);
-          }
-        } else if (propSchema.type === 'object') {
-          if (valType !== 'object' || value === null || Array.isArray(value)) {
-            errors.push(`Parameter '${key}' must be an object (got ${valType})`);
-          } else {
-            // Recurse for nested objects
-            const subRes = validateToolContract(propSchema, value);
-            if (!subRes.valid) {
-              for (const err of subRes.errors) {
-                errors.push(`Parameter '${key}': ${err}`);
-              }
-            }
-          }
-        }
-      }
+function validateString(schema, value, path, errors) {
+  if (Number.isFinite(schema.minLength) && value.length < schema.minLength) {
+    errors.push(`${label(path)} must contain at least ${schema.minLength} characters`);
+  }
+  if (Number.isFinite(schema.maxLength) && value.length > schema.maxLength) {
+    errors.push(`${label(path)} must contain at most ${schema.maxLength} characters`);
+  }
+  if (schema.pattern) {
+    let regex;
+    try {
+      regex = new RegExp(schema.pattern);
+    } catch {
+      errors.push(`${label(path)} uses invalid schema pattern '${schema.pattern}'`);
+      return;
     }
+    if (!regex.test(value)) errors.push(`${label(path)} must match pattern ${schema.pattern}`);
+  }
+  if (schema.format && FORMAT_CHECKS[schema.format] && !FORMAT_CHECKS[schema.format](value)) {
+    errors.push(`${label(path)} must match format ${schema.format}`);
   }
+}
+
+function validateNumber(schema, value, path, errors) {
+  if (Number.isFinite(schema.minimum) && value < schema.minimum) {
+    errors.push(`${label(path)} must be >= ${schema.minimum}`);
+  }
+  if (Number.isFinite(schema.maximum) && value > schema.maximum) {
+    errors.push(`${label(path)} must be <= ${schema.maximum}`);
+  }
+  if (Number.isFinite(schema.exclusiveMinimum) && value <= schema.exclusiveMinimum) {
+    errors.push(`${label(path)} must be > ${schema.exclusiveMinimum}`);
+  }
+  if (Number.isFinite(schema.exclusiveMaximum) && value >= schema.exclusiveMaximum) {
+    errors.push(`${label(path)} must be < ${schema.exclusiveMaximum}`);
+  }
+  if (Number.isFinite(schema.multipleOf) && !isMultipleOf(value, schema.multipleOf)) {
+    errors.push(`${label(path)} must be a multiple of ${schema.multipleOf}`);
+  }
+}
+
+function normalizeTypes(type) {
+  if (!type) return [];
+  return Array.isArray(type) ? type : [type];
+}
+
+function matchesType(type, value) {
+  if (type === 'null') return value === null;
+  if (type === 'array') return Array.isArray(value);
+  if (type === 'object') return value !== null && typeof value === 'object' && !Array.isArray(value);
+  if (type === 'integer') return Number.isInteger(value);
+  if (type === 'number') return typeof value === 'number' && Number.isFinite(value);
+  return typeof value === type;
+}
+
+function isEmail(value) {
+  if (typeof value !== 'string' || /\s/.test(value)) return false;
+  const atIndex = value.indexOf('@');
+  if (atIndex <= 0 || atIndex !== value.lastIndexOf('@')) return false;
+  const domain = value.slice(atIndex + 1);
+  const dotIndex = domain.lastIndexOf('.');
+  return dotIndex > 0 && dotIndex < domain.length - 1;
+}
+
+function jsonType(value) {
+  if (value === null) return 'null';
+  if (Array.isArray(value)) return 'array';
+  if (Number.isInteger(value)) return 'integer';
+  if (typeof value === 'number') return 'number';
+  return typeof value;
+}
+
+function typeError(path, types, value) {
+  const expected = types.length === 1 ? types[0] : types.join(' or ');
+  const actual = actualTypeName(value);
+  if (!path) return `Expected ${expected}, got ${actual}`;
+  return `Parameter '${path}' must be ${article(expected)} ${expected} (got ${actual})`;
+}
+
+function actualTypeName(value) {
+  if (value === null) return 'null';
+  if (Array.isArray(value)) return 'array';
+  return typeof value;
+}
+
+function article(value) {
+  return /^[aeiou]/i.test(value) ? 'an' : 'a';
+}
+
+function label(path) {
+  return path ? `Parameter '${path}'` : 'Value';
+}
+
+function joinPath(parent, child) {
+  return parent ? `${parent}.${child}` : child;
+}
+
+function display(value) {
+  if (typeof value === 'string') return `'${value}'`;
+  return stableStringify(value);
+}
+
+function stableStringify(value) {
+  if (!value || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+  const keys = Object.keys(value).sort((left, right) => left.localeCompare(right));
+  const properties = keys.map((key) => [
+    JSON.stringify(key),
+    stableStringify(value[key]),
+  ].join(':'));
+  return ['{', properties.join(','), '}'].join('');
+}
+
+function deepEqual(a, b) {
+  return stableStringify(a) === stableStringify(b);
+}
 
-  return {
-    valid: errors.length === 0,
-    errors,
-  };
+function isMultipleOf(value, multiple) {
+  const quotient = value / multiple;
+  return Math.abs(quotient - Math.round(quotient)) < Number.EPSILON * 10;
 }
 
-module.exports = { validateToolContract };
+module.exports = {
+  validateStructuredOutput,
+  validateToolContract,
+};

package/scripts/tool-kpi-tracker.js

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+'use strict';
+
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+const { resolveFeedbackDir } = require('./feedback-paths');
+const { readJsonl } = require('./fs-utils');
+
+function getKpiLogPath(options = {}) {
+  return path.join(
+    options.feedbackDir ? path.resolve(options.feedbackDir) : resolveFeedbackDir(),
+    'tool-kpi.jsonl',
+  );
+}
+
+function recordToolCall({
+  toolName,
+  serverName,
+  latencyMs,
+  success,
+  agentId,
+  metadata,
+  feedbackDir,
+} = {}) {
+  const logPath = getKpiLogPath({ feedbackDir });
+  fs.mkdirSync(path.dirname(logPath), { recursive: true });
+  const entry = {
+    id: `kpi_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`,
+    timestamp: new Date().toISOString(),
+    toolName: toolName || 'unknown',
+    serverName: serverName || 'default',
+    latencyMs: typeof latencyMs === 'number' ? latencyMs : 0,
+    success: success !== false,
+    agentId: agentId || 'unknown',
+    metadata: metadata || {},
+  };
+  fs.appendFileSync(logPath, `${JSON.stringify(entry)}\n`);
+  return entry;
+}
+
+function percentile(sorted, quantile) {
+  if (sorted.length === 0) return 0;
+  const index = Math.ceil((quantile / 100) * sorted.length) - 1;
+  return sorted[Math.max(0, index)];
+}
+
+function computeToolKpis({ periodHours = 24, feedbackDir } = {}) {
+  const entries = readJsonl(getKpiLogPath({ feedbackDir }));
+  const cutoff = Date.now() - periodHours * 60 * 60 * 1000;
+  const recent = entries.filter((entry) => new Date(entry.timestamp).getTime() > cutoff);
+  const byTool = {};
+  for (const entry of recent) {
+    const key = entry.toolName;
+    if (!byTool[key]) {
+      byTool[key] = {
+        toolName: key,
+        calls: [],
+        successes: 0,
+        failures: 0,
+      };
+    }
+    byTool[key].calls.push(entry.latencyMs);
+    if (entry.success) byTool[key].successes += 1;
+    else byTool[key].failures += 1;
+  }
+  const tools = Object.values(byTool)
+    .map((tool) => {
+      const sorted = tool.calls.slice().sort((left, right) => left - right);
+      const total = tool.successes + tool.failures;
+      return {
+        toolName: tool.toolName,
+        requestCount: total,
+        successRate: total > 0 ? Math.round((tool.successes / total) * 1000) / 10 : 100,
+        p50: Math.round(percentile(sorted, 50)),
+        p90: Math.round(percentile(sorted, 90)),
+        p95: Math.round(percentile(sorted, 95)),
+        successes: tool.successes,
+        failures: tool.failures,
+      };
+    })
+    .sort((left, right) => right.requestCount - left.requestCount);
+
+  const byServer = {};
+  for (const entry of recent) {
+    const key = entry.serverName;
+    if (!byServer[key]) byServer[key] = { serverName: key, total: 0, successes: 0 };
+    byServer[key].total += 1;
+    if (entry.success) byServer[key].successes += 1;
+  }
+  const servers = Object.values(byServer).map((server) => ({
+    serverName: server.serverName,
+    totalCalls: server.total,
+    successRate: server.total > 0
+      ? Math.round((server.successes / server.total) * 1000) / 10
+      : 100,
+  }));
+  return {
+    periodHours,
+    totalCalls: recent.length,
+    evidenceStatus: recent.length > 0 ? 'measured' : 'insufficient_evidence',
+    tools,
+    servers,
+  };
+}
+
+function getAtRiskTools({
+  successRateThreshold = 90,
+  p95Threshold = 500,
+  periodHours = 24,
+  feedbackDir,
+} = {}) {
+  const { tools } = computeToolKpis({ periodHours, feedbackDir });
+  return tools.filter((tool) => tool.requestCount >= 3
+    && (tool.successRate < successRateThreshold || tool.p95 > p95Threshold));
+}
+
+module.exports = {
+  computeToolKpis,
+  getAtRiskTools,
+  getKpiLogPath,
+  percentile,
+  recordToolCall,
+};