thumbgate 1.27.18 → 1.27.20

package/scripts/mcp-oauth.js

@@ -0,0 +1,293 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * mcp-oauth.js — OAuth 2.1 (PKCE) authorization-server machinery for the remote
+ * ThumbGate MCP connector, required by the Claude Connectors Directory
+ * ("Use OAuth 2.0 for authenticated services").
+ *
+ * Pure, dependency-free, IO-free functions + an injectable store, so the full
+ * flow is unit-testable without a network. The HTTP wiring (in src/api/server.js)
+ * is a thin shell over these.
+ *
+ * Implements:
+ *   - RFC 9728 protected-resource metadata
+ *   - RFC 8414 authorization-server metadata
+ *   - RFC 7591 dynamic client registration (open registration)
+ *   - Authorization-code grant with mandatory PKCE S256 (RFC 7636)
+ *   - Bearer access tokens bound to a ThumbGate API key (so existing gating is unchanged)
+ *
+ * SECURITY NOTES:
+ *   - PKCE S256 is mandatory; the plain method is rejected.
+ *   - Auth codes are single-use and short-lived (60s); access tokens TTL-bound.
+ *   - redirect_uri is matched exactly against the registered set.
+ *   - The token never exposes the bound ThumbGate key; it maps server-side only.
+ */
+
+const crypto = require('crypto');
+
+const AUTH_CODE_TTL_MS = 60 * 1000; // 1 minute
+const ACCESS_TOKEN_TTL_MS = 60 * 60 * 1000; // 1 hour
+const DEFAULT_SCOPE = 'mcp:read mcp:write';
+
+// Upper bounds on the in-memory store. The registration and authorization
+// endpoints are reachable pre-auth, so without a cap a malicious caller could
+// grow these Maps unboundedly and exhaust server memory. When a Map is full we
+// evict the oldest entry (FIFO) rather than deny service to legitimate clients.
+const MAX_CLIENTS = 10000;
+const MAX_CODES = 10000;
+const MAX_TOKENS = 50000;
+
+function now() {
+  return Date.now();
+}
+
+function randomToken(bytes = 32) {
+  return crypto.randomBytes(bytes).toString('base64url');
+}
+
+function base64UrlSha256(input) {
+  return crypto.createHash('sha256').update(String(input)).digest('base64url');
+}
+
+/**
+ * Create a fresh in-memory store.
+ *
+ * DURABILITY (known limitation): this uses plain Maps, so a process restart or a
+ * multi-instance / load-balanced deployment will drop issued tokens and
+ * registered clients (clients see 401s, in-flight authorizations break). For
+ * single-instance use this is fine; production multi-tenancy needs a durable,
+ * shared backing (Redis/DB) — tracked as the per-user-data-scoping follow-up.
+ * Entry counts are bounded (see MAX_* and capInsert) to prevent memory
+ * exhaustion from anonymous calls to the registration/authorization endpoints.
+ */
+function createStore() {
+  return {
+    clients: new Map(),
+    codes: new Map(),
+    tokens: new Map(),
+  };
+}
+
+/** Insert into a Map, evicting the oldest entry (FIFO) once `max` is reached. */
+function capInsert(map, key, value, max) {
+  if (map.size >= max && !map.has(key)) {
+    const oldest = map.keys().next().value;
+    if (oldest !== undefined) map.delete(oldest);
+  }
+  map.set(key, value);
+}
+
+// ---------------------------------------------------------------------------
+// Metadata (RFC 9728 / RFC 8414)
+// ---------------------------------------------------------------------------
+
+function trimSlash(u) {
+  // Non-regex trailing-slash strip (avoids a SonarCloud S5852 false-positive on a
+  // provably-linear pattern).
+  let s = String(u || '');
+  while (s.endsWith('/')) s = s.slice(0, -1);
+  return s;
+}
+
+function buildProtectedResourceMetadata(baseUrl) {
+  const base = trimSlash(baseUrl);
+  return {
+    resource: `${base}/mcp`,
+    authorization_servers: [base],
+    bearer_methods_supported: ['header'],
+    scopes_supported: ['mcp:read', 'mcp:write'],
+    resource_documentation: `${base}/docs/connectors`,
+  };
+}
+
+function buildAuthServerMetadata(baseUrl) {
+  const base = trimSlash(baseUrl);
+  return {
+    issuer: base,
+    authorization_endpoint: `${base}/oauth/authorize`,
+    token_endpoint: `${base}/oauth/token`,
+    registration_endpoint: `${base}/oauth/register`,
+    scopes_supported: ['mcp:read', 'mcp:write'],
+    response_types_supported: ['code'],
+    grant_types_supported: ['authorization_code'],
+    code_challenge_methods_supported: ['S256'],
+    token_endpoint_auth_methods_supported: ['none'],
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Dynamic client registration (RFC 7591)
+// ---------------------------------------------------------------------------
+
+// The MCP authorization spec is explicit: "All redirect URIs MUST be either
+// `localhost` or use HTTPS." We therefore accept only HTTPS and loopback
+// (http://localhost | http://127.0.0.1) and reject every other scheme —
+// including native-app custom schemes (myapp://, intent://, etc.), which the MCP
+// profile does not sanction and the real client (Claude) does not use.
+function isAllowedRedirectUri(uri) {
+  const u = String(uri || '');
+  if (/^https:\/\//i.test(u)) return true;
+  if (/^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?(\/|$)/i.test(u)) return true;
+  return false;
+}
+
+function registerClient(store, body = {}) {
+  const redirectUris = Array.isArray(body.redirect_uris) ? body.redirect_uris.filter(Boolean) : [];
+  if (redirectUris.length === 0) {
+    return { error: 'invalid_redirect_uri', error_description: 'redirect_uris is required' };
+  }
+  for (const uri of redirectUris) {
+    if (!isAllowedRedirectUri(uri)) {
+      return { error: 'invalid_redirect_uri', error_description: `unsupported redirect_uri: ${uri}` };
+    }
+  }
+  const clientId = `tg_${randomToken(16)}`;
+  const record = {
+    client_id: clientId,
+    redirect_uris: redirectUris,
+    token_endpoint_auth_method: 'none',
+    grant_types: ['authorization_code'],
+    response_types: ['code'],
+    client_name: typeof body.client_name === 'string' ? body.client_name.slice(0, 200) : 'mcp-client',
+    created_at: now(),
+  };
+  capInsert(store.clients, clientId, record, MAX_CLIENTS);
+  return record;
+}
+
+function getClient(store, clientId) {
+  return store.clients.get(clientId) || null;
+}
+
+// ---------------------------------------------------------------------------
+// Authorization code (PKCE S256)
+// ---------------------------------------------------------------------------
+
+/**
+ * Issue an auth code. `boundKey` is the ThumbGate API key the resulting access
+ * token will act as (resolved by the authorize step once the user consents).
+ */
+function createAuthorizationCode(store, {
+  clientId, redirectUri, codeChallenge, codeChallengeMethod, scope, boundKey, state, resource,
+} = {}) {
+  const client = getClient(store, clientId);
+  if (!client) return { error: 'invalid_client' };
+  if (!client.redirect_uris.includes(redirectUri)) return { error: 'invalid_request', error_description: 'redirect_uri mismatch' };
+  if (codeChallengeMethod !== 'S256') return { error: 'invalid_request', error_description: 'code_challenge_method must be S256' };
+  if (!codeChallenge || String(codeChallenge).length < 16) return { error: 'invalid_request', error_description: 'code_challenge required' };
+
+  const code = randomToken(24);
+  capInsert(store.codes, code, {
+    clientId,
+    redirectUri,
+    codeChallenge,
+    scope: scope || DEFAULT_SCOPE,
+    boundKey: boundKey || '',
+    resource: resource || '', // RFC 8707 resource indicator (the MCP server URL)
+    expiresAt: now() + AUTH_CODE_TTL_MS,
+    used: false,
+  }, MAX_CODES);
+  return { code, state };
+}
+
+function verifyPkce(codeChallenge, codeVerifier) {
+  if (!codeChallenge || !codeVerifier) return false;
+  // RFC 7636: verifier 43–128 chars.
+  if (String(codeVerifier).length < 43 || String(codeVerifier).length > 128) return false;
+  try {
+    return crypto.timingSafeEqual(
+      Buffer.from(base64UrlSha256(codeVerifier)),
+      Buffer.from(String(codeChallenge)),
+    );
+  } catch {
+    return false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Token exchange + validation
+// ---------------------------------------------------------------------------
+
+function exchangeCode(store, { code, codeVerifier, clientId, redirectUri, resource } = {}) {
+  const entry = store.codes.get(code);
+  if (!entry) return { error: 'invalid_grant', error_description: 'unknown code' };
+  // Single-use + expiry: consume regardless of outcome.
+  store.codes.delete(code);
+  if (entry.used) return { error: 'invalid_grant', error_description: 'code already used' };
+  if (now() > entry.expiresAt) return { error: 'invalid_grant', error_description: 'code expired' };
+  if (entry.clientId !== clientId) return { error: 'invalid_grant', error_description: 'client mismatch' };
+  if (entry.redirectUri !== redirectUri) return { error: 'invalid_grant', error_description: 'redirect_uri mismatch' };
+  if (!verifyPkce(entry.codeChallenge, codeVerifier)) return { error: 'invalid_grant', error_description: 'PKCE verification failed' };
+  // RFC 8707: the resource at token time must match the one bound at authorize time.
+  if (entry.resource && resource && entry.resource !== resource) {
+    return { error: 'invalid_target', error_description: 'resource indicator mismatch' };
+  }
+
+  const accessToken = `tgat_${randomToken(32)}`;
+  capInsert(store.tokens, accessToken, {
+    boundKey: entry.boundKey,
+    scope: entry.scope,
+    clientId,
+    aud: entry.resource || resource || '',
+    expiresAt: now() + ACCESS_TOKEN_TTL_MS,
+  }, MAX_TOKENS);
+  return {
+    access_token: accessToken,
+    token_type: 'Bearer',
+    expires_in: Math.floor(ACCESS_TOKEN_TTL_MS / 1000),
+    scope: entry.scope,
+  };
+}
+
+/** Resolve an access token to its session, or null if invalid/expired. */
+function resolveAccessToken(store, token) {
+  if (!token) return null;
+  const entry = store.tokens.get(token);
+  if (!entry) return null;
+  if (now() > entry.expiresAt) {
+    store.tokens.delete(token);
+    return null;
+  }
+  return { boundKey: entry.boundKey, scope: entry.scope, clientId: entry.clientId, aud: entry.aud };
+}
+
+/**
+ * RFC 8707 audience validation: a token is valid for `expectedResource` only if it
+ * was issued for it (or carries no audience, for back-compat). MCP servers MUST
+ * reject tokens minted for a different resource.
+ */
+function tokenAudienceValid(session, expectedResource) {
+  if (!session) return false;
+  if (!session.aud) return true; // no audience recorded — accept (back-compat)
+  return session.aud === expectedResource;
+}
+
+/** Best-effort GC of expired codes/tokens (call opportunistically). */
+function pruneExpired(store) {
+  const t = now();
+  for (const [k, v] of store.codes) if (t > v.expiresAt) store.codes.delete(k);
+  for (const [k, v] of store.tokens) if (t > v.expiresAt) store.tokens.delete(k);
+}
+
+module.exports = {
+  createStore,
+  buildProtectedResourceMetadata,
+  buildAuthServerMetadata,
+  registerClient,
+  getClient,
+  createAuthorizationCode,
+  verifyPkce,
+  exchangeCode,
+  resolveAccessToken,
+  tokenAudienceValid,
+  pruneExpired,
+  isAllowedRedirectUri,
+  base64UrlSha256,
+  AUTH_CODE_TTL_MS,
+  ACCESS_TOKEN_TTL_MS,
+  DEFAULT_SCOPE,
+  MAX_CLIENTS,
+  MAX_CODES,
+  MAX_TOKENS,
+};

package/scripts/memory-scope-readiness.js

@@ -2,6 +2,38 @@
 'use strict';
 
 const REQUIRED_SCOPE_FIELDS = ['entityId', 'projectId', 'processId', 'sessionId'];
+const MEMORY_OS_LAYERS = Object.freeze([
+  {
+    id: 'file_layer',
+    name: 'File Layer',
+    purpose: 'Raw feedback, tool receipts, sessions, and memory rows are durably stored before interpretation.',
+  },
+  {
+    id: 'vector_db_layer',
+    name: 'Vector DB Layer',
+    purpose: 'Semantic retrieval can find related lessons without stuffing every raw memory into context.',
+  },
+  {
+    id: 'structured_facts_layer',
+    name: 'Structured Facts Layer',
+    purpose: 'Confirmed account, project, policy, and budget facts are typed separately from fuzzy memories.',
+  },
+  {
+    id: 'auto_curation_layer',
+    name: 'Auto Curation Layer',
+    purpose: 'Duplicate, stale, contradictory, and unscoped memories are consolidated before retrieval quality decays.',
+  },
+  {
+    id: 'context_layer',
+    name: 'Context Layer',
+    purpose: 'Only relevant scoped memories enter a given tool call, PR, deployment, or support session.',
+  },
+  {
+    id: 'interface_layer',
+    name: 'Interface Layer',
+    purpose: 'The memory contract is exposed through CLI, MCP, hooks, dashboards, and agent adapters without model lock-in.',
+  },
+]);
 
 const FIELD_ALIASES = {
   entityId: [
@@ -228,6 +260,128 @@ function buildRecommendations({ unscopedRecords, crossScopeDuplicates }) {
   return recommendations;
 }
 
+function hasEmbeddingEvidence(record = {}) {
+  return Boolean(
+    record.embedding
+    || record.vector
+    || record.embeddingId
+    || record.metadata?.embedding
+    || record.metadata?.embeddingId
+    || record.metadata?.vectorId
+    || record.semanticKey
+    || record.metadata?.semanticKey
+  );
+}
+
+function hasStructuredFactEvidence(record = {}) {
+  const type = String(record.type || record.kind || record.memoryType || record.metadata?.type || '').toLowerCase();
+  return type === 'fact'
+    || type === 'structured_fact'
+    || Boolean(record.factKey || record.fact || record.metadata?.factKey || record.metadata?.fact);
+}
+
+function hasContextEvidence(record = {}) {
+  return Boolean(
+    record.contextPackId
+    || record.contextPack
+    || record.metadata?.contextPackId
+    || record.metadata?.contextPack
+    || record.retrievalQuery
+    || record.metadata?.retrievalQuery
+  );
+}
+
+function boolCapability(capabilities = {}, ...keys) {
+  return keys.some((key) => capabilities[key] === true);
+}
+
+function buildMemoryOsLayerReport(records = [], capabilities = {}) {
+  const scopeReport = buildMemoryScopeReadinessReport(records);
+  const semanticRecords = records.filter(hasEmbeddingEvidence);
+  const structuredFactRecords = records.filter(hasStructuredFactEvidence);
+  const contextRecords = records.filter(hasContextEvidence);
+  const curationReady = scopeReport.unscopedRecords === 0 && scopeReport.crossScopeDuplicates.length === 0;
+
+  const checks = [
+    {
+      id: 'file_layer',
+      ok: records.length > 0 || boolCapability(capabilities, 'rawStorage', 'fileLayer'),
+      evidence: {
+        records: records.length,
+        durableStore: Boolean(records.length > 0 || capabilities.rawStorage || capabilities.fileLayer),
+      },
+      recommendation: 'Capture raw feedback, action receipts, and tool outcomes before promoting memories.',
+    },
+    {
+      id: 'vector_db_layer',
+      ok: semanticRecords.length > 0 || boolCapability(capabilities, 'semanticSearch', 'vectorDbLayer'),
+      evidence: {
+        semanticRecords: semanticRecords.length,
+        semanticSearch: Boolean(capabilities.semanticSearch || capabilities.vectorDbLayer),
+      },
+      recommendation: 'Index lessons with semantic keys or embeddings so related failures are retrieved before action.',
+    },
+    {
+      id: 'structured_facts_layer',
+      ok: structuredFactRecords.length > 0 || boolCapability(capabilities, 'structuredFacts', 'structuredFactsLayer'),
+      evidence: {
+        structuredFactRecords: structuredFactRecords.length,
+        structuredFacts: Boolean(capabilities.structuredFacts || capabilities.structuredFactsLayer),
+      },
+      recommendation: 'Store confirmed customer, project, policy, and budget facts as typed records, not just prose.',
+    },
+    {
+      id: 'auto_curation_layer',
+      ok: curationReady && boolCapability(capabilities, 'autoCuration', 'dedupe', 'autoCurationLayer'),
+      evidence: {
+        unscopedRecords: scopeReport.unscopedRecords,
+        crossScopeDuplicates: scopeReport.crossScopeDuplicates.length,
+        autoCuration: Boolean(capabilities.autoCuration || capabilities.dedupe || capabilities.autoCurationLayer),
+      },
+      recommendation: 'Run dedupe, contradiction, stale-memory, and scope-isolation checks before memories can become gates.',
+    },
+    {
+      id: 'context_layer',
+      ok: contextRecords.length > 0 || boolCapability(capabilities, 'contextPacks', 'contextLayer', 'scopedRetrieval'),
+      evidence: {
+        contextRecords: contextRecords.length,
+        scopedRetrieval: Boolean(capabilities.contextPacks || capabilities.contextLayer || capabilities.scopedRetrieval),
+      },
+      recommendation: 'Inject scoped context packs per task instead of loading every memory into the model window.',
+    },
+    {
+      id: 'interface_layer',
+      ok: boolCapability(capabilities, 'mcp', 'cli', 'hooks', 'dashboard', 'interfaceLayer'),
+      evidence: {
+        cli: Boolean(capabilities.cli),
+        mcp: Boolean(capabilities.mcp),
+        hooks: Boolean(capabilities.hooks),
+        dashboard: Boolean(capabilities.dashboard),
+      },
+      recommendation: 'Expose the same memory contract through CLI, MCP, hooks, dashboard, and agent adapters.',
+    },
+  ].map((check) => {
+    const layer = MEMORY_OS_LAYERS.find((candidate) => candidate.id === check.id);
+    return {
+      ...layer,
+      ...check,
+    };
+  });
+
+  const missingLayers = checks.filter((check) => !check.ok).map((check) => check.id);
+
+  return {
+    ready: missingLayers.length === 0,
+    riskLevel: missingLayers.length === 0 ? 'low' : missingLayers.length <= 2 ? 'medium' : 'high',
+    layers: checks,
+    missingLayers,
+    scopeReport,
+    recommendations: checks
+      .filter((check) => !check.ok)
+      .map((check) => check.recommendation),
+  };
+}
+
 function selectRecordsForScope(records = [], requestedScope = {}, options = {}) {
   const requested = normalizeScope(requestedScope);
   const requestedKey = memoryScopeKey(requested);
@@ -265,6 +419,7 @@ function buildMemoriStyleBenchmarkRecords() {
       projectId: 'thumbgate',
       processId: 'agent-a',
       sessionId: 'session-1',
+      metadata: { semanticKey: 'checkout-readiness', contextPackId: 'checkout-pro' },
       content: 'Use the paid sprint checklist before changing checkout code.',
     },
     {
@@ -298,14 +453,18 @@ function buildMemoriStyleBenchmarkRecords() {
       processId: 'agent-a',
       sessionId: 'session-1',
       visibility: 'shared',
+      type: 'fact',
+      factKey: 'checkout.mutation_policy',
       content: 'Shared rule: checkout mutations require audit evidence.',
     },
   ];
 }
 
 module.exports = {
+  MEMORY_OS_LAYERS,
   REQUIRED_SCOPE_FIELDS,
   buildMemoriStyleBenchmarkRecords,
+  buildMemoryOsLayerReport,
   buildMemoryScopeReadinessReport,
   isSharedMemory,
   memoryScopeKey,

package/scripts/meta-agent-loop.js

@@ -282,6 +282,8 @@ function buildPromotedGate(candidate, metrics, runId) {
     occurrences: metrics.hits,
     promotedAt: new Date().toISOString(),
     source: 'meta-agent',
+    // origin distinguishes silent-failure-clustered candidates from feedback-derived ones
+    origin: candidate.origin || 'user-feedback',
     runId,
     score: parseFloat(metrics.score.toFixed(3)),
     hitRate: parseFloat(metrics.hitRate.toFixed(3)),
@@ -371,6 +373,38 @@ async function runMetaAgentLoop({ dryRun = false, verbose = false } = {}) {
     candidates = generateCandidatesHeuristic(failures, blockPatterns);
   }
 
+  // Tag existing-pipeline candidates with their origin so downstream precision
+  // measurement (silentFailureDerivedGates vs user-feedback-derived) is possible.
+  candidates = candidates.map((c) => (c.origin ? c : { ...c, origin: 'user-feedback' }));
+
+  // Step 3b: Silent-failure clustering — DEFAULT-ON as of 2026-05-21
+  // (flipped from opt-in by PR #2289). Opt out via
+  // THUMBGATE_SILENT_FAILURE_CLUSTERING=0 (or NODE_ENV=test). Candidates flow
+  // through the SAME scoring / fp-rate eval below; we do not bypass any
+  // guardrail. The point of this clustering is to cover the case where users
+  // are lazy and never give thumbs-down — keeping it opt-in meant the users
+  // who need it most never got the benefit.
+  let silentFailureStats = null;
+  const { isSilentFailureClusteringEnabled, generateSilentFailureCandidates } = require('./silent-failure-cluster');
+  if (isSilentFailureClusteringEnabled()) {
+    try {
+      const sfResult = generateSilentFailureCandidates({ feedbackLogPath });
+      silentFailureStats = sfResult.stats;
+      if (sfResult.candidates && sfResult.candidates.length > 0) {
+        candidates = candidates.concat(sfResult.candidates);
+      }
+      if (verbose) {
+        process.stdout.write(
+          `[meta-agent] silent-failure-cluster: candidates=${sfResult.candidates.length} `
+          + `failed=${sfResult.stats.failedCalls} clusters=${sfResult.stats.clusters} `
+          + `skipped=${sfResult.stats.skippedReason || 'none'}\n`
+        );
+      }
+    } catch (err) {
+      if (verbose) process.stdout.write(`[meta-agent] silent-failure-cluster failed (non-fatal): ${err.message}\n`);
+    }
+  }
+
   if (verbose) {
     process.stdout.write(`[meta-agent] candidates generated: ${candidates.length} (mode=${analysisMode})\n`);
   }
@@ -507,6 +541,8 @@ async function runMetaAgentLoop({ dryRun = false, verbose = false } = {}) {
         skipped: evolutionResult.skipped || false,
       }
       : null,
+    silentFailureCluster: silentFailureStats,
+    silentFailureDerivedGates: promotedGates.filter((g) => g.origin === 'silent-failure-cluster').length,
   };
 
   if (!dryRun) {