thumbgate 1.12.0 → 1.12.2

package/scripts/durability/step.js

@@ -0,0 +1,171 @@
+'use strict';
+
+/**
+ * step.js — lightweight durable-step helper.
+ *
+ * Inspired by the "use step" pattern in Vercel Workflows, without adopting
+ * the full durable-execution runtime. Gives each external call (HTTP,
+ * LanceDB, LLM) a uniform retry + idempotency wrapper:
+ *
+ *   const result = await runStep('zernio.publishPost', {
+ *     retries: 3,
+ *     idempotencyKey: idempotencyKey(content, platforms),
+ *   }, async ({ attempt }) => {
+ *     return zernioFetch('POST', '/posts', body, { idempotencyKey: ... });
+ *   });
+ *
+ * Why a custom helper instead of Vercel Workflows / Temporal / Inngest?
+ *   - We run on Railway, not Vercel.
+ *   - SQLite + existing workflow tables already cover the durable state
+ *     we need; the gap is per-call retry/idempotency, not orchestration.
+ *   - A 60-line helper captures ~70% of the reliability benefit without
+ *     the platform migration or new ops surface.
+ *
+ * Error classification:
+ *   - Errors with `retryable: true` or a `code` in TRANSIENT_CODES retry.
+ *   - Errors with `nonRetryable: true` bail immediately.
+ *   - HTTP status (from `err.status` or parsed from message):
+ *       * 429 or 5xx → retry
+ *       * 4xx        → fail (no point retrying validation errors)
+ *   - Unknown errors → retry (capped by `retries` count — fail-open on
+ *     uncertainty, but bounded).
+ */
+
+const crypto = require('node:crypto');
+
+const TRANSIENT_CODES = new Set([
+  'ECONNRESET',
+  'ETIMEDOUT',
+  'ENOTFOUND',
+  'EAI_AGAIN',
+  'ECONNREFUSED',
+  'EPIPE',
+  'UND_ERR_SOCKET',
+  'UND_ERR_CONNECT_TIMEOUT',
+]);
+
+const DEFAULT_BACKOFF_MS = Object.freeze([250, 1000, 4000]);
+
+function defaultClassify(err) {
+  if (!err) return 'fail';
+  if (err.nonRetryable === true) return 'fail';
+  if (err.retryable === true) return 'retry';
+  if (err.code && TRANSIENT_CODES.has(err.code)) return 'retry';
+
+  // HTTP status from either an explicit prop or a parsed message.
+  const statusFromProp = Number.isFinite(err.status) ? err.status : null;
+  const msg = typeof err.message === 'string' ? err.message : '';
+  const match = /\b(5\d{2}|4\d{2})\b/.exec(msg);
+  const status = statusFromProp || (match ? Number(match[1]) : null);
+
+  if (status === 429) return 'retry';
+  if (status && status >= 500 && status < 600) return 'retry';
+  if (status && status >= 400 && status < 500) return 'fail';
+
+  // Unknown — retry cautiously. Bounded by the `retries` option.
+  return 'retry';
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => { setTimeout(resolve, ms); });
+}
+
+/**
+ * Build a stable 32-hex-char idempotency key from arbitrary inputs.
+ * Same inputs → same key. Safe to use as an Idempotency-Key HTTP header,
+ * a LanceDB row id, or a cache key for mid-flight deduplication.
+ *
+ * Usage:
+ *   idempotencyKey(content, platformList, scheduledFor)
+ */
+function idempotencyKey(...parts) {
+  const h = crypto.createHash('sha256');
+  for (const p of parts) {
+    if (p == null) {
+      h.update('');
+    } else if (typeof p === 'string') {
+      h.update(p);
+    } else {
+      h.update(JSON.stringify(p));
+    }
+    h.update('\0'); // field separator — prevents ["a","b"] colliding with ["ab"]
+  }
+  return h.digest('hex').slice(0, 32);
+}
+
+/**
+ * Execute `fn` with retry + backoff + classification. Returns the value
+ * `fn` resolves to, or throws the last error after exhausting retries /
+ * hitting a non-retryable verdict.
+ *
+ * @param {string} name          Step name, used in logs. e.g. 'zernio.publishPost'.
+ * @param {object|function} options  { retries, backoffMs, classify, onRetry, onFail, logger }
+ *                                   (may be passed directly as the callback shorthand)
+ * @param {function({attempt:number}):Promise} fn  The actual work.
+ */
+function errMessage(err) {
+  return err?.message ?? err;
+}
+
+function handleStepError({ err, attempt, retries, classify, backoffMs, name, onRetry, onFail, logger }) {
+  const verdict = classify(err);
+  const terminal = verdict === 'fail' || attempt >= retries;
+  if (terminal) {
+    if (typeof onFail === 'function') onFail({ name, attempt, err, verdict });
+    if (typeof logger === 'function') {
+      logger(`[step:${name}] FAIL attempt=${attempt} verdict=${verdict} err=${errMessage(err)}`);
+    }
+    return { terminal: true };
+  }
+  const waitMs = backoffMs[Math.min(attempt, backoffMs.length - 1)];
+  if (typeof onRetry === 'function') onRetry({ name, attempt, err, waitMs, verdict });
+  if (typeof logger === 'function') {
+    logger(`[step:${name}] RETRY attempt=${attempt} waitMs=${waitMs} err=${errMessage(err)}`);
+  }
+  return { terminal: false, waitMs };
+}
+
+async function runStep(name, options, fn) {
+  if (typeof options === 'function') {
+    fn = options;
+    options = {};
+  }
+  const {
+    retries = 3,
+    backoffMs = DEFAULT_BACKOFF_MS,
+    classify = defaultClassify,
+    onAttempt,
+    onRetry,
+    onFail,
+    logger,
+    sleepFn = sleep,
+  } = options || {};
+
+  if (typeof fn !== 'function') {
+    throw new TypeError(`runStep(${name}): fn must be a function`);
+  }
+
+  let lastErr;
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    if (typeof onAttempt === 'function') onAttempt({ name, attempt });
+    try {
+      return await fn({ attempt });
+    } catch (err) {
+      lastErr = err;
+      const outcome = handleStepError({
+        err, attempt, retries, classify, backoffMs, name, onRetry, onFail, logger,
+      });
+      if (outcome.terminal) throw err;
+      await sleepFn(outcome.waitMs);
+    }
+  }
+  throw lastErr;
+}
+
+module.exports = {
+  runStep,
+  idempotencyKey,
+  defaultClassify,
+  TRANSIENT_CODES,
+  DEFAULT_BACKOFF_MS,
+};

package/scripts/gates-engine.js

@@ -83,6 +83,7 @@ const DEFAULT_PROTECTED_FILE_GLOBS = [
 ];
 const EDIT_LIKE_TOOLS = new Set(['Edit', 'Write', 'MultiEdit']);
 const HIGH_RISK_BASH_PATTERN = /\b(?:git\s+(?:add|commit|push)|gh\s+pr\s+(?:create|merge)|npm\s+publish|yarn\s+publish|pnpm\s+publish|rm\s+-rf)\b/i;
+const REMOTE_SIDE_EFFECT_BASH_PATTERN = /\b(?:git\s+push\b|gh\s+pr\s+(?:create|merge|close|reopen|ready|edit)\b|gh\s+release\s+(?:create|delete|edit|upload)\b|npm\s+publish\b|yarn\s+publish\b|pnpm\s+publish\b)\b/i;
 const BOOSTED_RISK_BLOCK_SCORE = 0.8;
 const BOOSTED_RISK_MIN_EXAMPLES = 3;
 const PR_THREAD_RESOLUTION_ACTION = 'pr_thread_resolution_verified_after_commit';
@@ -826,6 +827,59 @@ function evaluatePendingPrThreadResolutionGate(toolName, toolInput = {}) {
   };
 }
 
+function getLocalOnlyScopeSources(governanceState = {}, constraints = {}) {
+  const sources = [];
+  if (governanceState.taskScope && governanceState.taskScope.localOnly) {
+    sources.push('task scope');
+  }
+  if (governanceState.branchGovernance && governanceState.branchGovernance.localOnly) {
+    sources.push('branch governance');
+  }
+  if (constraints.local_only && constraints.local_only.value === true) {
+    sources.push('local_only constraint');
+  }
+  return sources;
+}
+
+function isRemoteSideEffectCommand(toolName, toolInput = {}) {
+  if (toolName !== 'Bash') return false;
+  return REMOTE_SIDE_EFFECT_BASH_PATTERN.test(String(toolInput.command || ''));
+}
+
+function evaluateLocalOnlyRemoteSideEffectGate(toolName, toolInput = {}, governanceState = {}, constraints = {}) {
+  if (!isRemoteSideEffectCommand(toolName, toolInput)) return null;
+  const sources = getLocalOnlyScopeSources(governanceState, constraints);
+  if (sources.length === 0) return null;
+
+  const command = String(toolInput.command || '').trim();
+  return {
+    decision: 'deny',
+    gate: 'local-only-remote-side-effect',
+    message: 'Task scope is local-only; remote git, PR, release, and publish actions are blocked until the local-only scope is cleared or explicitly changed.',
+    severity: 'critical',
+    reasoning: [
+      `Local-only source: ${sources.join(', ')}`,
+      `Blocked command: ${command.slice(0, 160)}`,
+      'Remote side effects are denied before configurable gates so wrapped commands cannot bypass local-only work boundaries',
+    ],
+  };
+}
+
+function recordStructuralGateBlock(toolName, toolInput, result) {
+  recordStat(result.gate, 'block');
+  const auditRecord = recordAuditEvent({
+    toolName,
+    toolInput,
+    decision: 'deny',
+    gateId: result.gate,
+    message: result.message,
+    severity: result.severity,
+    source: 'gates-engine',
+  });
+  auditToFeedback(auditRecord);
+  return result;
+}
+
 function isScopeEnforcedAction(toolName, toolInput = {}, affectedFiles = []) {
   if (EDIT_LIKE_TOOLS.has(toolName) && affectedFiles.length > 0) return true;
   if (toolName !== 'Bash') return false;
@@ -1333,7 +1387,18 @@ async function evaluateGatesAsync(toolName, toolInput, configPath) {
   }
 
   const constraints = loadConstraints();
+  const governanceState = loadGovernanceState();
   registerPrThreadResolutionClaimGate(toolName, toolInput);
+  const localOnlyRemoteSideEffectGate = evaluateLocalOnlyRemoteSideEffectGate(
+    toolName,
+    toolInput,
+    governanceState,
+    constraints,
+  );
+  if (localOnlyRemoteSideEffectGate) {
+    return recordStructuralGateBlock(toolName, toolInput, localOnlyRemoteSideEffectGate);
+  }
+
   const pendingThreadResolutionGate = evaluatePendingPrThreadResolutionGate(toolName, toolInput);
   if (pendingThreadResolutionGate) {
     recordStat(pendingThreadResolutionGate.gate, 'block');
@@ -1445,7 +1510,7 @@ async function evaluateGatesAsync(toolName, toolInput, configPath) {
   }
 
   const sentinelReport = evaluateWorkflowSentinel(toolName, toolInput, {
-    governanceState: loadGovernanceState(),
+    governanceState,
   });
   const sentinelDecision = recordSentinelDecision(sentinelReport, toolName, toolInput);
   const memoryGuard = evaluateMemoryGuard(toolName, toolInput);
@@ -1503,7 +1568,18 @@ function evaluateGates(toolName, toolInput, configPath) {
   }
 
   const constraints = loadConstraints();
+  const governanceState = loadGovernanceState();
   registerPrThreadResolutionClaimGate(toolName, toolInput);
+  const localOnlyRemoteSideEffectGate = evaluateLocalOnlyRemoteSideEffectGate(
+    toolName,
+    toolInput,
+    governanceState,
+    constraints,
+  );
+  if (localOnlyRemoteSideEffectGate) {
+    return recordStructuralGateBlock(toolName, toolInput, localOnlyRemoteSideEffectGate);
+  }
+
   const pendingThreadResolutionGate = evaluatePendingPrThreadResolutionGate(toolName, toolInput);
   if (pendingThreadResolutionGate) {
     recordStat(pendingThreadResolutionGate.gate, 'block');
@@ -1587,7 +1663,7 @@ function evaluateGates(toolName, toolInput, configPath) {
   }
 
   const sentinelReport = evaluateWorkflowSentinel(toolName, toolInput, {
-    governanceState: loadGovernanceState(),
+    governanceState,
   });
   const sentinelDecision = recordSentinelDecision(sentinelReport, toolName, toolInput);
   const memoryGuard = evaluateMemoryGuard(toolName, toolInput);
@@ -2250,6 +2326,9 @@ module.exports = {
   evaluateBoostedRiskTagGuard,
   registerPrThreadResolutionClaimGate,
   evaluatePendingPrThreadResolutionGate,
+  getLocalOnlyScopeSources,
+  isRemoteSideEffectCommand,
+  evaluateLocalOnlyRemoteSideEffectGate,
   PR_THREAD_RESOLUTION_ACTION,
 };
 

package/scripts/hf-papers.js

@@ -0,0 +1,317 @@
+'use strict';
+
+const { URL, URLSearchParams } = require('node:url');
+const {
+  NAMESPACES,
+  upsertContextObject,
+  recordProvenance,
+  constructTemplatedPack,
+} = require('./contextfs');
+
+const DEFAULT_HF_PAPERS_API_BASE = process.env.HF_PAPERS_API_BASE || 'https://huggingface.co/api';
+const DEFAULT_LIMIT = 5;
+
+function normalizeAuthors(authors) {
+  if (!Array.isArray(authors)) return [];
+  return authors
+    .map((author) => {
+      if (typeof author === 'string') return author.trim();
+      if (author && typeof author.name === 'string') return author.name.trim();
+      return '';
+    })
+    .filter(Boolean);
+}
+
+function normalizeTags(tags) {
+  if (!Array.isArray(tags)) return [];
+  return [...new Set(tags
+    .map((tag) => {
+      if (typeof tag === 'string') return tag.trim();
+      if (tag && typeof tag.label === 'string') return tag.label.trim();
+      if (tag && typeof tag.name === 'string') return tag.name.trim();
+      return '';
+    })
+    .filter(Boolean))];
+}
+
+function normalizePaper(record = {}) {
+  const paper = record && typeof record.paper === 'object' ? record.paper : record;
+  const paperId = String(
+    paper.id
+      || paper.paper_id
+      || paper.paperId
+      || paper.arxiv_id
+      || paper.arxivId
+      || record.id
+      || record.paper_id
+      || record.paperId
+      || record.arxiv_id
+      || record.arxivId
+      || ''
+  ).trim();
+  const title = String(
+    paper.title
+      || record.title
+      || (paperId ? `Paper ${paperId}` : 'Untitled paper')
+  ).trim();
+  const summary = String(
+    paper.summary
+      || paper.abstract
+      || record.summary
+      || record.abstract
+      || ''
+  ).trim();
+  const url = String(
+    paper.url
+      || paper.paper_url
+      || record.url
+      || record.paper_url
+      || (paperId ? `https://arxiv.org/abs/${paperId}` : '')
+  ).trim();
+
+  return {
+    paperId,
+    title,
+    summary,
+    url: url || null,
+    authors: normalizeAuthors(paper.authors || record.authors),
+    tags: normalizeTags(paper.tags || paper.categories || record.tags || record.categories),
+    publishedAt: paper.publishedAt || paper.published_at || record.publishedAt || record.published_at || null,
+    source: 'huggingface-papers',
+  };
+}
+
+function extractPaperItems(payload) {
+  if (Array.isArray(payload)) return payload;
+  if (!payload || typeof payload !== 'object') return [];
+  if (Array.isArray(payload.papers)) return payload.papers;
+  if (Array.isArray(payload.items)) return payload.items;
+  if (Array.isArray(payload.results)) return payload.results;
+  if (Array.isArray(payload.dailyPapers)) return payload.dailyPapers;
+  if (payload.paper && typeof payload.paper === 'object') return [payload.paper];
+  return [];
+}
+
+function buildSearchUrls({ query, limit = DEFAULT_LIMIT, baseUrl = DEFAULT_HF_PAPERS_API_BASE }) {
+  const normalizedBase = String(baseUrl || DEFAULT_HF_PAPERS_API_BASE).replace(/\/+$/, '');
+  const routes = [
+    ['/daily_papers', { query, limit: String(limit) }],
+    ['/papers/search', { q: query, limit: String(limit) }],
+    ['/papers', { query, limit: String(limit) }],
+  ];
+
+  return routes.map(([pathname, params]) => {
+    const url = new URL(`${normalizedBase}${pathname}`);
+    url.search = new URLSearchParams(params).toString();
+    return url.toString();
+  });
+}
+
+async function readJson(url, fetchImpl = global.fetch) {
+  if (typeof fetchImpl !== 'function') {
+    throw new Error('A fetch implementation is required');
+  }
+
+  const response = await fetchImpl(url, {
+    headers: {
+      accept: 'application/json',
+    },
+  });
+
+  if (!response.ok) {
+    const error = new Error(`HF papers request failed: ${response.status} ${response.statusText}`);
+    error.status = response.status;
+    throw error;
+  }
+
+  return response.json();
+}
+
+async function searchPapers({
+  query,
+  limit = DEFAULT_LIMIT,
+  baseUrl = DEFAULT_HF_PAPERS_API_BASE,
+  fetchImpl = global.fetch,
+} = {}) {
+  const normalizedQuery = String(query || '').trim();
+  if (!normalizedQuery) {
+    throw new Error('searchPapers requires query');
+  }
+
+  const urls = buildSearchUrls({
+    query: normalizedQuery,
+    limit: Math.max(1, Number(limit) || DEFAULT_LIMIT),
+    baseUrl,
+  });
+
+  let lastError = null;
+  for (const url of urls) {
+    try {
+      const payload = await readJson(url, fetchImpl);
+      const papers = extractPaperItems(payload)
+        .map(normalizePaper)
+        .filter((paper) => paper.paperId || paper.title);
+
+      if (papers.length > 0) {
+        return papers.slice(0, limit);
+      }
+    } catch (error) {
+      lastError = error;
+      if (error && error.status === 404) {
+        continue;
+      }
+    }
+  }
+
+  if (lastError) throw lastError;
+  return [];
+}
+
+function paperToMarkdown(paper) {
+  const normalized = normalizePaper(paper);
+  const lines = [
+    `# ${normalized.title}`,
+    '',
+    `Paper ID: ${normalized.paperId || 'unknown'}`,
+    `Source: ${normalized.source}`,
+  ];
+
+  if (normalized.url) {
+    lines.push(`URL: ${normalized.url}`);
+  }
+  if (normalized.publishedAt) {
+    lines.push(`Published: ${normalized.publishedAt}`);
+  }
+  if (normalized.authors.length > 0) {
+    lines.push(`Authors: ${normalized.authors.join(', ')}`);
+  }
+  if (normalized.tags.length > 0) {
+    lines.push(`Tags: ${normalized.tags.join(', ')}`);
+  }
+
+  lines.push('', '## Abstract', '', normalized.summary || 'No abstract available.', '');
+  return lines.join('\n');
+}
+
+function buildCitation(paper) {
+  return {
+    paperId: paper.paperId || null,
+    title: paper.title,
+    url: paper.url,
+  };
+}
+
+function ingestNormalizedPapers(papers, query) {
+  const ingested = papers.map((paper) => {
+    const normalizedTags = [...new Set([
+      'research',
+      'paper',
+      'hf-papers',
+      ...paper.tags.map((tag) => String(tag)),
+    ])].sort();
+
+    return upsertContextObject({
+      namespace: NAMESPACES.research,
+      title: `Paper: ${paper.title}`,
+      content: paperToMarkdown(paper),
+      tags: normalizedTags,
+      source: 'hf-papers',
+      metadata: {
+        provider: 'huggingface',
+        paperId: paper.paperId || null,
+        url: paper.url,
+        authors: paper.authors,
+        publishedAt: paper.publishedAt,
+        query,
+      },
+    });
+  });
+
+  recordProvenance({
+    type: 'hf_papers_ingested',
+    query,
+    count: ingested.length,
+    dedupedCount: ingested.filter((entry) => entry.deduped).length,
+    paperIds: papers.map((paper) => paper.paperId).filter(Boolean),
+  });
+
+  return ingested;
+}
+
+async function ingestPaperSearch({
+  query,
+  limit = DEFAULT_LIMIT,
+  baseUrl = DEFAULT_HF_PAPERS_API_BASE,
+  fetchImpl = global.fetch,
+  searchPapersImpl = searchPapers,
+} = {}) {
+  const papers = await searchPapersImpl({
+    query,
+    limit,
+    baseUrl,
+    fetchImpl,
+  });
+  const ingested = ingestNormalizedPapers(papers, query);
+
+  return {
+    query,
+    limit,
+    papers,
+    ingested,
+  };
+}
+
+async function buildResearchBrief({
+  query,
+  limit = DEFAULT_LIMIT,
+  template = 'research-brief',
+  baseUrl = DEFAULT_HF_PAPERS_API_BASE,
+  fetchImpl = global.fetch,
+  searchPapersImpl = searchPapers,
+} = {}) {
+  const result = await ingestPaperSearch({
+    query,
+    limit,
+    baseUrl,
+    fetchImpl,
+    searchPapersImpl,
+  });
+  const pack = constructTemplatedPack({ template, query });
+  const citations = result.papers.map(buildCitation);
+  const brief = pack.items
+    .map((item, index) => {
+      const digest = String(item.structuredContext && item.structuredContext.rawContent || '')
+        .split('\n')
+        .slice(0, 6)
+        .join(' ')
+        .trim();
+      return `${index + 1}. ${item.title} ${digest}`.trim();
+    })
+    .join('\n');
+
+  return {
+    query,
+    limit,
+    source: 'huggingface-papers',
+    template,
+    ingestedCount: result.ingested.length,
+    packId: pack.packId,
+    citations,
+    brief,
+    pack,
+  };
+}
+
+module.exports = {
+  DEFAULT_HF_PAPERS_API_BASE,
+  buildResearchBrief,
+  buildSearchUrls,
+  extractPaperItems,
+  ingestNormalizedPapers,
+  ingestPaperSearch,
+  normalizeAuthors,
+  normalizePaper,
+  normalizeTags,
+  paperToMarkdown,
+  searchPapers,
+};

package/scripts/mcp-config.js

@@ -189,13 +189,13 @@ function publishedCliAvailable(pkgVersion) {
 
 function resolveMcpEntry({ pkgRoot, pkgVersion, scope = 'project', targetDir = pkgRoot }) {
   if (!isSourceCheckout(pkgRoot)) {
-    return portableMcpEntry(pkgVersion);
+    return codexAutoUpdateMcpEntry();
   }
   if (scope === 'home' && publishedCliAvailable(pkgVersion)) {
-    return portableMcpEntry(pkgVersion);
+    return codexAutoUpdateMcpEntry();
   }
   if (scope === 'project' && !isSameCheckoutFamily(pkgRoot, targetDir) && publishedCliAvailable(pkgVersion)) {
-    return portableMcpEntry(pkgVersion);
+    return codexAutoUpdateMcpEntry();
   }
   return localMcpEntry(pkgRoot, scope);
 }