thumbgate 1.12.1 → 1.13.0

package/scripts/durability/step.js

@@ -0,0 +1,171 @@
+'use strict';
+
+/**
+ * step.js — lightweight durable-step helper.
+ *
+ * Inspired by the "use step" pattern in Vercel Workflows, without adopting
+ * the full durable-execution runtime. Gives each external call (HTTP,
+ * LanceDB, LLM) a uniform retry + idempotency wrapper:
+ *
+ *   const result = await runStep('zernio.publishPost', {
+ *     retries: 3,
+ *     idempotencyKey: idempotencyKey(content, platforms),
+ *   }, async ({ attempt }) => {
+ *     return zernioFetch('POST', '/posts', body, { idempotencyKey: ... });
+ *   });
+ *
+ * Why a custom helper instead of Vercel Workflows / Temporal / Inngest?
+ *   - We run on Railway, not Vercel.
+ *   - SQLite + existing workflow tables already cover the durable state
+ *     we need; the gap is per-call retry/idempotency, not orchestration.
+ *   - A 60-line helper captures ~70% of the reliability benefit without
+ *     the platform migration or new ops surface.
+ *
+ * Error classification:
+ *   - Errors with `retryable: true` or a `code` in TRANSIENT_CODES retry.
+ *   - Errors with `nonRetryable: true` bail immediately.
+ *   - HTTP status (from `err.status` or parsed from message):
+ *       * 429 or 5xx → retry
+ *       * 4xx        → fail (no point retrying validation errors)
+ *   - Unknown errors → retry (capped by `retries` count — fail-open on
+ *     uncertainty, but bounded).
+ */
+
+const crypto = require('node:crypto');
+
+const TRANSIENT_CODES = new Set([
+  'ECONNRESET',
+  'ETIMEDOUT',
+  'ENOTFOUND',
+  'EAI_AGAIN',
+  'ECONNREFUSED',
+  'EPIPE',
+  'UND_ERR_SOCKET',
+  'UND_ERR_CONNECT_TIMEOUT',
+]);
+
+const DEFAULT_BACKOFF_MS = Object.freeze([250, 1000, 4000]);
+
+function defaultClassify(err) {
+  if (!err) return 'fail';
+  if (err.nonRetryable === true) return 'fail';
+  if (err.retryable === true) return 'retry';
+  if (err.code && TRANSIENT_CODES.has(err.code)) return 'retry';
+
+  // HTTP status from either an explicit prop or a parsed message.
+  const statusFromProp = Number.isFinite(err.status) ? err.status : null;
+  const msg = typeof err.message === 'string' ? err.message : '';
+  const match = /\b(5\d{2}|4\d{2})\b/.exec(msg);
+  const status = statusFromProp || (match ? Number(match[1]) : null);
+
+  if (status === 429) return 'retry';
+  if (status && status >= 500 && status < 600) return 'retry';
+  if (status && status >= 400 && status < 500) return 'fail';
+
+  // Unknown — retry cautiously. Bounded by the `retries` option.
+  return 'retry';
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => { setTimeout(resolve, ms); });
+}
+
+/**
+ * Build a stable 32-hex-char idempotency key from arbitrary inputs.
+ * Same inputs → same key. Safe to use as an Idempotency-Key HTTP header,
+ * a LanceDB row id, or a cache key for mid-flight deduplication.
+ *
+ * Usage:
+ *   idempotencyKey(content, platformList, scheduledFor)
+ */
+function idempotencyKey(...parts) {
+  const h = crypto.createHash('sha256');
+  for (const p of parts) {
+    if (p == null) {
+      h.update('');
+    } else if (typeof p === 'string') {
+      h.update(p);
+    } else {
+      h.update(JSON.stringify(p));
+    }
+    h.update('\0'); // field separator — prevents ["a","b"] colliding with ["ab"]
+  }
+  return h.digest('hex').slice(0, 32);
+}
+
+/**
+ * Execute `fn` with retry + backoff + classification. Returns the value
+ * `fn` resolves to, or throws the last error after exhausting retries /
+ * hitting a non-retryable verdict.
+ *
+ * @param {string} name          Step name, used in logs. e.g. 'zernio.publishPost'.
+ * @param {object|function} options  { retries, backoffMs, classify, onRetry, onFail, logger }
+ *                                   (may be passed directly as the callback shorthand)
+ * @param {function({attempt:number}):Promise} fn  The actual work.
+ */
+function errMessage(err) {
+  return err?.message ?? err;
+}
+
+function handleStepError({ err, attempt, retries, classify, backoffMs, name, onRetry, onFail, logger }) {
+  const verdict = classify(err);
+  const terminal = verdict === 'fail' || attempt >= retries;
+  if (terminal) {
+    if (typeof onFail === 'function') onFail({ name, attempt, err, verdict });
+    if (typeof logger === 'function') {
+      logger(`[step:${name}] FAIL attempt=${attempt} verdict=${verdict} err=${errMessage(err)}`);
+    }
+    return { terminal: true };
+  }
+  const waitMs = backoffMs[Math.min(attempt, backoffMs.length - 1)];
+  if (typeof onRetry === 'function') onRetry({ name, attempt, err, waitMs, verdict });
+  if (typeof logger === 'function') {
+    logger(`[step:${name}] RETRY attempt=${attempt} waitMs=${waitMs} err=${errMessage(err)}`);
+  }
+  return { terminal: false, waitMs };
+}
+
+async function runStep(name, options, fn) {
+  if (typeof options === 'function') {
+    fn = options;
+    options = {};
+  }
+  const {
+    retries = 3,
+    backoffMs = DEFAULT_BACKOFF_MS,
+    classify = defaultClassify,
+    onAttempt,
+    onRetry,
+    onFail,
+    logger,
+    sleepFn = sleep,
+  } = options || {};
+
+  if (typeof fn !== 'function') {
+    throw new TypeError(`runStep(${name}): fn must be a function`);
+  }
+
+  let lastErr;
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    if (typeof onAttempt === 'function') onAttempt({ name, attempt });
+    try {
+      return await fn({ attempt });
+    } catch (err) {
+      lastErr = err;
+      const outcome = handleStepError({
+        err, attempt, retries, classify, backoffMs, name, onRetry, onFail, logger,
+      });
+      if (outcome.terminal) throw err;
+      await sleepFn(outcome.waitMs);
+    }
+  }
+  throw lastErr;
+}
+
+module.exports = {
+  runStep,
+  idempotencyKey,
+  defaultClassify,
+  TRANSIENT_CODES,
+  DEFAULT_BACKOFF_MS,
+};

package/scripts/hf-papers.js

@@ -0,0 +1,317 @@
+'use strict';
+
+const { URL, URLSearchParams } = require('node:url');
+const {
+  NAMESPACES,
+  upsertContextObject,
+  recordProvenance,
+  constructTemplatedPack,
+} = require('./contextfs');
+
+const DEFAULT_HF_PAPERS_API_BASE = process.env.HF_PAPERS_API_BASE || 'https://huggingface.co/api';
+const DEFAULT_LIMIT = 5;
+
+function normalizeAuthors(authors) {
+  if (!Array.isArray(authors)) return [];
+  return authors
+    .map((author) => {
+      if (typeof author === 'string') return author.trim();
+      if (author && typeof author.name === 'string') return author.name.trim();
+      return '';
+    })
+    .filter(Boolean);
+}
+
+function normalizeTags(tags) {
+  if (!Array.isArray(tags)) return [];
+  return [...new Set(tags
+    .map((tag) => {
+      if (typeof tag === 'string') return tag.trim();
+      if (tag && typeof tag.label === 'string') return tag.label.trim();
+      if (tag && typeof tag.name === 'string') return tag.name.trim();
+      return '';
+    })
+    .filter(Boolean))];
+}
+
+function normalizePaper(record = {}) {
+  const paper = record && typeof record.paper === 'object' ? record.paper : record;
+  const paperId = String(
+    paper.id
+      || paper.paper_id
+      || paper.paperId
+      || paper.arxiv_id
+      || paper.arxivId
+      || record.id
+      || record.paper_id
+      || record.paperId
+      || record.arxiv_id
+      || record.arxivId
+      || ''
+  ).trim();
+  const title = String(
+    paper.title
+      || record.title
+      || (paperId ? `Paper ${paperId}` : 'Untitled paper')
+  ).trim();
+  const summary = String(
+    paper.summary
+      || paper.abstract
+      || record.summary
+      || record.abstract
+      || ''
+  ).trim();
+  const url = String(
+    paper.url
+      || paper.paper_url
+      || record.url
+      || record.paper_url
+      || (paperId ? `https://arxiv.org/abs/${paperId}` : '')
+  ).trim();
+
+  return {
+    paperId,
+    title,
+    summary,
+    url: url || null,
+    authors: normalizeAuthors(paper.authors || record.authors),
+    tags: normalizeTags(paper.tags || paper.categories || record.tags || record.categories),
+    publishedAt: paper.publishedAt || paper.published_at || record.publishedAt || record.published_at || null,
+    source: 'huggingface-papers',
+  };
+}
+
+function extractPaperItems(payload) {
+  if (Array.isArray(payload)) return payload;
+  if (!payload || typeof payload !== 'object') return [];
+  if (Array.isArray(payload.papers)) return payload.papers;
+  if (Array.isArray(payload.items)) return payload.items;
+  if (Array.isArray(payload.results)) return payload.results;
+  if (Array.isArray(payload.dailyPapers)) return payload.dailyPapers;
+  if (payload.paper && typeof payload.paper === 'object') return [payload.paper];
+  return [];
+}
+
+function buildSearchUrls({ query, limit = DEFAULT_LIMIT, baseUrl = DEFAULT_HF_PAPERS_API_BASE }) {
+  const normalizedBase = String(baseUrl || DEFAULT_HF_PAPERS_API_BASE).replace(/\/+$/, '');
+  const routes = [
+    ['/daily_papers', { query, limit: String(limit) }],
+    ['/papers/search', { q: query, limit: String(limit) }],
+    ['/papers', { query, limit: String(limit) }],
+  ];
+
+  return routes.map(([pathname, params]) => {
+    const url = new URL(`${normalizedBase}${pathname}`);
+    url.search = new URLSearchParams(params).toString();
+    return url.toString();
+  });
+}
+
+async function readJson(url, fetchImpl = global.fetch) {
+  if (typeof fetchImpl !== 'function') {
+    throw new Error('A fetch implementation is required');
+  }
+
+  const response = await fetchImpl(url, {
+    headers: {
+      accept: 'application/json',
+    },
+  });
+
+  if (!response.ok) {
+    const error = new Error(`HF papers request failed: ${response.status} ${response.statusText}`);
+    error.status = response.status;
+    throw error;
+  }
+
+  return response.json();
+}
+
+async function searchPapers({
+  query,
+  limit = DEFAULT_LIMIT,
+  baseUrl = DEFAULT_HF_PAPERS_API_BASE,
+  fetchImpl = global.fetch,
+} = {}) {
+  const normalizedQuery = String(query || '').trim();
+  if (!normalizedQuery) {
+    throw new Error('searchPapers requires query');
+  }
+
+  const urls = buildSearchUrls({
+    query: normalizedQuery,
+    limit: Math.max(1, Number(limit) || DEFAULT_LIMIT),
+    baseUrl,
+  });
+
+  let lastError = null;
+  for (const url of urls) {
+    try {
+      const payload = await readJson(url, fetchImpl);
+      const papers = extractPaperItems(payload)
+        .map(normalizePaper)
+        .filter((paper) => paper.paperId || paper.title);
+
+      if (papers.length > 0) {
+        return papers.slice(0, limit);
+      }
+    } catch (error) {
+      lastError = error;
+      if (error && error.status === 404) {
+        continue;
+      }
+    }
+  }
+
+  if (lastError) throw lastError;
+  return [];
+}
+
+function paperToMarkdown(paper) {
+  const normalized = normalizePaper(paper);
+  const lines = [
+    `# ${normalized.title}`,
+    '',
+    `Paper ID: ${normalized.paperId || 'unknown'}`,
+    `Source: ${normalized.source}`,
+  ];
+
+  if (normalized.url) {
+    lines.push(`URL: ${normalized.url}`);
+  }
+  if (normalized.publishedAt) {
+    lines.push(`Published: ${normalized.publishedAt}`);
+  }
+  if (normalized.authors.length > 0) {
+    lines.push(`Authors: ${normalized.authors.join(', ')}`);
+  }
+  if (normalized.tags.length > 0) {
+    lines.push(`Tags: ${normalized.tags.join(', ')}`);
+  }
+
+  lines.push('', '## Abstract', '', normalized.summary || 'No abstract available.', '');
+  return lines.join('\n');
+}
+
+function buildCitation(paper) {
+  return {
+    paperId: paper.paperId || null,
+    title: paper.title,
+    url: paper.url,
+  };
+}
+
+function ingestNormalizedPapers(papers, query) {
+  const ingested = papers.map((paper) => {
+    const normalizedTags = [...new Set([
+      'research',
+      'paper',
+      'hf-papers',
+      ...paper.tags.map((tag) => String(tag)),
+    ])].sort();
+
+    return upsertContextObject({
+      namespace: NAMESPACES.research,
+      title: `Paper: ${paper.title}`,
+      content: paperToMarkdown(paper),
+      tags: normalizedTags,
+      source: 'hf-papers',
+      metadata: {
+        provider: 'huggingface',
+        paperId: paper.paperId || null,
+        url: paper.url,
+        authors: paper.authors,
+        publishedAt: paper.publishedAt,
+        query,
+      },
+    });
+  });
+
+  recordProvenance({
+    type: 'hf_papers_ingested',
+    query,
+    count: ingested.length,
+    dedupedCount: ingested.filter((entry) => entry.deduped).length,
+    paperIds: papers.map((paper) => paper.paperId).filter(Boolean),
+  });
+
+  return ingested;
+}
+
+async function ingestPaperSearch({
+  query,
+  limit = DEFAULT_LIMIT,
+  baseUrl = DEFAULT_HF_PAPERS_API_BASE,
+  fetchImpl = global.fetch,
+  searchPapersImpl = searchPapers,
+} = {}) {
+  const papers = await searchPapersImpl({
+    query,
+    limit,
+    baseUrl,
+    fetchImpl,
+  });
+  const ingested = ingestNormalizedPapers(papers, query);
+
+  return {
+    query,
+    limit,
+    papers,
+    ingested,
+  };
+}
+
+async function buildResearchBrief({
+  query,
+  limit = DEFAULT_LIMIT,
+  template = 'research-brief',
+  baseUrl = DEFAULT_HF_PAPERS_API_BASE,
+  fetchImpl = global.fetch,
+  searchPapersImpl = searchPapers,
+} = {}) {
+  const result = await ingestPaperSearch({
+    query,
+    limit,
+    baseUrl,
+    fetchImpl,
+    searchPapersImpl,
+  });
+  const pack = constructTemplatedPack({ template, query });
+  const citations = result.papers.map(buildCitation);
+  const brief = pack.items
+    .map((item, index) => {
+      const digest = String(item.structuredContext && item.structuredContext.rawContent || '')
+        .split('\n')
+        .slice(0, 6)
+        .join(' ')
+        .trim();
+      return `${index + 1}. ${item.title} ${digest}`.trim();
+    })
+    .join('\n');
+
+  return {
+    query,
+    limit,
+    source: 'huggingface-papers',
+    template,
+    ingestedCount: result.ingested.length,
+    packId: pack.packId,
+    citations,
+    brief,
+    pack,
+  };
+}
+
+module.exports = {
+  DEFAULT_HF_PAPERS_API_BASE,
+  buildResearchBrief,
+  buildSearchUrls,
+  extractPaperItems,
+  ingestNormalizedPapers,
+  ingestPaperSearch,
+  normalizeAuthors,
+  normalizePaper,
+  normalizeTags,
+  paperToMarkdown,
+  searchPapers,
+};

package/scripts/session-report.js

@@ -0,0 +1,120 @@
+'use strict';
+
+const MIN_WINDOW_HOURS = 1;
+const MAX_WINDOW_HOURS = 24 * 30;
+const DEFAULT_WINDOW_HOURS = 24;
+
+function normalizeWindowHours(input) {
+  if (input === null || input === undefined || input === '') return DEFAULT_WINDOW_HOURS;
+  const n = Number(input);
+  if (!Number.isFinite(n)) return DEFAULT_WINDOW_HOURS;
+  if (n < MIN_WINDOW_HOURS) return MIN_WINDOW_HOURS;
+  if (n > MAX_WINDOW_HOURS) return MAX_WINDOW_HOURS;
+  return Math.floor(n);
+}
+
+function topNegativeTags(tags, limit = 5) {
+  if (!tags || typeof tags !== 'object') return [];
+  return Object.entries(tags)
+    .map(([tag, counts]) => ({
+      tag,
+      negative: (counts && counts.negative) || 0,
+      positive: (counts && counts.positive) || 0,
+      total: (counts && counts.total) || 0,
+    }))
+    .filter((row) => row.negative > 0)
+    .sort((a, b) => b.negative - a.negative)
+    .slice(0, limit);
+}
+
+function topGates(byGate, limit = 5) {
+  if (!byGate || typeof byGate !== 'object') return [];
+  return Object.entries(byGate)
+    .map(([gate, counts]) => ({
+      gate,
+      blocked: (counts && counts.blocked) || 0,
+      warned: (counts && counts.warned) || 0,
+      pendingApproval: (counts && counts.pendingApproval) || 0,
+    }))
+    .sort((a, b) => b.blocked - a.blocked || b.warned - a.warned)
+    .slice(0, limit);
+}
+
+function summarizeProvenance(events, sinceMs) {
+  if (!Array.isArray(events)) return { total: 0, byType: {} };
+  const byType = {};
+  let total = 0;
+  for (const evt of events) {
+    const ts = Date.parse(evt && evt.timestamp ? evt.timestamp : '');
+    if (!Number.isFinite(ts) || ts < sinceMs) continue;
+    total += 1;
+    const type = (evt && evt.type) || 'unknown';
+    byType[type] = (byType[type] || 0) + 1;
+  }
+  return { total, byType };
+}
+
+function buildSessionReport({ windowHours } = {}) {
+  const hours = normalizeWindowHours(windowHours);
+  const sinceMs = Date.now() - hours * 60 * 60 * 1000;
+  const report = {
+    generatedAt: new Date().toISOString(),
+    windowHours: hours,
+    since: new Date(sinceMs).toISOString(),
+    feedback: { totalPositive: 0, totalNegative: 0, topNegativeTags: [] },
+    gates: { blocked: 0, warned: 0, passed: 0, topGates: [] },
+    provenance: { total: 0, byType: {} },
+    errors: {},
+  };
+
+  try {
+    const { analyzeFeedback } = require('./feedback-loop');
+    const feedback = analyzeFeedback() || {};
+    report.feedback = {
+      totalPositive: feedback.totalPositive || 0,
+      totalNegative: feedback.totalNegative || 0,
+      topNegativeTags: topNegativeTags(feedback.tags || {}),
+    };
+  } catch (err) {
+    report.errors.feedback = String(err && err.message ? err.message : err);
+  }
+
+  try {
+    const { loadStats } = require('./gates-engine');
+    const stats = loadStats() || {};
+    report.gates = {
+      blocked: stats.blocked || 0,
+      warned: stats.warned || 0,
+      passed: stats.passed || 0,
+      pendingApproval: stats.pendingApproval || 0,
+      topGates: topGates(stats.byGate || {}),
+    };
+  } catch (err) {
+    report.errors.gates = String(err && err.message ? err.message : err);
+  }
+
+  try {
+    const { getProvenance } = require('./contextfs');
+    const events = getProvenance(500) || [];
+    report.provenance = summarizeProvenance(events, sinceMs);
+  } catch (err) {
+    report.errors.provenance = String(err && err.message ? err.message : err);
+  }
+
+  if (Object.keys(report.errors).length === 0) {
+    delete report.errors;
+  }
+
+  return report;
+}
+
+module.exports = {
+  buildSessionReport,
+  normalizeWindowHours,
+  topNegativeTags,
+  topGates,
+  summarizeProvenance,
+  MIN_WINDOW_HOURS,
+  MAX_WINDOW_HOURS,
+  DEFAULT_WINDOW_HOURS,
+};

package/scripts/swarm-coordinator.js

@@ -0,0 +1,81 @@
+'use strict';
+
+const { constructContextPack, recordProvenance } = require('./contextfs');
+
+const DEFAULT_TTL_MS = 15 * 60 * 1000;
+const MAX_AGENTS = 32;
+
+function normalizeAgents(agents) {
+  if (!Array.isArray(agents)) {
+    throw new Error('agents must be a non-empty array of agent names');
+  }
+  const normalized = [];
+  const seen = new Set();
+  for (const raw of agents) {
+    const name = typeof raw === 'string' ? raw.trim() : String((raw && raw.name) || '').trim();
+    if (!name) continue;
+    if (seen.has(name)) continue;
+    seen.add(name);
+    normalized.push(name);
+  }
+  if (normalized.length === 0) {
+    throw new Error('agents must include at least one named agent');
+  }
+  if (normalized.length > MAX_AGENTS) {
+    throw new Error(`agents list exceeds MAX_AGENTS (${MAX_AGENTS})`);
+  }
+  return normalized;
+}
+
+function distributeContextToAgents({
+  query = '',
+  agents,
+  maxItems,
+  maxChars,
+  namespaces,
+  ttlMs,
+} = {}) {
+  const agentNames = normalizeAgents(agents);
+  const ttl = Number.isFinite(Number(ttlMs)) && Number(ttlMs) > 0 ? Number(ttlMs) : DEFAULT_TTL_MS;
+  const expiresAt = new Date(Date.now() + ttl).toISOString();
+
+  const pack = constructContextPack({
+    query,
+    maxItems: Number.isFinite(Number(maxItems)) && Number(maxItems) > 0 ? Number(maxItems) : undefined,
+    maxChars: Number.isFinite(Number(maxChars)) && Number(maxChars) > 0 ? Number(maxChars) : undefined,
+    namespaces: Array.isArray(namespaces) ? namespaces : [],
+  });
+
+  const itemCount = Array.isArray(pack.items) ? pack.items.length : 0;
+  const distributions = agentNames.map((agent) => {
+    const provenance = recordProvenance({
+      type: 'context_pack_distributed',
+      packId: pack.packId,
+      agent,
+      query: pack.query,
+      itemCount,
+      expiresAt,
+    });
+    return {
+      agent,
+      packId: pack.packId,
+      provenanceId: provenance.id,
+      expiresAt,
+    };
+  });
+
+  return {
+    packId: pack.packId,
+    query: pack.query,
+    totalAgents: distributions.length,
+    itemCount,
+    expiresAt,
+    distributions,
+  };
+}
+
+module.exports = {
+  distributeContextToAgents,
+  DEFAULT_TTL_MS,
+  MAX_AGENTS,
+};