thatcher 1.0.60 → 1.0.61

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "thatcher",
-  "version": "1.0.60",
+  "version": "1.0.61",
   "description": "A config-driven application framework for building data-intensive web apps without code.",
   "main": "src/index.js",
   "type": "module",

package/src/index.js

@@ -66,6 +66,9 @@ export class Thatcher {
     const { registerAutomationEngine } = await import(resolveModule('./lib/automation-engine.js'));
     registerAutomationEngine();
 
+    const { registerWebhookEngine } = await import(resolveModule('./lib/webhook-engine.js'));
+    registerWebhookEngine();
+
     // Hot reload
     if (this.options.server.hotReload) {
       this.setupHotReload();

package/src/lib/config-generator-engine.js

@@ -41,10 +41,45 @@ function withMultiTenancyDefaults(masterConfig) {
   return { ...masterConfig, entities };
 }
 
+const WEBHOOK_ENTITY_DEFAULT = {
+  label: 'Webhook',
+  label_plural: 'Webhooks',
+  system_entity: true,
+  fields: {
+    entity: { type: 'text', required: true, label: 'Entity' },
+    trigger: { type: 'text', required: true, label: 'Trigger' },
+    url: { type: 'text', required: true, label: 'URL' },
+    secret: { type: 'text', required: true, label: 'Secret', hidden: true },
+    enabled: { type: 'bool', default: true, label: 'Enabled' },
+  },
+};
+
+const WEBHOOK_DELIVERY_ENTITY_DEFAULT = {
+  label: 'Webhook Delivery',
+  label_plural: 'Webhook Deliveries',
+  system_entity: true,
+  fields: {
+    webhook_id: { type: 'ref', ref: 'webhook', required: true },
+    event: { type: 'text' },
+    status_code: { type: 'int' },
+    success: { type: 'bool' },
+    attempt: { type: 'int' },
+    error: { type: 'text' },
+  },
+};
+
+function withWebhookDefaults(masterConfig) {
+  const entities = { ...(masterConfig.entities || {}) };
+  let changed = false;
+  if (!entities.webhook) { entities.webhook = WEBHOOK_ENTITY_DEFAULT; changed = true; }
+  if (!entities.webhook_delivery) { entities.webhook_delivery = WEBHOOK_DELIVERY_ENTITY_DEFAULT; changed = true; }
+  return changed ? { ...masterConfig, entities } : masterConfig;
+}
+
 export class ConfigGeneratorEngine {
   constructor(masterConfig) {
     if (!masterConfig) throw new Error('[ConfigGeneratorEngine] masterConfig is required');
-    this.masterConfig = deepFreeze(withMultiTenancyDefaults(masterConfig));
+    this.masterConfig = deepFreeze(withWebhookDefaults(withMultiTenancyDefaults(masterConfig)));
     this.specCache = new LRUCache(100);
     this.debugMode = false;
     this._plugins = new Map();

package/src/lib/webhook-engine.js

@@ -0,0 +1,187 @@
+import crypto from 'crypto';
+import dns from 'dns/promises';
+import { hookEngine } from './hook-engine.js';
+import { list, create, get } from './busybase/store.js';
+import { getConfigEngineSync } from './config-generator-engine.js';
+import { createLogger } from './logger.js';
+
+const log = createLogger('[WebhookEngine]');
+
+const MAX_ATTEMPTS = 3;
+const RETRY_BACKOFF_MS = [500, 2000, 5000];
+const REQUEST_TIMEOUT_MS = 10000;
+
+function ipv4ToLong(ip) {
+  const parts = ip.split('.').map(Number);
+  if (parts.length !== 4 || parts.some(p => isNaN(p) || p < 0 || p > 255)) return null;
+  return (parts[0] << 24) + (parts[1] << 16) + (parts[2] << 8) + parts[3];
+}
+
+function inRange(ip, base, bits) {
+  const ipLong = ipv4ToLong(ip);
+  const baseLong = ipv4ToLong(base);
+  if (ipLong === null || baseLong === null) return false;
+  const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0;
+  return (ipLong & mask) === (baseLong & mask);
+}
+
+const PRIVATE_RANGES = [
+  ['10.0.0.0', 8],
+  ['172.16.0.0', 12],
+  ['192.168.0.0', 16],
+  ['127.0.0.0', 8],
+  ['169.254.0.0', 16],
+  ['0.0.0.0', 8],
+];
+
+export function isPrivateOrLoopbackIp(ip) {
+  if (ip === '::1' || ip.startsWith('fe80:') || ip.startsWith('fc') || ip.startsWith('fd')) return true;
+  return PRIVATE_RANGES.some(([base, bits]) => inRange(ip, base, bits));
+}
+
+export async function validateWebhookUrl(rawUrl, options = {}) {
+  let parsed;
+  try {
+    parsed = new URL(rawUrl);
+  } catch {
+    return { ok: false, error: 'Invalid URL' };
+  }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    return { ok: false, error: `Scheme "${parsed.protocol}" not allowed, only http/https` };
+  }
+  const hostname = parsed.hostname;
+  if (hostname === 'localhost') {
+    if (options.allowPrivateTargets) return { ok: true };
+    return { ok: false, error: 'localhost targets are not allowed' };
+  }
+  const ipv4Literal = ipv4ToLong(hostname) !== null;
+  if (ipv4Literal) {
+    if (isPrivateOrLoopbackIp(hostname) && !options.allowPrivateTargets) {
+      return { ok: false, error: `Private/loopback IP target "${hostname}" not allowed` };
+    }
+    return { ok: true };
+  }
+  if (options.allowPrivateTargets) return { ok: true };
+  try {
+    const resolved = await dns.lookup(hostname, { all: true });
+    for (const { address, family } of resolved) {
+      if (family === 4 && isPrivateOrLoopbackIp(address)) {
+        return { ok: false, error: `Hostname "${hostname}" resolves to private/loopback address "${address}"` };
+      }
+      if (family === 6 && (address === '::1' || address.startsWith('fe80:') || address.startsWith('fc') || address.startsWith('fd'))) {
+        return { ok: false, error: `Hostname "${hostname}" resolves to private/loopback address "${address}"` };
+      }
+    }
+    return { ok: true };
+  } catch (e) {
+    return { ok: false, error: `Could not resolve hostname "${hostname}": ${e.message}` };
+  }
+}
+
+export function signPayload(secret, payloadString) {
+  return crypto.createHmac('sha256', secret).update(payloadString).digest('hex');
+}
+
+export function verifySignature(secret, payloadString, signature) {
+  const expected = signPayload(secret, payloadString);
+  const expectedBuf = Buffer.from(expected, 'hex');
+  const givenBuf = Buffer.from(signature || '', 'hex');
+  if (expectedBuf.length !== givenBuf.length) return false;
+  return crypto.timingSafeEqual(expectedBuf, givenBuf);
+}
+
+async function deliverOnce(webhook, payloadString) {
+  const signature = signPayload(webhook.secret, payloadString);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+  try {
+    const res = await fetch(webhook.url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'X-Thatcher-Signature': signature },
+      body: payloadString,
+      signal: controller.signal,
+    });
+    return { success: res.ok, statusCode: res.status, error: res.ok ? null : `HTTP ${res.status}` };
+  } catch (e) {
+    return { success: false, statusCode: 0, error: e.message };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+async function deliverWithRetry(webhook, payloadString) {
+  let lastResult = null;
+  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+    lastResult = await deliverOnce(webhook, payloadString);
+    try {
+      await create('webhook_delivery', {
+        webhook_id: webhook.id,
+        event: webhook.trigger,
+        status_code: lastResult.statusCode,
+        success: lastResult.success,
+        attempt,
+        error: lastResult.error || '',
+      }, null);
+    } catch (e) {
+      log.error('failed to write webhook_delivery log:', { message: e.message });
+    }
+    if (lastResult.success) return lastResult;
+    if (attempt < MAX_ATTEMPTS) {
+      await new Promise(r => setTimeout(r, RETRY_BACKOFF_MS[attempt - 1] || RETRY_BACKOFF_MS[RETRY_BACKOFF_MS.length - 1]));
+    }
+  }
+  return lastResult;
+}
+
+function normalizeContext(context) {
+  const data = context.data || context.record || {};
+  return { ...context, data, entity: context.entity, id: context.id ?? data.id };
+}
+
+async function dispatchWebhooks(entityName, trigger, rawContext) {
+  let webhooks;
+  try {
+    webhooks = await list('webhook', { entity: entityName, trigger, enabled: true });
+  } catch (e) {
+    log.error('failed to list webhooks:', { message: e.message });
+    return;
+  }
+  if (!webhooks.length) return;
+
+  const context = normalizeContext(rawContext);
+  const payload = { event: trigger, entity: entityName, id: context.id, data: context.data, timestamp: Math.floor(Date.now() / 1000) };
+  const payloadString = JSON.stringify(payload);
+
+  for (const webhook of webhooks) {
+    const validation = await validateWebhookUrl(webhook.url);
+    if (!validation.ok) {
+      log.error(`webhook ${webhook.id} URL rejected: ${validation.error}`);
+      continue;
+    }
+    deliverWithRetry(webhook, payloadString).catch(e => log.error('webhook delivery failed:', { message: e.message }));
+  }
+}
+
+let registered = false;
+
+export function registerWebhookEngine() {
+  if (registered) return;
+  registered = true;
+
+  const config = getConfigEngineSync().getConfig();
+  const entityNames = Object.keys(config?.entities || {});
+
+  for (const entityName of entityNames) {
+    if (entityName === 'webhook' || entityName === 'webhook_delivery') continue;
+    for (const trigger of ['create', 'update', 'delete']) {
+      hookEngine.register(`${trigger}:${entityName}:after`, async (context) => {
+        dispatchWebhooks(entityName, trigger, context).catch(e => log.error('webhook dispatch failed:', { message: e.message }));
+        return context;
+      });
+    }
+    hookEngine.register(`transition:${entityName}`, async (context) => {
+      dispatchWebhooks(entityName, 'transition', context).catch(e => log.error('webhook dispatch failed:', { message: e.message }));
+      return context;
+    });
+  }
+}

package/src/server/server.js

@@ -122,6 +122,18 @@ export function createServer(options) {
           return await handleUpdatePermissionTemplate(req, res, id, thatcher, configEngine);
         }
 
+        if (req.method === 'POST' && entity === 'webhook' && id === 'create' && !action) {
+          return await handleCreateWebhook(req, res, thatcher, configEngine);
+        }
+
+        if (req.method === 'POST' && entity === 'webhook' && id && action === 'update') {
+          return await handleUpdateWebhookRecord(req, res, id, thatcher, configEngine);
+        }
+
+        if (req.method === 'POST' && entity === 'webhook' && id && action === 'delete') {
+          return await handleDeleteWebhook(req, res, id, thatcher, configEngine);
+        }
+
         // Check if user has custom route for this
         const userRoutePath = path.join(process.cwd(), 'app/api', ...parts, 'route.js');
         const routeExists = await fileExists(userRoutePath);
@@ -351,6 +363,137 @@ async function handleGenericCrud(req, res, entity, id, action, thatcher, configE
 
 const PERMISSION_ACTIONS = new Set(['list', 'view', 'create', 'edit', 'delete', 'archive', 'export', 'manage_settings']);
 
+async function requireAuthedPartner(req, res) {
+  const user = await resolveRequestUser(req);
+  if (!user) {
+    res.writeHead(401, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ error: 'Authentication required' }));
+    return null;
+  }
+  const { isPartner } = await import('../ui/permissions-ui.js');
+  if (!isPartner(user)) {
+    res.writeHead(403, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ error: 'Forbidden' }));
+    return null;
+  }
+  return user;
+}
+
+async function handleCreateWebhook(req, res, thatcher, configEngineArg) {
+  const user = await requireAuthedPartner(req, res);
+  if (!user) return;
+
+  let body;
+  try {
+    body = await readBody(req);
+  } catch (e) {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: e.message }));
+    return;
+  }
+  const { entity, trigger, url: targetUrl } = body || {};
+  if (!entity || typeof entity !== 'string') {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: 'entity required' }));
+    return;
+  }
+  if (!['create', 'update', 'delete', 'transition'].includes(trigger)) {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: 'trigger must be one of create/update/delete/transition' }));
+    return;
+  }
+  const { validateWebhookUrl } = await import('../lib/webhook-engine.js');
+  const validation = await validateWebhookUrl(targetUrl || '');
+  if (!validation.ok) {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: validation.error }));
+    return;
+  }
+
+  try {
+    const crypto = await import('crypto');
+    const secret = crypto.randomBytes(32).toString('hex');
+    const { create } = await import('../lib/busybase/store.js');
+    const record = await create('webhook', { entity, trigger, url: targetUrl, secret, enabled: true }, user);
+    res.writeHead(201, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ ok: true, id: record.id }));
+  } catch (err) {
+    apiLog.error(err.message);
+    res.writeHead(500);
+    res.end(JSON.stringify({ error: err.message }));
+  }
+}
+
+async function handleUpdateWebhookRecord(req, res, webhookId, thatcher, configEngineArg) {
+  const user = await requireAuthedPartner(req, res);
+  if (!user) return;
+
+  let body;
+  try {
+    body = await readBody(req);
+  } catch (e) {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: e.message }));
+    return;
+  }
+  const patch = {};
+  if (typeof body?.enabled === 'boolean') patch.enabled = body.enabled;
+  if (typeof body?.url === 'string') {
+    const { validateWebhookUrl } = await import('../lib/webhook-engine.js');
+    const validation = await validateWebhookUrl(body.url);
+    if (!validation.ok) {
+      res.writeHead(400);
+      res.end(JSON.stringify({ error: validation.error }));
+      return;
+    }
+    patch.url = body.url;
+  }
+  if (!Object.keys(patch).length) {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: 'nothing to update' }));
+    return;
+  }
+
+  try {
+    const { update, get } = await import('../lib/busybase/store.js');
+    const existing = await get('webhook', webhookId);
+    if (!existing) {
+      res.writeHead(404);
+      res.end(JSON.stringify({ error: 'Webhook not found' }));
+      return;
+    }
+    await update('webhook', webhookId, patch);
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ ok: true, id: webhookId }));
+  } catch (err) {
+    apiLog.error(err.message);
+    res.writeHead(500);
+    res.end(JSON.stringify({ error: err.message }));
+  }
+}
+
+async function handleDeleteWebhook(req, res, webhookId, thatcher, configEngineArg) {
+  const user = await requireAuthedPartner(req, res);
+  if (!user) return;
+
+  try {
+    const { remove, get } = await import('../lib/busybase/store.js');
+    const existing = await get('webhook', webhookId);
+    if (!existing) {
+      res.writeHead(404);
+      res.end(JSON.stringify({ error: 'Webhook not found' }));
+      return;
+    }
+    await remove('webhook', webhookId);
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ ok: true }));
+  } catch (err) {
+    apiLog.error(err.message);
+    res.writeHead(500);
+    res.end(JSON.stringify({ error: err.message }));
+  }
+}
+
 async function handleUpdatePermissionTemplate(req, res, templateName, thatcher, configEngineArg) {
   const user = await resolveRequestUser(req);
   if (!user) {

package/src/ui/page-handler-admin.js

@@ -8,6 +8,7 @@ import { renderChecklistsManagement } from '@/ui/checklist-renderer.js';
 import { renderJobManagement } from '@/ui/job-management-renderer.js';
 import { renderWorkflowList, renderWorkflowEditor } from '@/ui/workflow-builder-renderer.js';
 import { renderRolesList, renderTemplateList, renderPermissionMatrix } from '@/ui/rbac-renderer.js';
+import { renderWebhookList, renderWebhookDetail } from '@/ui/webhook-renderer.js';
 import { isPartner, isManager } from '@/ui/permissions-ui.js';
 import { getSystemConfig, getSettingsCounts, getAuditData, getSystemHealth, renderBuildLogsContent } from '@/ui/page-handler-helpers.js';
 import { fileURLToPath } from 'url';
@@ -163,5 +164,19 @@ export async function handleAdminPage(normalized, segments, user) {
     const roleNames = Object.keys(engine.getRoles());
     return renderPermissionMatrix(user, segments[2], roleNames, roleActionsMap);
   }
+  if (normalized === '/admin/webhooks') {
+    let webhooks = []; try { webhooks = await list('webhook', {}); } catch {}
+    const { getConfigEngineSync } = await import('@/lib/config-generator-engine.js');
+    const engine = getConfigEngineSync();
+    const entityNames = engine.getAllEntities().filter(e => e !== 'webhook' && e !== 'webhook_delivery');
+    return renderWebhookList(user, webhooks, entityNames);
+  }
+  if (segments.length === 3 && segments[1] === 'webhooks') {
+    const webhook = await get('webhook', segments[2]);
+    if (!webhook) return null;
+    let deliveries = [];
+    try { deliveries = (await list('webhook_delivery', { webhook_id: segments[2] }, { sort: { field: 'created_at', dir: 'DESC' } })).slice(0, 20); } catch {}
+    return renderWebhookDetail(user, webhook, deliveries);
+  }
   return null;
 }

package/src/ui/webhook-renderer.js

@@ -0,0 +1,83 @@
+import { page } from '@/ui/layout.js';
+import { esc } from '@/ui/render-helpers.js';
+
+export function renderWebhookList(user, webhooks, entityNames) {
+  const rows = webhooks.map(w =>
+    `<tr data-row data-navigate="/admin/webhooks/${esc(w.id)}" style="cursor:pointer">
+      <td>${esc(w.entity)}</td><td>${esc(w.trigger)}</td><td>${esc(w.url)}</td>
+      <td>${w.enabled ? 'Enabled' : 'Disabled'}</td>
+    </tr>`
+  ).join('') || '<tr><td colspan="4">No webhooks configured</td></tr>';
+
+  const entityOpts = entityNames.map(e => `<option value="${esc(e)}">${esc(e)}</option>`).join('');
+  const triggerOpts = ['create', 'update', 'delete', 'transition'].map(t => `<option value="${esc(t)}">${esc(t)}</option>`).join('');
+
+  const content = `<div class="page-header"><h1 class="page-title">Webhooks</h1></div>
+    <div class="table-wrap"><table class="data-table"><thead><tr><th>Entity</th><th>Trigger</th><th>URL</th><th>Status</th></tr></thead><tbody>${rows}</tbody></table></div>
+    <div class="card-clean" style="margin-top:16px"><div class="card-clean-body">
+      <h3 style="margin-bottom:8px">Add Webhook</h3>
+      <div style="display:flex;gap:8px;flex-wrap:wrap;align-items:center">
+        <select id="new-wh-entity">${entityOpts}</select>
+        <select id="new-wh-trigger">${triggerOpts}</select>
+        <input type="text" id="new-wh-url" placeholder="https://example.com/hook" style="flex:1;min-width:200px">
+        <button type="button" class="btn-primary-clean" data-action="createWebhook">Add</button>
+      </div>
+      <span id="create-status" style="margin-left:8px;font-size:13px"></span>
+    </div></div>`;
+
+  const script = `(function(){
+    window.createWebhook=function(){
+      var entity=document.getElementById('new-wh-entity').value;
+      var trigger=document.getElementById('new-wh-trigger').value;
+      var urlInput=document.getElementById('new-wh-url');
+      var url=urlInput.value.trim();
+      var status=document.getElementById('create-status');
+      if(!url){status.textContent='URL required';return}
+      status.textContent='Creating...';
+      fetch('/api/webhook/create',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({entity:entity,trigger:trigger,url:url})})
+        .then(function(r){return r.json().then(function(d){return {ok:r.ok,d:d}})})
+        .then(function(res){if(res.ok){status.textContent='Created';location.reload()}else{status.textContent='Error: '+(res.d.error||'create failed')}})
+        .catch(function(err){status.textContent='Error: '+err.message});
+    };
+  })();`;
+
+  return page(user, 'Webhooks | Thatcher', [{ href: '/admin/settings', label: 'Settings' }, { label: 'Webhooks' }], content, [script]);
+}
+
+export function renderWebhookDetail(user, webhook, deliveries) {
+  const deliveryRows = deliveries.map(d =>
+    `<tr><td>${esc(String(d.attempt))}</td><td>${d.success ? 'Success' : 'Failed'}</td><td>${esc(String(d.status_code ?? '-'))}</td><td>${esc(d.error || '-')}</td></tr>`
+  ).join('') || '<tr><td colspan="4">No deliveries yet</td></tr>';
+
+  const content = `<div class="page-header"><h1 class="page-title">Webhook: ${esc(webhook.entity)}.${esc(webhook.trigger)}</h1></div>
+    <div class="card-clean" style="margin-bottom:16px"><div class="card-clean-body">
+      <div style="margin-bottom:8px"><strong>URL:</strong> ${esc(webhook.url)}</div>
+      <div style="margin-bottom:8px"><strong>Status:</strong> <span id="wh-status-label">${webhook.enabled ? 'Enabled' : 'Disabled'}</span></div>
+      <button type="button" class="btn-ghost-clean" data-action="toggleWebhook" data-args='["${esc(webhook.id)}",${webhook.enabled ? 'false' : 'true'}]'>${webhook.enabled ? 'Disable' : 'Enable'}</button>
+      <button type="button" class="btn-danger-clean" data-action="deleteWebhook" data-args='["${esc(webhook.id)}"]'>Delete</button>
+      <span id="action-status" style="margin-left:8px;font-size:13px"></span>
+    </div></div>
+    <h3>Recent Deliveries</h3>
+    <div class="table-wrap"><table class="data-table"><thead><tr><th>Attempt</th><th>Result</th><th>Status Code</th><th>Error</th></tr></thead><tbody>${deliveryRows}</tbody></table></div>`;
+
+  const script = `(function(){
+    window.toggleWebhook=function(id,nextEnabled){
+      var status=document.getElementById('action-status');
+      status.textContent='Saving...';
+      fetch('/api/webhook/'+id+'/update',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({enabled:nextEnabled})})
+        .then(function(r){return r.json().then(function(d){return {ok:r.ok,d:d}})})
+        .then(function(res){if(res.ok){location.reload()}else{status.textContent='Error: '+(res.d.error||'update failed')}})
+        .catch(function(err){status.textContent='Error: '+err.message});
+    };
+    window.deleteWebhook=function(id){
+      var status=document.getElementById('action-status');
+      status.textContent='Deleting...';
+      fetch('/api/webhook/'+id+'/delete',{method:'POST'})
+        .then(function(r){return r.json().then(function(d){return {ok:r.ok,d:d}})})
+        .then(function(res){if(res.ok){window.location='/admin/webhooks'}else{status.textContent='Error: '+(res.d.error||'delete failed')}})
+        .catch(function(err){status.textContent='Error: '+err.message});
+    };
+  })();`;
+
+  return page(user, `Webhook: ${webhook.entity}.${webhook.trigger} | Thatcher`, [{ href: '/admin/settings', label: 'Settings' }, { href: '/admin/webhooks', label: 'Webhooks' }, { label: `${webhook.entity}.${webhook.trigger}` }], content, [script]);
+}