thatcher 1.0.64 → 1.0.66

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "thatcher",
-  "version": "1.0.64",
+  "version": "1.0.66",
   "description": "A config-driven application framework for building data-intensive web apps without code.",
   "main": "src/index.js",
   "type": "module",

package/src/lib/config-generator-engine.js

@@ -79,12 +79,30 @@ const ENTITY_TEMPLATE_ENTITY_DEFAULT = {
   },
 };
 
+const SCHEDULED_JOB_ENTITY_DEFAULT = {
+  label: 'Scheduled Job',
+  label_plural: 'Scheduled Jobs',
+  system_entity: true,
+  fields: {
+    name: { type: 'text', required: true, label: 'Name' },
+    entity: { type: 'text', required: true, label: 'Entity' },
+    action: { type: 'json', required: true, label: 'Action' },
+    filter: { type: 'json', label: 'Filter' },
+    interval_minutes: { type: 'int', required: true, label: 'Interval (minutes)' },
+    last_run_at: { type: 'int', label: 'Last Run At' },
+    next_run_at: { type: 'int', label: 'Next Run At' },
+    enabled: { type: 'bool', default: true, label: 'Enabled' },
+    owner_id: { type: 'ref', ref: 'user', required: true, label: 'Owner' },
+  },
+};
+
 function withWebhookDefaults(masterConfig) {
   const entities = { ...(masterConfig.entities || {}) };
   let changed = false;
   if (!entities.webhook) { entities.webhook = WEBHOOK_ENTITY_DEFAULT; changed = true; }
   if (!entities.webhook_delivery) { entities.webhook_delivery = WEBHOOK_DELIVERY_ENTITY_DEFAULT; changed = true; }
   if (!entities.entity_template) { entities.entity_template = ENTITY_TEMPLATE_ENTITY_DEFAULT; changed = true; }
+  if (!entities.scheduled_job) { entities.scheduled_job = SCHEDULED_JOB_ENTITY_DEFAULT; changed = true; }
   return changed ? { ...masterConfig, entities } : masterConfig;
 }
 

package/src/lib/rate-limiter.js

@@ -0,0 +1,37 @@
+// Fixed-window per-key rate limiter, in-memory. Keyed by authenticated user id
+// when available, otherwise by remote IP, so one anonymous IP hammering the
+// API can't exhaust a budget shared with real users behind the same NAT --
+// each authenticated user gets their own independent window.
+const WINDOW_MS = 60 * 1000;
+const DEFAULT_LIMIT = 300;
+const buckets = new Map();
+
+let sweepHandle = null;
+function ensureSweep() {
+  if (sweepHandle) return;
+  sweepHandle = setInterval(() => {
+    const now = Date.now();
+    for (const [key, bucket] of buckets) {
+      if (now - bucket.windowStart > WINDOW_MS) buckets.delete(key);
+    }
+  }, WINDOW_MS);
+  if (sweepHandle.unref) sweepHandle.unref();
+}
+
+export function checkRateLimit(key, limit = DEFAULT_LIMIT) {
+  ensureSweep();
+  const now = Date.now();
+  let bucket = buckets.get(key);
+  if (!bucket || now - bucket.windowStart >= WINDOW_MS) {
+    bucket = { windowStart: now, count: 0 };
+    buckets.set(key, bucket);
+  }
+  bucket.count += 1;
+  const remaining = Math.max(0, limit - bucket.count);
+  const resetMs = bucket.windowStart + WINDOW_MS - now;
+  return { allowed: bucket.count <= limit, remaining, resetMs, limit };
+}
+
+export function resetRateLimiter() {
+  buckets.clear();
+}

package/src/lib/scheduler-engine.js

@@ -0,0 +1,69 @@
+import { list, update, getBy } from './busybase/store.js';
+import { runBulkOperation } from './bulk-operations.js';
+import { getConfigEngineSync } from './config-generator-engine.js';
+import { now } from './id-helpers.js';
+import { createLogger } from './logger.js';
+
+const log = createLogger('[Scheduler]');
+const CHECK_INTERVAL_MS = 60 * 1000;
+let intervalHandle = null;
+
+export async function runDueJobs() {
+  const nowTs = now();
+  const dueJobs = await list('scheduled_job', { enabled: true });
+  const results = [];
+  for (const job of dueJobs) {
+    if (!job.enabled) continue;
+    if (job.next_run_at && job.next_run_at > nowTs) continue;
+    results.push(await runOneJob(job, nowTs));
+  }
+  return results;
+}
+
+async function runOneJob(job, nowTs) {
+  try {
+    const owner = await getBy('users', 'id', job.owner_id);
+    if (!owner) throw new Error(`Owner ${job.owner_id} not found`);
+
+    const configEngine = getConfigEngineSync();
+    const spec = configEngine.generateEntitySpec(job.entity);
+
+    const filter = job.filter ? (typeof job.filter === 'string' ? JSON.parse(job.filter) : job.filter) : {};
+    const action = typeof job.action === 'string' ? JSON.parse(job.action) : job.action;
+
+    // list() is org/row-access scoped by passing {user: owner} -- the same
+    // scoping crud-handlers.js's read path enforces for an interactive
+    // request, so a job cannot reach records outside its owner's org just
+    // because it runs unattended.
+    const targets = await list(job.entity, filter, { user: owner });
+    const ids = targets.map(r => r.id);
+
+    let result = { ok: true, total: 0, succeeded: 0, failed: 0, results: [] };
+    if (ids.length) {
+      result = await runBulkOperation(job.entity, spec, ids, action, owner);
+    }
+
+    const nextRunAt = nowTs + job.interval_minutes * 60;
+    await update('scheduled_job', job.id, { last_run_at: nowTs, next_run_at: nextRunAt });
+    log.info(`Job "${job.name}" ran: ${result.succeeded}/${result.total} succeeded`);
+    return { job_id: job.id, ok: true, ...result };
+  } catch (err) {
+    log.error(`Job "${job.name}" failed: ${err.message}`);
+    const nextRunAt = nowTs + job.interval_minutes * 60;
+    await update('scheduled_job', job.id, { last_run_at: nowTs, next_run_at: nextRunAt }).catch(() => {});
+    return { job_id: job.id, ok: false, error: err.message };
+  }
+}
+
+export function startScheduler() {
+  if (intervalHandle) return intervalHandle;
+  intervalHandle = setInterval(() => {
+    runDueJobs().catch(e => log.error(e.message));
+  }, CHECK_INTERVAL_MS);
+  if (intervalHandle.unref) intervalHandle.unref();
+  return intervalHandle;
+}
+
+export function stopScheduler() {
+  if (intervalHandle) { clearInterval(intervalHandle); intervalHandle = null; }
+}

package/src/server/server.js

@@ -73,6 +73,8 @@ export function createServer(options) {
     });
   };
 
+  import('../lib/scheduler-engine.js').then(({ startScheduler }) => startScheduler()).catch(e => log.error(e.message));
+
   const server = http.createServer(async (req, res) => {
     globalThis.__debug__.activeRequests.count++;
 
@@ -91,6 +93,20 @@ export function createServer(options) {
       // API routes
       if (pathname.startsWith('/api/')) {
         res.setHeader('Content-Type', 'application/json; charset=utf-8');
+
+        const { checkRateLimit } = await import('../lib/rate-limiter.js');
+        const rateUser = await resolveRequestUser(req);
+        const rateKey = rateUser ? `user:${rateUser.id}` : `ip:${req.socket?.remoteAddress || 'unknown'}`;
+        const rate = checkRateLimit(rateKey);
+        res.setHeader('X-RateLimit-Limit', String(rate.limit));
+        res.setHeader('X-RateLimit-Remaining', String(rate.remaining));
+        res.setHeader('X-RateLimit-Reset', String(Math.ceil(rate.resetMs / 1000)));
+        if (!rate.allowed) {
+          res.setHeader('Retry-After', String(Math.ceil(rate.resetMs / 1000)));
+          res.writeHead(429);
+          res.end(JSON.stringify({ error: 'Rate limit exceeded, try again later' }));
+          return;
+        }
         const parts = pathname.slice(5).split('/').filter(Boolean); // remove /api/
 
         if (parts.length === 0) {
@@ -151,6 +167,18 @@ export function createServer(options) {
           return await handleFileUpload(req, res, thatcher, configEngine);
         }
 
+        if (req.method === 'POST' && entity === 'scheduled_job' && id === 'create' && !action) {
+          return await handleCreateScheduledJob(req, res, thatcher, configEngine);
+        }
+
+        if (req.method === 'POST' && entity === 'scheduled_job' && id && action === 'update') {
+          return await handleUpdateScheduledJob(req, res, id, thatcher, configEngine);
+        }
+
+        if (req.method === 'POST' && entity === 'scheduled_job' && id && action === 'delete') {
+          return await handleDeleteScheduledJob(req, res, id, thatcher, configEngine);
+        }
+
         // Check if user has custom route for this
         const userRoutePath = path.join(process.cwd(), 'app/api', ...parts, 'route.js');
         const routeExists = await fileExists(userRoutePath);
@@ -541,6 +569,145 @@ async function handleDeleteEntityTemplate(req, res, templateId, thatcher, config
   }
 }
 
+async function handleCreateScheduledJob(req, res, thatcher, configEngineArg) {
+  const user = await requireAuthedPartner(req, res);
+  if (!user) return;
+
+  let configEngine = configEngineArg || thatcher?.configEngine || globalThis.__thatcherConfigEngine;
+  if (!configEngine) {
+    const { getConfigEngineSync } = await import('../lib/config-generator-engine.js');
+    configEngine = getConfigEngineSync();
+  }
+
+  let body;
+  try {
+    body = await readBody(req);
+  } catch (e) {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: e.message }));
+    return;
+  }
+  const { name, entity, action, filter, interval_minutes } = body || {};
+  if (!entity || typeof entity !== 'string') {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: 'entity required' }));
+    return;
+  }
+  try {
+    configEngine.generateEntitySpec(entity);
+  } catch {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: `Unknown entity "${entity}"` }));
+    return;
+  }
+  if (!name || typeof name !== 'string' || !name.trim()) {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: 'name required' }));
+    return;
+  }
+  if (!action || typeof action !== 'object' || typeof action.type !== 'string') {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: 'action.type required' }));
+    return;
+  }
+  const intervalMinutes = Number(interval_minutes);
+  if (!Number.isFinite(intervalMinutes) || intervalMinutes <= 0) {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: 'interval_minutes must be a positive number' }));
+    return;
+  }
+
+  try {
+    const { create } = await import('../lib/busybase/store.js');
+    const { now } = await import('../lib/id-helpers.js');
+    const nowTs = now();
+    const record = await create('scheduled_job', {
+      name: name.trim(),
+      entity,
+      action,
+      filter: filter || {},
+      interval_minutes: intervalMinutes,
+      last_run_at: null,
+      next_run_at: nowTs,
+      enabled: true,
+      owner_id: user.id,
+    }, user);
+    res.writeHead(201, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ ok: true, id: record.id }));
+  } catch (err) {
+    apiLog.error(err.message);
+    res.writeHead(500);
+    res.end(JSON.stringify({ error: err.message }));
+  }
+}
+
+async function handleUpdateScheduledJob(req, res, jobId, thatcher, configEngineArg) {
+  const user = await requireAuthedPartner(req, res);
+  if (!user) return;
+
+  let body;
+  try {
+    body = await readBody(req);
+  } catch (e) {
+    res.writeHead(400);
+    res.end(JSON.stringify({ error: e.message }));
+    return;
+  }
+
+  try {
+    const { update, get } = await import('../lib/busybase/store.js');
+    const existing = await get('scheduled_job', jobId);
+    if (!existing) {
+      res.writeHead(404);
+      res.end(JSON.stringify({ error: 'Job not found' }));
+      return;
+    }
+    const patch = {};
+    if (typeof body?.enabled === 'boolean') patch.enabled = body.enabled;
+    if (typeof body?.name === 'string' && body.name.trim()) patch.name = body.name.trim();
+    if (body?.interval_minutes !== undefined) {
+      const intervalMinutes = Number(body.interval_minutes);
+      if (!Number.isFinite(intervalMinutes) || intervalMinutes <= 0) {
+        res.writeHead(400);
+        res.end(JSON.stringify({ error: 'interval_minutes must be a positive number' }));
+        return;
+      }
+      patch.interval_minutes = intervalMinutes;
+    }
+    if (body?.filter !== undefined) patch.filter = body.filter;
+    if (body?.action !== undefined) patch.action = body.action;
+    const record = await update('scheduled_job', jobId, patch);
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ ok: true, data: record }));
+  } catch (err) {
+    apiLog.error(err.message);
+    res.writeHead(500);
+    res.end(JSON.stringify({ error: err.message }));
+  }
+}
+
+async function handleDeleteScheduledJob(req, res, jobId, thatcher, configEngineArg) {
+  const user = await requireAuthedPartner(req, res);
+  if (!user) return;
+
+  try {
+    const { remove, get } = await import('../lib/busybase/store.js');
+    const existing = await get('scheduled_job', jobId);
+    if (!existing) {
+      res.writeHead(404);
+      res.end(JSON.stringify({ error: 'Job not found' }));
+      return;
+    }
+    await remove('scheduled_job', jobId);
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ ok: true }));
+  } catch (err) {
+    apiLog.error(err.message);
+    res.writeHead(500);
+    res.end(JSON.stringify({ error: err.message }));
+  }
+}
+
 async function handleCreateWebhook(req, res, thatcher, configEngineArg) {
   const user = await requireAuthedPartner(req, res);
   if (!user) return;

package/src/ui/dashboard-renderer.js

@@ -89,12 +89,12 @@ export function renderAuditDashboard(user, auditData = {}) {
   const { summary = {}, recentActivity = [] } = auditData;
   const actRows = recentActivity.slice(0, 20).map(a =>
     `<tr data-row>
-      <td data-col="time">${new Date((a.timestamp||a.created_at)*1000).toLocaleString('en-ZA')}</td>
-      <td data-col="action"><span class="pill pill-info">${a.action||'-'}</span></td>
-      <td data-col="entity">${a.entity_type||'-'}</td>
-      <td data-col="id" style="font-size:12px">${a.entity_id||'-'}</td>
-      <td data-col="user">${a.user_name||a.user_id||'-'}</td>
-      <td data-col="reason" style="font-size:12px;color:var(--color-text-muted)">${a.reason||'-'}</td>
+      <td data-col="time">${esc(new Date((a.timestamp||a.created_at)*1000).toLocaleString('en-ZA'))}</td>
+      <td data-col="action"><span class="pill pill-info">${esc(a.action||'-')}</span></td>
+      <td data-col="entity">${esc(a.entity_type||'-')}</td>
+      <td data-col="id" style="font-size:12px">${esc(a.entity_id||'-')}</td>
+      <td data-col="user">${esc(a.user_name||a.user_id||'-')}</td>
+      <td data-col="reason" style="font-size:12px;color:var(--color-text-muted)">${esc(a.reason||'-')}</td>
     </tr>`
    ).join('') || emptyRow(6, 'No audit records found');
 
@@ -124,7 +124,7 @@ export function renderAuditDashboard(user, auditData = {}) {
 export function renderSystemHealth(user, healthData = {}) {
   const { database = {}, server: srv = {}, entities = {} } = healthData;
   const entRows = Object.entries(entities).map(([n, c]) =>
-    `<tr data-row><td data-col="entity">${n}</td><td data-col="count" style="text-align:right">${c}</td></tr>`
+    `<tr data-row><td data-col="entity">${esc(n)}</td><td data-col="count" style="text-align:right">${esc(String(c))}</td></tr>`
    ).join('') || emptyRow(2, 'No data');
 
   const statsHtml = `<div class="stats-row">${[

package/src/ui/page-handler-admin.js

@@ -10,6 +10,7 @@ import { renderWorkflowList, renderWorkflowEditor } from '@/ui/workflow-builder-
 import { renderRolesList, renderTemplateList, renderPermissionMatrix } from '@/ui/rbac-renderer.js';
 import { renderWebhookList, renderWebhookDetail } from '@/ui/webhook-renderer.js';
 import { renderTemplateList as renderEntityTemplateList } from '@/ui/template-renderer.js';
+import { renderScheduledJobList } from '@/ui/scheduled-job-renderer.js';
 import { isPartner, isManager } from '@/ui/permissions-ui.js';
 import { getSystemConfig, getSettingsCounts, getAuditData, getSystemHealth, renderBuildLogsContent } from '@/ui/page-handler-helpers.js';
 import { fileURLToPath } from 'url';
@@ -191,5 +192,13 @@ export async function handleAdminPage(normalized, segments, user) {
     }
     return renderEntityTemplateList(user, entityNames, templatesByEntity);
   }
+  if (normalized === '/admin/scheduled-jobs') {
+    if (!isPartner(user)) return renderAccessDenied(user, 'admin', 'view');
+    const { getConfigEngineSync } = await import('@/lib/config-generator-engine.js');
+    const engine = getConfigEngineSync();
+    const entityNames = engine.getAllEntities().filter(e => e !== 'scheduled_job' && e !== 'entity_template' && e !== 'webhook' && e !== 'webhook_delivery' && e !== 'user_organization');
+    let jobs = []; try { jobs = await list('scheduled_job', {}); } catch {}
+    return renderScheduledJobList(user, jobs, entityNames);
+  }
   return null;
 }

package/src/ui/scheduled-job-renderer.js

@@ -0,0 +1,92 @@
+import { page } from '@/ui/layout.js';
+import { esc } from '@/ui/render-helpers.js';
+
+function fmtTs(ts) {
+  if (!ts) return '-';
+  return new Date(ts * 1000).toISOString().replace('T', ' ').slice(0, 19);
+}
+
+export function renderScheduledJobList(user, jobs, entityNames) {
+  const rows = jobs.map(j => {
+    const statusPill = j.enabled ? `<span class="pill pill-success">Enabled</span>` : `<span class="pill pill-neutral">Disabled</span>`;
+    const toggleLabel = j.enabled ? 'Disable' : 'Enable';
+    return `<tr>
+      <td>${esc(j.name)}</td>
+      <td>${esc(j.entity)}</td>
+      <td>${esc(String(j.interval_minutes))} min</td>
+      <td>${statusPill}</td>
+      <td>${esc(fmtTs(j.last_run_at))}</td>
+      <td>${esc(fmtTs(j.next_run_at))}</td>
+      <td style="display:flex;gap:8px">
+        <button type="button" class="btn-ghost-clean" data-action="toggleJob" data-args='["${esc(j.id)}", ${j.enabled ? 'false' : 'true'}]'>${toggleLabel}</button>
+        <button type="button" class="btn-ghost-clean" data-action="deleteJob" data-args='["${esc(j.id)}"]'>Delete</button>
+      </td>
+    </tr>`;
+  }).join('') || '<tr><td colspan="7">No scheduled jobs</td></tr>';
+
+  const entityOpts = entityNames.map(e => `<option value="${esc(e)}">${esc(e)}</option>`).join('');
+
+  const content = `<div class="page-header"><h1 class="page-title">Scheduled Jobs</h1></div>
+    <div class="card-clean" style="margin-bottom:16px"><div class="card-clean-body">
+      <table class="data-table"><thead><tr><th>Name</th><th>Entity</th><th>Interval</th><th>Status</th><th>Last Run</th><th>Next Run</th><th></th></tr></thead><tbody>${rows}</tbody></table>
+    </div></div>
+    <div class="card-clean"><div class="card-clean-body">
+      <h3 style="margin-bottom:8px">New Scheduled Job</h3>
+      <div style="display:flex;flex-direction:column;gap:8px;max-width:480px">
+        <input type="text" id="new-job-name" placeholder="Job name" class="form-input">
+        <select id="new-job-entity" class="form-input"><option value="">Select entity...</option>${entityOpts}</select>
+        <select id="new-job-action-type" class="form-input">
+          <option value="delete">Delete matching records</option>
+          <option value="set_field">Set field on matching records</option>
+        </select>
+        <input type="text" id="new-job-field" placeholder="Field name (for set_field)" class="form-input">
+        <input type="text" id="new-job-value" placeholder="Value (for set_field)" class="form-input">
+        <input type="text" id="new-job-filter" placeholder='Filter JSON, e.g. {"status":"pending"}' class="form-input">
+        <input type="number" id="new-job-interval" placeholder="Interval (minutes)" class="form-input" min="1">
+        <button type="button" class="btn-primary-clean" data-action="createJob">Create Job</button>
+      </div>
+    </div></div>
+    <span id="job-status" style="font-size:13px"></span>`;
+
+  const script = `(function(){
+    window.createJob=function(){
+      var status=document.getElementById('job-status');
+      var name=(document.getElementById('new-job-name').value||'').trim();
+      var entity=document.getElementById('new-job-entity').value;
+      var actionType=document.getElementById('new-job-action-type').value;
+      var field=(document.getElementById('new-job-field').value||'').trim();
+      var value=document.getElementById('new-job-value').value;
+      var filterRaw=(document.getElementById('new-job-filter').value||'').trim();
+      var interval=Number(document.getElementById('new-job-interval').value);
+      if(!name){status.textContent='Name required';return}
+      if(!entity){status.textContent='Entity required';return}
+      if(!interval||interval<=0){status.textContent='Interval must be a positive number';return}
+      var filter={};
+      if(filterRaw){try{filter=JSON.parse(filterRaw)}catch(e){status.textContent='Filter must be valid JSON';return}}
+      var action=actionType==='set_field'?{type:'set_field',field:field,value:value}:{type:'delete'};
+      status.textContent='Creating...';
+      fetch('/api/scheduled_job/create',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:name,entity:entity,action:action,filter:filter,interval_minutes:interval})})
+        .then(function(r){return r.json().then(function(d){return {ok:r.ok,d:d}})})
+        .then(function(res){if(res.ok){location.reload()}else{status.textContent='Error: '+(res.d.error||'create failed')}})
+        .catch(function(err){status.textContent='Error: '+err.message});
+    };
+    window.toggleJob=function(id,enabled){
+      var status=document.getElementById('job-status');
+      status.textContent='Updating...';
+      fetch('/api/scheduled_job/'+id+'/update',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({enabled:enabled})})
+        .then(function(r){return r.json().then(function(d){return {ok:r.ok,d:d}})})
+        .then(function(res){if(res.ok){location.reload()}else{status.textContent='Error: '+(res.d.error||'update failed')}})
+        .catch(function(err){status.textContent='Error: '+err.message});
+    };
+    window.deleteJob=function(id){
+      var status=document.getElementById('job-status');
+      status.textContent='Deleting...';
+      fetch('/api/scheduled_job/'+id+'/delete',{method:'POST'})
+        .then(function(r){return r.json().then(function(d){return {ok:r.ok,d:d}})})
+        .then(function(res){if(res.ok){location.reload()}else{status.textContent='Error: '+(res.d.error||'delete failed')}})
+        .catch(function(err){status.textContent='Error: '+err.message});
+    };
+  })();`;
+
+  return page(user, 'Scheduled Jobs | Thatcher', [{ href: '/admin/settings', label: 'Settings' }, { label: 'Scheduled Jobs' }], content, [script]);
+}