thatcher 1.0.89 → 1.0.91

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "thatcher",
-  "version": "1.0.89",
+  "version": "1.0.91",
   "description": "A config-driven application framework for building data-intensive web apps without code.",
   "main": "src/index.js",
   "type": "module",

package/src/lib/config-generator-engine.js

@@ -277,6 +277,37 @@ function withResourceManagementDefaults(masterConfig) {
   return changed ? { ...masterConfig, entities } : masterConfig;
 }
 
+// Generic outbound CRM-sync connector config, not a bespoke Salesforce/HubSpot
+// integration -- provider/base_url/entity_mappings make it configurable to
+// any downstream system's sync endpoint. organization_id is an EXPLICIT
+// manually-defined field despite system_entity:true (same precedent as
+// USER_ORGANIZATION_ENTITY_DEFAULT above) because integration credentials
+// leaking cross-org via the system_entity multi-tenancy exemption would be a
+// real vulnerability, not the usual global-by-design system entity case.
+const INTEGRATION_CONNECTION_ENTITY_DEFAULT = {
+  label: 'Integration Connection',
+  label_plural: 'Integration Connections',
+  system_entity: true,
+  fields: {
+    organization_id: { type: 'ref', ref: 'organization', label: 'Organization' },
+    provider: { type: 'text', required: true, label: 'Provider' },
+    base_url: { type: 'text', required: true, label: 'Base URL' },
+    auth_type: { type: 'enum', required: true, label: 'Auth Type', options: [{ value: 'api_key', label: 'API Key' }, { value: 'oauth_token', label: 'OAuth Token' }] },
+    credential: { type: 'text', required: true, label: 'Credential', encrypted: true },
+    entity_mappings: { type: 'json', label: 'Entity Mappings' },
+    sync_direction: { type: 'enum', required: true, default: 'outbound', label: 'Sync Direction', options: [{ value: 'outbound', label: 'Outbound' }, { value: 'inbound', label: 'Inbound' }, { value: 'bidirectional', label: 'Bidirectional' }] },
+    enabled: { type: 'bool', default: true, label: 'Enabled' },
+    last_synced_at: { type: 'int', readonly: true, label: 'Last Synced At' },
+  },
+};
+
+function withIntegrationDefaults(masterConfig) {
+  const entities = { ...(masterConfig.entities || {}) };
+  let changed = false;
+  if (!entities.integration_connection) { entities.integration_connection = INTEGRATION_CONNECTION_ENTITY_DEFAULT; changed = true; }
+  return changed ? { ...masterConfig, entities } : masterConfig;
+}
+
 const CUSTOM_ENTITY_DEF_ENTITY_DEFAULT = {
   label: 'Custom Entity Definition',
   label_plural: 'Custom Entity Definitions',
@@ -398,7 +429,7 @@ function withContractDefaults(masterConfig) {
 export class ConfigGeneratorEngine {
   constructor(masterConfig) {
     if (!masterConfig) throw new Error('[ConfigGeneratorEngine] masterConfig is required');
-    this.masterConfig = deepFreeze(withCustomEntityDefaults(withResourceManagementDefaults(withContractDefaults(withTimeTrackingDefaults(withInventoryDefaults(withProjectDefaults(withCrmDefaults(withWebhookDefaults(withMultiTenancyDefaults(masterConfig))))))))));
+    this.masterConfig = deepFreeze(withCustomEntityDefaults(withIntegrationDefaults(withResourceManagementDefaults(withContractDefaults(withTimeTrackingDefaults(withInventoryDefaults(withProjectDefaults(withCrmDefaults(withWebhookDefaults(withMultiTenancyDefaults(masterConfig)))))))))));
     this.specCache = new LRUCache(100);
     this.debugMode = false;
     this._plugins = new Map();

package/src/lib/integration-sync-engine.js

@@ -0,0 +1,82 @@
+import { list } from './busybase/store.js';
+import { validateWebhookUrl } from './webhook-engine.js';
+import { createLogger } from './logger.js';
+
+const log = createLogger('[IntegrationSync]');
+const REQUEST_TIMEOUT_MS = 15000;
+
+// Records changed since the connection's last sync, read through the SAME
+// {user}-scoped list() every interactive read path uses -- a sync job cannot
+// reach records outside its owner's org just because it runs unattended.
+export async function getRecordsChangedSince(entityName, sinceTs, user) {
+  const records = await list(entityName, {}, { user });
+  return records.filter(r => (r.updated_at || 0) > (sinceTs || 0));
+}
+
+function authHeader(connection) {
+  if (connection.auth_type === 'oauth_token') return { Authorization: `Bearer ${connection.credential}` };
+  return { 'X-Api-Key': connection.credential };
+}
+
+function mapRecordFields(record, mapping) {
+  if (!mapping || typeof mapping !== 'object') return record;
+  const mapped = {};
+  for (const [localField, remoteField] of Object.entries(mapping)) {
+    mapped[remoteField] = record[localField];
+  }
+  return mapped;
+}
+
+// connection.credential arrives here already decrypted (store.js decrypts on
+// every list()/get()) -- this function only reads it into the auth header,
+// never logs or persists it further.
+export async function syncRecordToIntegration(connection, entityName, record) {
+  const mappings = typeof connection.entity_mappings === 'string'
+    ? JSON.parse(connection.entity_mappings || '{}')
+    : (connection.entity_mappings || {});
+  const entityMapping = mappings[entityName];
+  if (!entityMapping) return { success: false, error: `No entity_mapping for "${entityName}"` };
+
+  const targetUrl = `${connection.base_url.replace(/\/$/, '')}/api/sync/${entityMapping}`;
+  // Same SSRF guard webhook-engine.js's dispatchWebhooks applies before any
+  // outbound fetch -- reused verbatim, not reimplemented, so a connection
+  // pointed at a private/loopback/link-local target is rejected here, before
+  // any network call is made.
+  const validation = await validateWebhookUrl(targetUrl);
+  if (!validation.ok) {
+    log.error(`integration_connection ${connection.id} target rejected: ${validation.error}`);
+    return { success: false, error: validation.error };
+  }
+
+  const payload = mapRecordFields(record, mappings[`${entityName}_fields`] || mappings.fields);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+  try {
+    const res = await fetch(targetUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', ...authHeader(connection) },
+      body: JSON.stringify(payload),
+      signal: controller.signal,
+    });
+    return { success: res.ok, statusCode: res.status, error: res.ok ? null : `HTTP ${res.status}` };
+  } catch (e) {
+    return { success: false, statusCode: 0, error: e.message };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+export async function syncConnection(connection, user) {
+  const mappings = typeof connection.entity_mappings === 'string'
+    ? JSON.parse(connection.entity_mappings || '{}')
+    : (connection.entity_mappings || {});
+  const entityNames = Object.keys(mappings).filter(k => !k.endsWith('_fields') && k !== 'fields');
+  const results = [];
+  for (const entityName of entityNames) {
+    const changed = await getRecordsChangedSince(entityName, connection.last_synced_at, user);
+    for (const record of changed) {
+      results.push({ entity: entityName, id: record.id, ...(await syncRecordToIntegration(connection, entityName, record)) });
+    }
+  }
+  return results;
+}

package/src/lib/scheduler-engine.js

@@ -47,6 +47,26 @@ async function runOneJob(job, nowTs) {
       targets = targets.filter(c => isWithinNoticeWindow(c, nowTs));
     }
 
+    // integration_sync is a distinct action shape from bulk field-updates --
+    // it fans out to an external system per connection rather than mutating
+    // Thatcher records, so it bypasses runBulkOperation entirely but still
+    // rides the SAME scheduled_job table/timer, not a new cron mechanism.
+    if (job.entity === 'integration_connection' && action?.type === 'integration_sync') {
+      const { syncConnection } = await import('./integration-sync-engine.js');
+      const syncResults = [];
+      for (const connection of targets) {
+        if (!connection.enabled) continue;
+        const results = await syncConnection(connection, owner);
+        syncResults.push(...results);
+        await update('integration_connection', connection.id, { last_synced_at: nowTs });
+      }
+      const nextRunAt = nowTs + job.interval_minutes * 60;
+      await update('scheduled_job', job.id, { last_run_at: nowTs, next_run_at: nextRunAt });
+      const succeeded = syncResults.filter(r => r.success).length;
+      log.info(`Job "${job.name}" ran: ${succeeded}/${syncResults.length} synced`);
+      return { job_id: job.id, ok: true, total: syncResults.length, succeeded, failed: syncResults.length - succeeded, results: syncResults };
+    }
+
     const ids = targets.map(r => r.id);
 
     let result = { ok: true, total: 0, succeeded: 0, failed: 0, results: [] };

package/src/server/server.js

@@ -493,6 +493,14 @@ async function handleGenericCrud(req, res, entity, id, action, thatcher, configE
           return;
         }
         permissionService.enforceEditPermissions(user, spec, body);
+        // The edit form renders encrypted:true fields blank with a "leave
+        // blank to keep unchanged" note -- a blank submission must not
+        // overwrite the stored secret, since update() spreads body directly
+        // into the patch and encryptFields only skips ABSENT keys, not
+        // empty-string values actually present in the object.
+        for (const [fieldKey, fieldDef] of Object.entries(spec.fields || {})) {
+          if (fieldDef.encrypted && body[fieldKey] === '') delete body[fieldKey];
+        }
         {
           // validateUpdate is the authoritative server-side check for both
           // generic field-type constraints AND entity-specific business

package/src/ui/entity-renderer.js

@@ -65,6 +65,11 @@ function roleLabel(r) {
 }
 
 function formatFieldValue(k, v, entityName, f) {
+  // encrypted:true fields decrypt on every list()/get() read (store.js) so
+  // the plaintext DOES reach this renderer -- redact here, the display
+  // boundary, rather than in store.js which legitimately serves the real
+  // value to business logic that needs it.
+  if (f?.encrypted) return v ? '<span class="pill pill-neutral">••••••••</span>' : '-'
   if (entityName === 'user' && k === 'role') return `<span class="pill pill-neutral">${roleLabel(v)}</span>`
   if (entityName === 'user' && k === 'status') {
     const cls = v === 'active' ? 'pill-success' : v === 'deleted' ? 'pill-danger' : 'pill-neutral'
@@ -346,6 +351,10 @@ export function renderEntityForm(entityName, item, spec, user, isNew = false, re
       const existingNote = existing?.filename ? `<div style="font-size:0.8rem;color:var(--color-text-muted)" data-existing-file="${k}">Current: ${esc(existing.filename)}</div>` : ''
       return `<div class="form-field">${lbl(k,f,f.required)}<input type="file" id="field-${k}" data-attachment="${k}" class="form-input" ${existing ? '' : req}/>${existingNote}<input type="hidden" id="field-${k}-value" name="${k}" value="${existing ? esc(JSON.stringify(existing)) : ''}"/></div>`
     }
+    if (f.encrypted) {
+      const encNote = val ? `<div style="font-size:0.8rem;color:var(--color-text-muted)">Currently set. Leave blank to keep unchanged.</div>` : ''
+      return `<div class="form-field">${lbl(k,f,f.required)}<input type="password" id="field-${k}" name="${k}" value="" class="form-input" autocomplete="new-password" placeholder="Enter ${esc((f.label||k).toLowerCase())}"/>${encNote}</div>`
+    }
     return `<div class="form-field">${lbl(k,f,f.required)}<input type="${type}" id="field-${k}" name="${k}" value="${esc(val)}" class="form-input" ${req} ${placeholder}/></div>`
   }).join('\n')
 

package/src/ui/page-handler.js

@@ -9,7 +9,7 @@ import { renderEngagementGrid } from '@/ui/engagement-grid-renderer.js';
 import { renderBoardView } from '@/ui/board-view-renderer.js';
 import { renderGridView } from '@/ui/grid-view-renderer.js';
 import { renderCalendarView, renderTimelineView } from '@/ui/calendar-view-renderer.js';
-import { renderCountByFieldReport, renderCountOverTimeReport, renderSumByFieldReport, renderRollupReport, renderInventoryForecastReport, renderDemandForecastReport, renderPivotReport } from '@/ui/report-renderer.js';
+import { renderCountByFieldReport, renderCountOverTimeReport, renderSumByFieldReport, renderRollupReport, renderInventoryForecastReport, renderDemandForecastReport, renderPivotReport, renderCohortOverTimeReport } from '@/ui/report-renderer.js';
 import { renderClientProgress } from '@/ui/client-progress-renderer.js';
 import { renderLetterWorkflow } from '@/ui/letter-workflow-renderer.js';
 import { renderAdvancedSearch } from '@/ui/advanced-search-renderer.js';
@@ -377,6 +377,12 @@ export async function handlePage(pathname, req, res) {
       const agg = params.get('agg') || 'count';
       return renderPivotReport(user, entityName, spec, items, rowField, colField, valueField, agg);
     }
+    if (report === 'cohort-over-time') {
+      const dateField = params.get('date_field') || '';
+      const cohortField = params.get('cohort_field') || '';
+      const granularity = params.get('granularity') || 'month';
+      return renderCohortOverTimeReport(user, entityName, spec, items, dateField, cohortField, granularity);
+    }
     if (report === 'rollup') {
       const refField = params.get('ref_field') || '';
       const rollupField = params.get('rollup_field') || '';

package/src/ui/report-renderer.js

@@ -329,6 +329,62 @@ export function renderPivotReport(user, entityName, spec, records, rowField, col
   return page(user, `${label} Report | Thatcher`, null, content);
 }
 
+// Trend-by-cohort: how a metric evolves over time (rows = date periods),
+// broken down by a cohort dimension (cols = cohort values) -- a genuinely
+// distinct shape from pivotByFields' point-in-time cross-tab and
+// countOverTime's ungrouped time series. Reuses pivotByFields directly
+// rather than a parallel implementation: a synthetic '__period' field is
+// pre-computed onto cloned records via the SAME bucketDateKey granularity
+// logic countOverTime already uses, and a matching plain-text spec.fields
+// entry is added so bucketLabel's enum-lookup branch is skipped (falls
+// through to String(rawValue), exactly what a period string needs) -- then
+// it's an ordinary two-field pivot. ISO-formatted period keys (YYYY-MM,
+// YYYY-MM-DD, YYYY-Www) sort correctly under pivotByFields' existing
+// lexical row sort, so no separate chronological sort is needed.
+export function cohortOverTime(records, spec, dateField, cohortField, granularity = 'month') {
+  const periodField = '__period';
+  const augmentedRecords = [];
+  let unresolved = 0;
+  for (const r of records) {
+    const key = bucketDateKey(r[dateField], granularity);
+    if (key === null) { unresolved++; continue; }
+    augmentedRecords.push({ ...r, [periodField]: key });
+  }
+  const augmentedSpec = { ...spec, fields: { ...spec.fields, [periodField]: { type: 'text', label: 'Period' } } };
+  const { rows, cols, cells } = pivotByFields(augmentedRecords, augmentedSpec, periodField, cohortField, null, 'count');
+  return { rows, cols, cells, unresolved };
+}
+
+export function renderCohortOverTimeReport(user, entityName, spec, records, dateField, cohortField, granularity) {
+  const label = getEntityLabel(spec, true) || entityName;
+  const dateFieldDef = spec.fields?.[dateField];
+  const cohortFieldDef = spec.fields?.[cohortField];
+  if (!dateFieldDef || !cohortFieldDef) {
+    const badField = !dateFieldDef ? dateField : cohortField;
+    return page(user, `${label} | Report`, null,
+      `<div class="page-header"><h1 class="page-title">${esc(label)}</h1></div>
+      <div class="report-empty-state">Unknown field "${esc(badField)}" for this entity.</div>`);
+  }
+  const { rows, cols, cells, unresolved } = cohortOverTime(records, spec, dateField, cohortField, granularity);
+  const notice = unresolved
+    ? `<div class="report-notice">${unresolved} record${unresolved === 1 ? '' : 's'} without a resolvable date excluded</div>`
+    : '';
+  const headerCells = cols.map(c => `<th>${esc(c)}</th>`).join('');
+  const bodyRows = rows.map((r, i) => {
+    const dataCells = cells[i].map(v => `<td style="text-align:right">${esc(String(v))}</td>`).join('');
+    return `<tr><td>${esc(r)}</td>${dataCells}</tr>`;
+  }).join('') || emptyState('No data to report on', 'bar-chart');
+  const content = `<div class="page-header">
+      <div><h1 class="page-title">${esc(label)}: ${esc(cohortFieldDef.label || cohortField)} over time</h1><p class="page-subtitle">${records.length} total records, grouped by ${esc(granularity)}</p></div>
+    </div>
+    ${notice}
+    <div class="table-wrap"><table class="data-table">
+      <thead><tr><th>Period</th>${headerCells}</tr></thead>
+      <tbody>${bodyRows}</tbody>
+    </table></div>`;
+  return page(user, `${label} Report | Thatcher`, null, content);
+}
+
 export function renderDemandForecastReport(user, spec, buckets, totalOpportunities) {
   const label = getEntityLabel(spec, true) || 'Opportunities';
   const valueFieldDef = spec.fields?.value || { type: 'currency' };