thatcher 1.0.90 → 1.0.92

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "thatcher",
-  "version": "1.0.90",
+  "version": "1.0.92",
   "description": "A config-driven application framework for building data-intensive web apps without code.",
   "main": "src/index.js",
   "type": "module",

package/src/lib/demand-forecast.js

@@ -33,6 +33,30 @@ function weightedValueOf(opportunity) {
   return (value * probability) / 100;
 }
 
+// The historical mirror of projectDemandByMonth: past months' ACTUAL won
+// value, not future open-pipeline projection. This is the time series a
+// trend prediction (statistical-forecast.js's linearRegression) needs --
+// projectDemandByMonth's forward-looking open-pipeline sum is the wrong
+// input for "predict next period from history" since it contains no past
+// data points to fit a trend against.
+export function historicalWonValueByMonth(opportunities, nowSeconds = Math.floor(Date.now() / 1000), monthsBack = 6) {
+  const currentBucketKey = monthBucketKey(nowSeconds);
+  const buckets = new Map();
+
+  for (const opp of opportunities) {
+    if (opp.stage !== 'won') continue;
+    const closeDate = opp.actual_close_date ?? opp.expected_close_date;
+    if (closeDate == null) continue;
+    const bucketKey = monthBucketKey(Number(closeDate));
+    if (bucketKey > currentBucketKey) continue;
+    const value = Number(opp.value) || 0;
+    buckets.set(bucketKey, (buckets.get(bucketKey) || 0) + value);
+  }
+
+  const sorted = [...buckets.entries()].sort((a, b) => a[0].localeCompare(b[0]));
+  return sorted.length > monthsBack ? sorted.slice(-monthsBack) : sorted;
+}
+
 export function projectDemandByMonth(opportunities, nowSeconds = Math.floor(Date.now() / 1000)) {
   const currentBucketKey = monthBucketKey(nowSeconds);
   const buckets = new Map();

package/src/lib/insights-engine.js

@@ -7,6 +7,7 @@
 import { daysUntilStockout, isReorderDue } from './inventory-forecast.js';
 import { daysUntilExpiry } from './contract-expiry.js';
 import { userCommittedHours, DEFAULT_WEEKLY_CAPACITY_HOURS } from './resource-capacity.js';
+import { zScoreAnomaly } from './statistical-forecast.js';
 
 const SEVERITY_ORDER = { critical: 0, warning: 1, info: 2 };
 
@@ -87,6 +88,36 @@ export function pipelineDropInsights(monthlyBuckets) {
   return results;
 }
 
+// Statistical, not rule-based: flags a product whose most recent day's
+// consumption is a z-score outlier relative to ITS OWN historical daily
+// consumption -- the same absolute consumption number can be perfectly
+// normal for a high-variance product and anomalous for a low-variance one,
+// which stockoutRiskInsights' fixed day-threshold cannot express since it
+// only reasons about days-until-stockout, never about whether today's rate
+// itself is unusual for this specific product.
+export function statisticalAnomalyInsights(products, movementsByProductId) {
+  const results = [];
+  for (const product of products) {
+    const movements = movementsByProductId[product.id] || [];
+    const byDay = new Map();
+    for (const m of movements) {
+      if (Number(m.quantity) >= 0) continue;
+      const day = new Date((Number(m.created_at) || 0) * 1000).toISOString().slice(0, 10);
+      byDay.set(day, (byDay.get(day) || 0) + Math.abs(Number(m.quantity) || 0));
+    }
+    const days = [...byDay.entries()].sort((a, b) => a[0].localeCompare(b[0]));
+    if (days.length < 3) continue;
+    const latestValue = days[days.length - 1][1];
+    const history = days.slice(0, -1).map(([, v]) => v);
+    const { isAnomaly, zScore } = zScoreAnomaly(history, latestValue, 2);
+    if (isAnomaly) {
+      const direction = latestValue > (history.reduce((a, b) => a + b, 0) / history.length) ? 'above' : 'below';
+      results.push(insight('warning', 'anomaly', `${product.name || product.id}'s latest daily consumption is statistically ${direction} its historical pattern (z-score ${zScore.toFixed(2)})`, 'product', product.id));
+    }
+  }
+  return results;
+}
+
 export function sortBySeverity(insights) {
   return [...insights].sort((a, b) => (SEVERITY_ORDER[a.severity] ?? 3) - (SEVERITY_ORDER[b.severity] ?? 3));
 }
@@ -109,6 +140,7 @@ export async function generateInsights(user, entityType = null) {
       movementsByProductId[product.id] = await list('stock_movement', { product_id: product.id });
     }
     results.push(...stockoutRiskInsights(products, movementsByProductId));
+    results.push(...statisticalAnomalyInsights(products, movementsByProductId));
   }
 
   if (!entityType || entityType === 'contract') {

package/src/lib/statistical-forecast.js

@@ -0,0 +1,54 @@
+// Data-derived predictive math, distinct from insights-engine.js's rule-based
+// fixed-threshold checks (days<=14, pct>=90, dropPct>30) -- both functions
+// here derive their parameters FROM the input series itself (slope/intercept,
+// mean/stddev) rather than from a hardcoded constant, the standard
+// lightweight statistical technique available without an ML training
+// pipeline. Same small-pure-module shape as contract-expiry.js/
+// inventory-forecast.js/resource-capacity.js.
+
+// Least-squares fit over [x, y] points. x is typically a period index
+// (0, 1, 2...), y the metric value at that period.
+export function linearRegression(points) {
+  const n = points.length;
+  if (n < 2) return null;
+  let sumX = 0, sumY = 0, sumXY = 0, sumXX = 0;
+  for (const [x, y] of points) {
+    sumX += x; sumY += y; sumXY += x * y; sumXX += x * x;
+  }
+  const denominator = n * sumXX - sumX * sumX;
+  if (denominator === 0) return null;
+  const slope = (n * sumXY - sumX * sumY) / denominator;
+  const intercept = (sumY - slope * sumX) / n;
+  return { slope, intercept };
+}
+
+export function predictNext(points) {
+  const fit = linearRegression(points);
+  if (!fit) return null;
+  const nextX = Math.max(...points.map(([x]) => x)) + 1;
+  return fit.slope * nextX + fit.intercept;
+}
+
+function mean(values) {
+  return values.reduce((a, b) => a + b, 0) / values.length;
+}
+
+function stddev(values, avg) {
+  const variance = values.reduce((sum, v) => sum + (v - avg) ** 2, 0) / values.length;
+  return Math.sqrt(variance);
+}
+
+// Flags latestValue as anomalous relative to series' OWN historical mean/
+// stddev -- a value that would be normal for a high-variance series and
+// anomalous for a low-variance one, which a fixed threshold cannot express.
+// stddev===0 (a perfectly flat history) means ANY deviation is infinitely
+// many stddevs away, so any latestValue != mean is reported as anomalous
+// with zScore=Infinity rather than dividing by zero into NaN.
+export function zScoreAnomaly(series, latestValue, threshold = 2) {
+  if (series.length < 2) return { isAnomaly: false, zScore: null, mean: null, stddev: null };
+  const avg = mean(series);
+  const sd = stddev(series, avg);
+  if (sd === 0) return { isAnomaly: latestValue !== avg, zScore: latestValue === avg ? 0 : Infinity, mean: avg, stddev: 0 };
+  const zScore = (latestValue - avg) / sd;
+  return { isAnomaly: Math.abs(zScore) >= threshold, zScore, mean: avg, stddev: sd };
+}

package/src/ui/page-handler.js

@@ -9,7 +9,7 @@ import { renderEngagementGrid } from '@/ui/engagement-grid-renderer.js';
 import { renderBoardView } from '@/ui/board-view-renderer.js';
 import { renderGridView } from '@/ui/grid-view-renderer.js';
 import { renderCalendarView, renderTimelineView } from '@/ui/calendar-view-renderer.js';
-import { renderCountByFieldReport, renderCountOverTimeReport, renderSumByFieldReport, renderRollupReport, renderInventoryForecastReport, renderDemandForecastReport, renderPivotReport } from '@/ui/report-renderer.js';
+import { renderCountByFieldReport, renderCountOverTimeReport, renderSumByFieldReport, renderRollupReport, renderInventoryForecastReport, renderDemandForecastReport, renderPivotReport, renderCohortOverTimeReport } from '@/ui/report-renderer.js';
 import { renderClientProgress } from '@/ui/client-progress-renderer.js';
 import { renderLetterWorkflow } from '@/ui/letter-workflow-renderer.js';
 import { renderAdvancedSearch } from '@/ui/advanced-search-renderer.js';
@@ -377,6 +377,12 @@ export async function handlePage(pathname, req, res) {
       const agg = params.get('agg') || 'count';
       return renderPivotReport(user, entityName, spec, items, rowField, colField, valueField, agg);
     }
+    if (report === 'cohort-over-time') {
+      const dateField = params.get('date_field') || '';
+      const cohortField = params.get('cohort_field') || '';
+      const granularity = params.get('granularity') || 'month';
+      return renderCohortOverTimeReport(user, entityName, spec, items, dateField, cohortField, granularity);
+    }
     if (report === 'rollup') {
       const refField = params.get('ref_field') || '';
       const rollupField = params.get('rollup_field') || '';
@@ -415,9 +421,18 @@ export async function handlePage(pathname, req, res) {
       // entity route uses) -- no new unscoped query, and each item already
       // carries a computed weighted_value from list()'s own formula-field
       // pass, reused directly rather than recomputed.
-      const { projectDemandByMonth } = await import('@/lib/demand-forecast.js');
+      const { projectDemandByMonth, historicalWonValueByMonth } = await import('@/lib/demand-forecast.js');
+      const { predictNext } = await import('@/lib/statistical-forecast.js');
       const buckets = projectDemandByMonth(items);
-      return renderDemandForecastReport(user, spec, buckets, items.length);
+      // Trend prediction over PAST won-value, distinct from `buckets`'
+      // forward-looking open-pipeline sum -- reuses the same {user}-scoped
+      // `items` list, no new query. Fewer than 2 historical months means no
+      // regression can be fit (linearRegression's own null-return case).
+      const historical = historicalWonValueByMonth(items);
+      const predictedNextValue = historical.length >= 2
+        ? predictNext(historical.map(([, v], i) => [i, v]))
+        : null;
+      return renderDemandForecastReport(user, spec, buckets, items.length, predictedNextValue);
     }
     const view = params.get('view');
     if (view === 'board') return renderBoardView(user, entityName, spec, items);

package/src/ui/report-renderer.js

@@ -329,17 +329,82 @@ export function renderPivotReport(user, entityName, spec, records, rowField, col
   return page(user, `${label} Report | Thatcher`, null, content);
 }
 
-export function renderDemandForecastReport(user, spec, buckets, totalOpportunities) {
+// Trend-by-cohort: how a metric evolves over time (rows = date periods),
+// broken down by a cohort dimension (cols = cohort values) -- a genuinely
+// distinct shape from pivotByFields' point-in-time cross-tab and
+// countOverTime's ungrouped time series. Reuses pivotByFields directly
+// rather than a parallel implementation: a synthetic '__period' field is
+// pre-computed onto cloned records via the SAME bucketDateKey granularity
+// logic countOverTime already uses, and a matching plain-text spec.fields
+// entry is added so bucketLabel's enum-lookup branch is skipped (falls
+// through to String(rawValue), exactly what a period string needs) -- then
+// it's an ordinary two-field pivot. ISO-formatted period keys (YYYY-MM,
+// YYYY-MM-DD, YYYY-Www) sort correctly under pivotByFields' existing
+// lexical row sort, so no separate chronological sort is needed.
+export function cohortOverTime(records, spec, dateField, cohortField, granularity = 'month') {
+  const periodField = '__period';
+  const augmentedRecords = [];
+  let unresolved = 0;
+  for (const r of records) {
+    const key = bucketDateKey(r[dateField], granularity);
+    if (key === null) { unresolved++; continue; }
+    augmentedRecords.push({ ...r, [periodField]: key });
+  }
+  const augmentedSpec = { ...spec, fields: { ...spec.fields, [periodField]: { type: 'text', label: 'Period' } } };
+  const { rows, cols, cells } = pivotByFields(augmentedRecords, augmentedSpec, periodField, cohortField, null, 'count');
+  return { rows, cols, cells, unresolved };
+}
+
+export function renderCohortOverTimeReport(user, entityName, spec, records, dateField, cohortField, granularity) {
+  const label = getEntityLabel(spec, true) || entityName;
+  const dateFieldDef = spec.fields?.[dateField];
+  const cohortFieldDef = spec.fields?.[cohortField];
+  if (!dateFieldDef || !cohortFieldDef) {
+    const badField = !dateFieldDef ? dateField : cohortField;
+    return page(user, `${label} | Report`, null,
+      `<div class="page-header"><h1 class="page-title">${esc(label)}</h1></div>
+      <div class="report-empty-state">Unknown field "${esc(badField)}" for this entity.</div>`);
+  }
+  const { rows, cols, cells, unresolved } = cohortOverTime(records, spec, dateField, cohortField, granularity);
+  const notice = unresolved
+    ? `<div class="report-notice">${unresolved} record${unresolved === 1 ? '' : 's'} without a resolvable date excluded</div>`
+    : '';
+  const headerCells = cols.map(c => `<th>${esc(c)}</th>`).join('');
+  const bodyRows = rows.map((r, i) => {
+    const dataCells = cells[i].map(v => `<td style="text-align:right">${esc(String(v))}</td>`).join('');
+    return `<tr><td>${esc(r)}</td>${dataCells}</tr>`;
+  }).join('') || emptyState('No data to report on', 'bar-chart');
+  const content = `<div class="page-header">
+      <div><h1 class="page-title">${esc(label)}: ${esc(cohortFieldDef.label || cohortField)} over time</h1><p class="page-subtitle">${records.length} total records, grouped by ${esc(granularity)}</p></div>
+    </div>
+    ${notice}
+    <div class="table-wrap"><table class="data-table">
+      <thead><tr><th>Period</th>${headerCells}</tr></thead>
+      <tbody>${bodyRows}</tbody>
+    </table></div>`;
+  return page(user, `${label} Report | Thatcher`, null, content);
+}
+
+// predictedNextValue (optional) is a statistical trend prediction --
+// distinct from `buckets`' own totalProjected sum, which is a sum of
+// already-weighted open pipeline, not a prediction fitted from historical
+// won-value data. null/undefined renders no trend line, same page shape as
+// before this parameter existed.
+export function renderDemandForecastReport(user, spec, buckets, totalOpportunities, predictedNextValue = null) {
   const label = getEntityLabel(spec, true) || 'Opportunities';
   const valueFieldDef = spec.fields?.value || { type: 'currency' };
   const totalProjected = buckets.reduce((sum, [, v]) => sum + v, 0);
   const notice = !buckets.length
     ? `<div class="report-notice">No open opportunities with a future expected close date</div>`
     : '';
+  const trendNotice = predictedNextValue != null
+    ? `<div class="report-notice">Statistical trend prediction (linear regression over historical won value): next month ~${esc(formatSumValue(predictedNextValue, valueFieldDef))}</div>`
+    : '';
   const content = `<div class="page-header">
       <div><h1 class="page-title">${esc(label)}: Demand Forecast</h1><p class="page-subtitle">${totalOpportunities} total opportunities, ${esc(formatSumValue(totalProjected, valueFieldDef))} projected across ${buckets.length} future month${buckets.length === 1 ? '' : 's'}</p></div>
     </div>
     ${notice}
+    ${trendNotice}
     ${sumBarChart(buckets, valueFieldDef)}`;
   return page(user, `Demand Forecast | Thatcher`, null, content);
 }