thatcher 1.0.4 → 1.0.6

package/src/lib/hyperformula-service.js

@@ -0,0 +1,304 @@
+/**
+ * HyperFormula Compute Engine - Spreadsheet formula evaluation for Thatcher
+ * Provides calculated fields, business rules, and data transformations
+ */
+
+import { HyperFormula } from 'hyperformula';
+import { createLogger } from './logger.js';
+
+const logger = createLogger('[HyperFormula]');
+
+const _sheets = new Map();
+const _evaluationLog = [];
+const MAX_LOG_SIZE = 1000;
+
+export class HyperFormulaService {
+  constructor(options = {}) {
+    this.options = {
+      licenseKey: options.licenseKey || 'gpl-v3',
+      maxRows: options.maxRows || 100000,
+      maxColumns: options.maxColumns || 1000,
+      useColumnIndex: options.useColumnIndex !== false,
+      useStats: options.useStats !== false,
+      ...options,
+    };
+    this._instance = null;
+    this._customFunctions = new Map();
+    this._sheetMeta = new Map();
+  }
+
+  async init() {
+    if (this._instance) return this;
+
+    this._instance = HyperFormula.buildEmpty({
+      licenseKey: this.options.licenseKey,
+      maxRows: this.options.maxRows,
+      maxColumns: this.options.maxColumns,
+      useColumnIndex: this.options.useColumnIndex,
+      useStats: this.options.useStats,
+    });
+
+    this._registerCustomFunctions();
+
+    if (globalThis.__debug__) {
+      globalThis.__debug__.expose('formula', {
+        sheets: () => this.listSheets(),
+        sheet: (name) => this.getSheetInfo(name),
+        instance: () => this._instance,
+        stats: () => this.getStats(),
+        log: () => [..._evaluationLog],
+        customFunctions: () => Array.from(this._customFunctions.keys()),
+      }, 'HyperFormula Service');
+    }
+
+    logger.info('HyperFormula initialized');
+    return this;
+  }
+
+  _registerCustomFunctions() {
+    const thatcherFunctions = [
+      { name: 'DB_LOOKUP', parameters: 4 },
+      { name: 'DB_COUNT', parameters: 4 },
+      { name: 'DB_SUM', parameters: 4 },
+      { name: 'DB_FILTER', parameters: 3 },
+      { name: 'WORKFLOW_STATE', parameters: 1 },
+      { name: 'USER_ROLE', parameters: 0 },
+      { name: 'NOW_TS', parameters: 0 },
+      { name: 'UUID', parameters: 0 },
+    ];
+
+    for (const fn of thatcherFunctions) {
+      this._customFunctions.set(fn.name, fn);
+    }
+  }
+
+  createSheet(name, data = []) {
+    if (!this._instance) throw new Error('HyperFormula not initialized');
+
+    this._instance.addSheet(name);
+    const sheetId = this._instance.getSheetId(name);
+
+    if (data.length > 0) {
+      this._instance.setSheetContent(sheetId, data);
+    }
+
+    this._sheetMeta.set(name, {
+      id: sheetId,
+      created: Date.now(),
+      rowCount: data.length,
+      colCount: data.length > 0 ? Math.max(...data.map(r => r.length)) : 0,
+    });
+
+    logger.info('Sheet created', { name, sheetId });
+    return { name, sheetId };
+  }
+
+  removeSheet(name) {
+    const meta = this._sheetMeta.get(name);
+    if (!meta) throw new Error(`Sheet "${name}" not found`);
+
+    this._instance.removeSheet(meta.id);
+    this._sheetMeta.delete(name);
+    logger.info('Sheet removed', { name });
+  }
+
+  renameSheet(oldName, newName) {
+    const meta = this._sheetMeta.get(oldName);
+    if (!meta) throw new Error(`Sheet "${oldName}" not found`);
+
+    this._instance.renameSheet(meta.id, newName);
+    this._sheetMeta.delete(oldName);
+    this._sheetMeta.set(newName, { ...meta, name: newName });
+  }
+
+  setCellContents(name, cellAddress, value) {
+    const meta = this._sheetMeta.get(name);
+    if (!meta) throw new Error(`Sheet "${name}" not found`);
+
+    const start = performance.now();
+    this._instance.setCellContents({ sheet: meta.id, ...cellAddress }, value);
+    const duration = performance.now() - start;
+
+    this._logEvaluation('setCellContents', { sheet: name, cell: cellAddress, value, duration });
+  }
+
+  getCellValue(name, cellAddress) {
+    const meta = this._sheetMeta.get(name);
+    if (!meta) throw new Error(`Sheet "${name}" not found`);
+
+    const start = performance.now();
+    const value = this._instance.getCellValue({ sheet: meta.id, ...cellAddress });
+    const duration = performance.now() - start;
+
+    this._logEvaluation('getCellValue', { sheet: name, cell: cellAddress, value, duration });
+    return value;
+  }
+
+  getSheetValues(name) {
+    const meta = this._sheetMeta.get(name);
+    if (!meta) throw new Error(`Sheet "${name}" not found`);
+
+    const start = performance.now();
+    const values = this._instance.getSheetValues(meta.id);
+    const duration = performance.now() - start;
+
+    this._logEvaluation('getSheetValues', { sheet: name, rowCount: values.length, duration });
+    return values;
+  }
+
+  getRangeValues(name, startRow, startCol, endRow, endCol) {
+    const meta = this._sheetMeta.get(name);
+    if (!meta) throw new Error(`Sheet "${name}" not found`);
+
+    const start = performance.now();
+    const values = this._instance.getRangeValue({
+      sheet: meta.id,
+      startRow,
+      startCol,
+      endRow,
+      endCol,
+    });
+    const duration = performance.now() - start;
+
+    this._logEvaluation('getRangeValues', { sheet: name, range: { startRow, startCol, endRow, endCol }, duration });
+    return values;
+  }
+
+  setSheetContent(name, data) {
+    const meta = this._sheetMeta.get(name);
+    if (!meta) throw new Error(`Sheet "${name}" not found`);
+
+    const start = performance.now();
+    this._instance.setSheetContent(meta.id, data);
+    const duration = performance.now() - start;
+
+    this._sheetMeta.set(name, {
+      ...meta,
+      rowCount: data.length,
+      colCount: data.length > 0 ? Math.max(...data.map(r => r.length)) : 0,
+    });
+
+    this._logEvaluation('setSheetContent', { sheet: name, rowCount: data.length, duration });
+  }
+
+  evaluateFormula(formula) {
+    const start = performance.now();
+
+    const tempSheet = this.createSheet(`_temp_${Date.now()}`);
+    try {
+      this.setCellContents(tempSheet.name, { row: 0, col: 0 }, [[formula]]);
+      const result = this.getCellValue(tempSheet.name, { row: 0, col: 0 });
+      const duration = performance.now() - start;
+
+      this._logEvaluation('evaluateFormula', { formula, result, duration });
+      return { result, duration };
+    } finally {
+      this.removeSheet(tempSheet.name);
+    }
+  }
+
+  validateFormula(formula) {
+    try {
+      const ast = HyperFormula.buildFromArray([[formula]], { licenseKey: 'gpl-v3' });
+      const errors = ast.getAllFormulas().filter(f => f?.error);
+      return { valid: errors.length === 0, errors };
+    } catch (e) {
+      return { valid: false, errors: [e.message] };
+    }
+  }
+
+  getCellFormula(name, cellAddress) {
+    const meta = this._sheetMeta.get(name);
+    if (!meta) throw new Error(`Sheet "${name}" not found`);
+
+    return this._instance.getCellFormula({ sheet: meta.id, ...cellAddress });
+  }
+
+  getDependencies(name, cellAddress) {
+    const meta = this._sheetMeta.get(name);
+    if (!meta) throw new Error(`Sheet "${name}" not found`);
+
+    return this._instance.dependencies.getDependencies({ sheet: meta.id, ...cellAddress });
+  }
+
+  undo() {
+    return this._instance.undo();
+  }
+
+  redo() {
+    return this._instance.redo();
+  }
+
+  listSheets() {
+    return Array.from(this._sheetMeta.entries()).map(([name, meta]) => ({
+      name,
+      ...meta,
+    }));
+  }
+
+  getSheetInfo(name) {
+    const meta = this._sheetMeta.get(name);
+    if (!meta) return null;
+
+    const dimensions = this._instance.getSheetDimensions(meta.id);
+    return {
+      ...meta,
+      dimensions,
+      filledRange: this._instance.getFilledRange(meta.id),
+    };
+  }
+
+  getStats() {
+    return {
+      sheetCount: this._sheetMeta.size,
+      customFunctions: this._customFunctions.size,
+      evaluationLogSize: _evaluationLog.length,
+      version: HyperFormula.version,
+    };
+  }
+
+  _logEvaluation(operation, details) {
+    _evaluationLog.push({
+      operation,
+      timestamp: Date.now(),
+      ...details,
+    });
+
+    while (_evaluationLog.length > MAX_LOG_SIZE) {
+      _evaluationLog.shift();
+    }
+  }
+
+  getInstance() {
+    return this._instance;
+  }
+
+  async close() {
+    if (this._instance) {
+      this._instance.destroy();
+      this._instance = null;
+    }
+    this._sheetMeta.clear();
+    _evaluationLog.length = 0;
+  }
+}
+
+let _hfService = null;
+
+export async function createHyperFormulaService(options = {}) {
+  if (!_hfService) {
+    _hfService = new HyperFormulaService(options);
+    await _hfService.init();
+  }
+  return _hfService;
+}
+
+export function getHyperFormulaService() {
+  return _hfService;
+}
+
+export function getHyperFormulaInstance() {
+  return _hfService?._instance;
+}
+
+export default HyperFormulaService;

package/src/lib/metrics-collector.js

@@ -1,5 +1,12 @@
 /**
  * Metrics Collector - Request metrics and stats
+ *
+ * Thatcher keeps its original lightweight request-summary API
+ * (recordRequest/getMetrics/getSummary/resetMetrics) for backward compat.
+ * Merged in the richer per-channel collector API consumed by
+ * resource-monitor / db-monitor / request-tracker / metrics & health routes
+ * (recordError/recordDatabase/recordResource/recordCustom, getAllMetrics,
+ * clearMetrics, getStats, and the per-channel getters).
  */
 
 const metrics = {
@@ -45,6 +52,15 @@ export function recordRequest(endpoint, method, durationMs, statusCode) {
   if (statusCode >= 400) {
     metrics.errors.push(record);
   }
+
+  // Mirror into the per-channel request collector used by getAllMetrics().
+  const key = `${method}:${endpoint}`;
+  if (!channels.requests.has(key)) {
+    channels.requests.set(key, []);
+  }
+  const arr = channels.requests.get(key);
+  arr.push({ duration: durationMs, status: statusCode, ts: Date.now() });
+  if (arr.length > 1000) arr.shift();
 }
 
 /**
@@ -99,4 +115,184 @@ export function resetMetrics() {
   metrics.errors = [];
   _requestCount = 0;
   metrics.startTime = Date.now();
+  clearMetrics();
+}
+
+// ---------------------------------------------------------------------------
+// Per-channel collector API (ported from moonlanding feature set)
+// Consumed by resource-monitor.js, db-monitor.js, request-tracker.js,
+// api/metrics/route.js, api/health/route.js, monitoring-init.js.
+// ---------------------------------------------------------------------------
+
+const channels = {
+  requests: new Map(),
+  errors: new Map(),
+  database: new Map(),
+  resources: new Map(),
+  custom: new Map(),
+};
+
+const errorCounts = new Map();
+const resourceSamples = [];
+
+/**
+ * Record an error against an endpoint.
+ */
+export function recordError(path, method, error, stack) {
+  const key = `${method}:${path}`;
+  if (!channels.errors.has(key)) {
+    channels.errors.set(key, []);
+  }
+  const arr = channels.errors.get(key);
+  arr.push({ error, stack, ts: Date.now() });
+  if (arr.length > 100) arr.shift();
+
+  const countKey = error || 'unknown';
+  errorCounts.set(countKey, (errorCounts.get(countKey) || 0) + 1);
+}
+
+/**
+ * Record a database operation timing.
+ */
+export function recordDatabase(operation, duration, query) {
+  const key = operation;
+  if (!channels.database.has(key)) {
+    channels.database.set(key, []);
+  }
+  const arr = channels.database.get(key);
+  arr.push({ duration, query: query?.substring(0, 200), ts: Date.now() });
+  if (arr.length > 1000) arr.shift();
+}
+
+/**
+ * Record a resource sample (cpu/memory/disk).
+ */
+export function recordResource(cpu, memory, disk) {
+  resourceSamples.push({ cpu, memory, disk, ts: Date.now() });
+  if (resourceSamples.length > 1000) resourceSamples.shift();
+}
+
+/**
+ * Record a custom metric value with optional tags.
+ */
+export function recordCustom(name, value, tags = {}) {
+  const key = name;
+  if (!channels.custom.has(key)) {
+    channels.custom.set(key, []);
+  }
+  const arr = channels.custom.get(key);
+  arr.push({ value, tags, ts: Date.now() });
+  if (arr.length > 1000) arr.shift();
+}
+
+function getPercentile(values, percentile) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.ceil(sorted.length * percentile) - 1;
+  return sorted[Math.max(0, index)];
+}
+
+/**
+ * Compute count/min/max/avg/p50/p95/p99 stats over a data array.
+ */
+export function getStats(dataArray, field = 'duration') {
+  if (!dataArray || dataArray.length === 0) {
+    return { count: 0, min: 0, max: 0, avg: 0, p50: 0, p95: 0, p99: 0 };
+  }
+
+  const values = dataArray.map(d => d[field] || 0).filter(v => typeof v === 'number');
+  if (values.length === 0) {
+    return { count: 0, min: 0, max: 0, avg: 0, p50: 0, p95: 0, p99: 0 };
+  }
+
+  const sum = values.reduce((a, b) => a + b, 0);
+  return {
+    count: values.length,
+    min: Math.min(...values),
+    max: Math.max(...values),
+    avg: sum / values.length,
+    p50: getPercentile(values, 0.50),
+    p95: getPercentile(values, 0.95),
+    p99: getPercentile(values, 0.99),
+  };
+}
+
+export function getRequestMetrics() {
+  const result = {};
+  for (const [key, data] of channels.requests.entries()) {
+    result[key] = getStats(data, 'duration');
+  }
+  return result;
+}
+
+export function getErrorMetrics() {
+  const result = {
+    byEndpoint: {},
+    byCause: Object.fromEntries(errorCounts),
+  };
+  for (const [key, data] of channels.errors.entries()) {
+    result.byEndpoint[key] = { count: data.length, recent: data.slice(-5) };
+  }
+  return result;
+}
+
+export function getDatabaseMetrics() {
+  const result = {};
+  for (const [key, data] of channels.database.entries()) {
+    result[key] = getStats(data, 'duration');
+  }
+  return result;
+}
+
+export function getResourceMetrics() {
+  if (resourceSamples.length === 0) return null;
+  const cpuVals = resourceSamples.map(s => s.cpu).filter(v => v != null);
+  const memVals = resourceSamples.map(s => s.memory).filter(v => v != null);
+  const diskVals = resourceSamples.map(s => s.disk).filter(v => v != null);
+
+  return {
+    cpu: cpuVals.length > 0 ? getStats(cpuVals.map(v => ({ duration: v }))) : null,
+    memory: memVals.length > 0 ? getStats(memVals.map(v => ({ duration: v }))) : null,
+    disk: diskVals.length > 0 ? getStats(diskVals.map(v => ({ duration: v }))) : null,
+  };
+}
+
+/**
+ * Aggregate snapshot across all channels.
+ */
+export function getAllMetrics() {
+  return {
+    requests: getRequestMetrics(),
+    errors: getErrorMetrics(),
+    database: getDatabaseMetrics(),
+    resources: getResourceMetrics(),
+    custom: Object.fromEntries(
+      Array.from(channels.custom.entries()).map(([k, v]) => [k, getStats(v, 'value')])
+    ),
+    timestamp: Date.now(),
+  };
+}
+
+/**
+ * Clear the per-channel collector state.
+ */
+export function clearMetrics() {
+  channels.requests.clear();
+  channels.errors.clear();
+  channels.database.clear();
+  channels.custom.clear();
+  resourceSamples.length = 0;
+  errorCounts.clear();
+}
+
+if (typeof globalThis !== 'undefined') {
+  globalThis.__metrics = {
+    recordRequest,
+    recordError,
+    recordDatabase,
+    recordResource,
+    recordCustom,
+    getAllMetrics,
+    clearMetrics,
+  };
 }

package/src/lib/observability-bootstrap.js

@@ -0,0 +1,121 @@
+/**
+ * Observability Bootstrap - Initializes all observability subsystems on server startup
+ * Starts tracing, registers debug exposures, begins metric collection
+ */
+
+import { tracer } from './tracing.js';
+import { perfProfiler } from './perf-profiler.js';
+import { debugRegistry } from './debug-registry.js';
+import { createExportSink } from './export-sink.js';
+import { withHookTracing } from './request-tracing.js';
+import { hookEngine } from './hook-engine.js';
+import { createLogger } from './logger.js';
+
+const logger = createLogger('[ObservabilityBootstrap]');
+
+export async function bootstrapObservability(options = {}) {
+  const results = {
+    tracing: false,
+    profiler: false,
+    registry: false,
+    export: false,
+    hookTracing: false,
+    errors: [],
+  };
+
+  try {
+    debugRegistry.expose('system', {
+      pid: () => process.pid,
+      uptime: () => process.uptime(),
+      memory: () => process.memoryUsage(),
+      cpuUsage: () => process.cpuUsage(),
+      version: () => process.version,
+      platform: () => process.platform,
+    }, 'System information');
+
+    results.registry = true;
+    logger.info('Debug registry initialized');
+  } catch (error) {
+    results.errors.push({ subsystem: 'registry', error: error.message });
+    logger.error('Registry bootstrap failed', { error: error.message });
+  }
+
+  try {
+    if (globalThis.__debug__) {
+      globalThis.__debug__.expose('tracing', {
+        activeSpans: () => tracer.getActiveSpans(),
+        recentTraces: (limit) => tracer.getRecentTraces(limit),
+        traceById: (id) => tracer.getTraceById(id),
+        stats: () => tracer.getStats(),
+        currentTraceId: () => tracer.getCurrentTraceId(),
+      }, 'Tracing Core');
+    }
+
+    results.tracing = true;
+    logger.info('Tracing initialized');
+  } catch (error) {
+    results.errors.push({ subsystem: 'tracing', error: error.message });
+    logger.error('Tracing bootstrap failed', { error: error.message });
+  }
+
+  try {
+    if (globalThis.__debug__) {
+      globalThis.__debug__.expose('perf', {
+        stats: (op) => op ? perfProfiler.getStats(op) : perfProfiler.getAllStats(),
+        slow: () => perfProfiler.getSlowOperations(),
+        regressions: () => perfProfiler.detectRegressions(),
+        alerts: () => perfProfiler.getAlerts(),
+        thresholds: () => Object.fromEntries(perfProfiler._thresholds),
+      }, 'Performance Profiler');
+    }
+
+    results.profiler = true;
+    logger.info('Performance profiler initialized');
+  } catch (error) {
+    results.errors.push({ subsystem: 'profiler', error: error.message });
+    logger.error('Profiler bootstrap failed', { error: error.message });
+  }
+
+  try {
+    const exportTarget = options.exportTarget || process.env.OBSERVABILITY_EXPORT_TARGET;
+    if (exportTarget) {
+      await createExportSink({
+        target: exportTarget,
+        url: options.exportUrl || process.env.OBSERVABILITY_EXPORT_URL,
+        batchSize: options.batchSize,
+        batchIntervalMs: options.batchIntervalMs,
+      });
+      results.export = true;
+      logger.info('Export sink initialized', { target: exportTarget });
+    }
+  } catch (error) {
+    results.errors.push({ subsystem: 'export', error: error.message });
+    logger.error('Export sink bootstrap failed', { error: error.message });
+  }
+
+  try {
+    withHookTracing(hookEngine);
+    results.hookTracing = true;
+    logger.info('Hook tracing initialized');
+  } catch (error) {
+    results.errors.push({ subsystem: 'hookTracing', error: error.message });
+    logger.error('Hook tracing bootstrap failed', { error: error.message });
+  }
+
+  return results;
+}
+
+export async function shutdownObservability() {
+  const { getExportSink } = await import('./export-sink.js');
+  const sink = getExportSink();
+  if (sink) {
+    await sink.close();
+  }
+
+  tracer.clear();
+  perfProfiler.clear();
+
+  logger.info('Observability shutdown complete');
+}
+
+export default bootstrapObservability;