thatcher 1.0.4

package/src/lib/query-engine.js

@@ -0,0 +1,399 @@
+/**
+ * Query Engine - Read operations
+ * Adapted from moonlanding/src/lib/query-engine.js
+ */
+
+import { getDatabase } from './database-core.js';
+import { getSpec } from '../config/spec-helpers.js';
+import { RECORD_STATUS } from '../config/constants.js';
+
+const db = getDatabase();
+const logger = createLogger('[QueryEngine]');
+
+/**
+ * Execute a query with error handling
+ * @param {string} sql
+ * @param {Array} params
+ * @param {object} context
+ * @returns {Array}
+ */
+function execQuery(sql, params = [], context = {}) {
+  try {
+    return db.prepare(sql).all(...params);
+  } catch (e) {
+    logger.error(`${context.operation || 'Query'} ${context.entity || ''}`, { sql, error: e.message });
+    throw new Error(`Database query failed: ${e.message}`);
+  }
+}
+
+/**
+ * Execute a single-row query
+ * @param {string} sql
+ * @param {Array} params
+ * @param {object} context
+ * @returns {object|null}
+ */
+function execGet(sql, params = [], context = {}) {
+  try {
+    return db.prepare(sql).get(...params);
+  } catch (e) {
+    logger.error(`${context.operation || 'Get'} ${context.entity || ''}`, { sql, error: e.message });
+    throw new Error(`Database get failed: ${e.message}`);
+  }
+}
+
+/**
+ * Get table name for entity (users vs user)
+ * @param {object} spec
+ * @returns {string}
+ */
+function tableName(spec) {
+  return spec.name === 'user' ? 'users' : spec.name;
+}
+
+/**
+ * Build SELECT query with joins, where, sorting, pagination
+ * @param {object} spec - Entity specification
+ * @param {object} where - Where conditions
+ * @param {object} options - Query options
+ * @returns {{sql: string, params: Array}}
+ */
+function buildSpecQuery(spec, where = {}, options = {}) {
+  const tbl = tableName(spec);
+  const table = `"${tbl}"`;
+  const selects = [`${table}.*`];
+  const joins = [];
+
+  // Add computed fields
+  if (spec.computed) {
+    Object.entries(spec.computed).forEach(([k, c]) => {
+      selects.push(`${c.sql} as "${k}"`);
+    });
+  }
+
+  // Add joins for ref fields with display
+  Object.entries(spec.fields || {}).forEach(([k, f]) => {
+    if (f.type === 'ref' && f.display) {
+      const refTbl = f.ref === 'user' ? 'users' : f.ref;
+      const alias = `"${refTbl}_${k}"`;
+      joins.push(`LEFT JOIN "${refTbl}" ${alias} ON ${table}."${k}" = ${alias}.id`);
+
+      const displayField = f.display.split('.')[1] || 'name';
+      selects.push(`${alias}."${displayField}" as "${k}_display"`);
+    }
+  });
+
+  const wc = [];
+  const p = [];
+
+  // Where conditions
+  Object.entries(where).forEach(([k, v]) => {
+    if (v !== undefined && v !== null) {
+      wc.push(`${table}."${k}" = ?`);
+      p.push(v);
+    }
+  });
+
+  // Exclude soft-deleted by default
+  if (spec.fields?.status && !where.status && !options.includeDeleted) {
+    wc.push(`${table}."status" != '${RECORD_STATUS.DELETED}'`);
+  }
+  if (spec.fields?.archived && !where.archived && !options.includeArchived) {
+    wc.push(`${table}."archived" = 0`);
+  }
+
+  let sql = `SELECT ${selects.join(', ')} FROM ${table}`;
+  if (joins.length) {
+    sql += ' ' + joins.join(' ');
+  }
+  if (wc.length) {
+    sql += ` WHERE ` + wc.join(` AND `);
+  }
+
+  // Sorting
+  const sort = options.sort || spec.list?.defaultSort;
+  if (sort && sort.field && spec.fields?.[sort.field]) {
+    const dir = (sort.dir || 'ASC').toUpperCase() === 'DESC' ? 'DESC' : 'ASC';
+    sql += ` ORDER BY ${table}."${sort.field}" ${dir}`;
+  }
+
+  // Limit/offset
+  if (options.limit) {
+    sql += ` LIMIT ${parseInt(options.limit, 10)}`;
+    if (options.offset) {
+      sql += ` OFFSET ${parseInt(options.offset, 10)}`;
+    }
+  }
+
+  return { sql, params: p };
+}
+
+/**
+ * Get total count for entity (optionally with where)
+ * @param {string} entity
+ * @param {object} where
+ * @param {object} options
+ * @returns {number}
+ */
+export function count(entity, where = {}, options = {}) {
+  const spec = getSpec(entity);
+  const tbl = tableName(spec);
+  const table = `"${tbl}"`;
+  const wc = [];
+  const p = [];
+
+  Object.entries(where).forEach(([k, v]) => {
+    if (v !== undefined && v !== null) {
+      wc.push(`${table}."${k}" = ?`);
+      p.push(v);
+    }
+  });
+
+  if (spec.fields?.status && !where.status && !options.includeDeleted) {
+    wc.push(`${table}."status" != '${RECORD_STATUS.DELETED}'`);
+  }
+
+  const whereClause = wc.length ? `WHERE ${wc.join(' AND ')}` : '';
+  const sql = `SELECT COUNT(*) as cnt FROM ${table} ${whereClause}`;
+
+  const result = execGet(sql, p, { entity, operation: 'Count' });
+  return result?.cnt || 0;
+}
+
+/**
+ * List all records (with optional where)
+ * @param {string} entity
+ * @param {object} where
+ * @param {object} options
+ * @returns {Array}
+ */
+export function list(entity, where = {}, options = {}) {
+  const spec = getSpec(entity);
+  const { sql, params } = buildSpecQuery(spec, where, options);
+  return execQuery(sql, params, { entity, operation: 'List' });
+}
+
+/**
+ * List with pagination
+ * @param {string} entity
+ * @param {object} where
+ * @param {number} page
+ * @param {number|null} pageSize
+ * @returns {{items: Array, pagination: object}}
+ */
+export async function listWithPagination(entity, where = {}, page = 1, pageSize = null) {
+  const spec = getSpec(entity);
+  const tbl = tableName(spec);
+
+  const paginationCfg = getPaginationConfig(spec);
+  const defaultPageSize = spec.list?.pageSize || paginationCfg.default_page_size;
+  const finalPageSize = pageSize || defaultPageSize;
+  const finalPage = Math.max(1, page);
+
+  const offset = (finalPage - 1) * finalPageSize;
+
+  const { sql, params } = buildSpecQuery(spec, where, {
+    ...options,
+    limit: finalPageSize,
+    offset,
+  });
+
+  const items = execQuery(sql, params, { entity, operation: 'ListWithPagination' });
+
+  const total = count(entity, where, options);
+
+  return {
+    items,
+    pagination: {
+      page: finalPage,
+      pageSize: finalPageSize,
+      total,
+      totalPages: Math.ceil(total / finalPageSize),
+    },
+  };
+}
+
+/**
+ * Get single record by ID
+ * @param {string} entity
+ * @param {string|number} id
+ * @returns {object|null}
+ */
+export function get(entity, id) {
+  const spec = getSpec(entity);
+  const tbl = tableName(spec);
+  const sql = `SELECT * FROM "${tbl}" WHERE id = ?`;
+  return execGet(sql, [id], { entity, operation: 'Get' });
+}
+
+/**
+ * Get single record by field value
+ * @param {string} entity
+ * @param {string} field
+ * @param {any} value
+ * @returns {object|null}
+ */
+export function getBy(entity, field, value) {
+  const spec = getSpec(entity);
+  const tbl = tableName(spec);
+  const sql = `SELECT * FROM "${tbl}" WHERE "${field}" = ? LIMIT 1`;
+  return execGet(sql, [value], { entity, operation: 'GetBy' });
+}
+
+/**
+ * Search using FTS
+ * @param {string} entity
+ * @param {string} query
+ * @param {object} where
+ * @param {object} options
+ * @returns {Array}
+ */
+export function search(entity, query, where = {}, options = {}) {
+  const spec = getSpec(entity);
+  const tbl = tableName(spec);
+  const ftsTable = `${tbl}_fts`;
+
+  // Check if FTS table exists
+  const ftsExists = db.prepare(`
+    SELECT name FROM sqlite_master WHERE type='table' AND name=?
+  `).get(ftsTable);
+
+  if (!ftsExists) {
+    // Fallback to simple LIKE search
+    const { sql, params } = buildSpecQuery(spec, where, options);
+    const baseSql = sql.replace('SELECT *', `SELECT *`);
+    const searchTerm = `%${query}%`;
+    const searchFields = Object.keys(spec.fields || {}).filter(
+      f => ['text', 'textarea', 'email'].includes(spec.fields[f].type)
+    );
+
+    if (searchFields.length > 0) {
+      const conditions = searchFields.map(f => `"${f}" LIKE ?`).join(' OR ');
+      const whereAnd = sql.includes('WHERE') ? ' AND ' : ' WHERE ';
+      const fullSql = baseSql + whereAnd + `(${conditions})`;
+      const searchParams = [...params, ...searchFields.map(() => searchTerm)];
+      return execQuery(fullSql, searchParams, { entity, operation: 'SearchFallback' });
+    }
+
+    return [];
+  }
+
+  // Use FTS
+  const ftsResults = db.prepare(`
+    SELECT rowid as id FROM ${ftsTable}
+    WHERE ${ftsTable} MATCH ?
+  `).all(query);
+
+  if (!ftsResults.length) return [];
+
+  const ids = ftsResults.map(r => r.id);
+  const idPlaceholders = ids.map(() => '?').join(',');
+  const { sql, params } = buildSpecQuery(spec, { ...where, id: { $in: ids } }, options);
+  const finalSql = sql.replace('WHERE', `WHERE "${tbl}"."id" IN (${idPlaceholders}) AND `);
+
+  return execQuery(finalSql, [...ids, ...params], { entity, operation: 'Search' });
+}
+
+/**
+ * Search with pagination
+ * @param {string} entity
+ * @param {string} query
+ * @param {object} where
+ * @param {number} page
+ * @param {number} pageSize
+ * @returns {{items: Array, pagination: object}}
+ */
+export async function searchWithPagination(entity, query, where = {}, page = 1, pageSize = null) {
+  const spec = getSpec(entity);
+  const paginationCfg = getPaginationConfig(spec);
+  const defaultPageSize = spec.list?.pageSize || paginationCfg.default_page_size;
+  const finalPageSize = pageSize || defaultPageSize;
+  const finalPage = Math.max(1, page);
+
+  const items = search(entity, query, where, {
+    limit: finalPageSize,
+    offset: (finalPage - 1) * finalPageSize,
+  });
+
+  // For total count, we'd need a separate FTS count query - simplified here
+  const total = items.length; // Approximate
+
+  return {
+    items,
+    pagination: {
+      page: finalPage,
+      pageSize: finalPageSize,
+      total,
+      totalPages: Math.ceil(total / finalPageSize),
+    },
+  };
+}
+
+/**
+ * Get children of a parent entity
+ * @param {string} parentEntity
+ * @param {string|number} parentId
+ * @param {object} childDef
+ * @returns {Array}
+ */
+export function getChildren(parentEntity, parentId, childDef) {
+  const childSpec = getSpec(childDef.entity);
+  const childTbl = tableName(childSpec);
+  const fk = childDef.fk || `${parentEntity}_id`;
+
+  const sql = `SELECT * FROM "${childTbl}" WHERE "${fk}" = ? AND status != 'deleted'`;
+  return execQuery(sql, [parentId], { entity: childDef.entity, operation: 'GetChildren' });
+}
+
+/**
+ * Get pagination config from system settings
+ * @param {object} spec - Entity spec
+ * @returns {{default_page_size: number, max_page_size: number}}
+ */
+function getPaginationConfig(spec) {
+  // Try to get from config engine, fallback to defaults
+  try {
+    // Will be provided by caller or fetched from global config
+    return {
+      default_page_size: spec.list?.pageSize || 50,
+      max_page_size: 500,
+    };
+  } catch {
+    return { default_page_size: 50, max_page_size: 500 };
+  }
+}
+
+/**
+ * Simple logger
+ */
+function createLogger(prefix) {
+  return {
+    error: (msg, meta = {}) => {
+      console.error(`${prefix} ${msg}`, meta);
+    },
+    warn: (msg, meta = {}) => {
+      console.warn(`${prefix} ${msg}`, meta);
+    },
+    info: (msg, meta = {}) => {
+      console.info(`${prefix} ${msg}`, meta);
+    },
+  };
+}
+
+/**
+ * Run in transaction
+ * @param {Function} callback
+ * @returns {Promise<any>}
+ */
+export async function withTransaction(callback) {
+  const dbInstance = getDatabase();
+  try {
+    dbInstance.prepare('BEGIN').run();
+    const result = await callback();
+    dbInstance.prepare('COMMIT').run();
+    return result;
+  } catch (e) {
+    dbInstance.prepare('ROLLBACK').run();
+    throw e;
+  }
+}

package/src/lib/query-perf.js

@@ -0,0 +1,117 @@
+const queryMetrics = new Map();
+const slowQueries = [];
+const MAX_SLOW_QUERIES = 100;
+const SLOW_QUERY_THRESHOLD = 100;
+
+export const trackQuery = (sql, duration, params = []) => {
+  const key = sql.substring(0, 200);
+
+  if (!queryMetrics.has(key)) {
+    queryMetrics.set(key, {
+      sql: key,
+      count: 0,
+      totalTime: 0,
+      minTime: Infinity,
+      maxTime: 0,
+      times: []
+    });
+  }
+
+  const metric = queryMetrics.get(key);
+  metric.count++;
+  metric.totalTime += duration;
+  metric.minTime = Math.min(metric.minTime, duration);
+  metric.maxTime = Math.max(metric.maxTime, duration);
+  metric.times.push(duration);
+
+  if (metric.times.length > 1000) metric.times.shift();
+
+  if (duration >= SLOW_QUERY_THRESHOLD) {
+    slowQueries.push({
+      sql: key,
+      duration,
+      params: params.slice(0, 10),
+      timestamp: Date.now()
+    });
+
+    if (slowQueries.length > MAX_SLOW_QUERIES) slowQueries.shift();
+  }
+};
+
+const percentile = (arr, p) => {
+  if (arr.length === 0) return 0;
+  const sorted = [...arr].sort((a, b) => a - b);
+  const index = Math.ceil(sorted.length * p / 100) - 1;
+  return sorted[Math.max(0, index)];
+};
+
+export const getMetrics = () => {
+  const metrics = [];
+
+  for (const [, metric] of queryMetrics) {
+    const avgTime = metric.count > 0 ? metric.totalTime / metric.count : 0;
+    const p50 = percentile(metric.times, 50);
+    const p95 = percentile(metric.times, 95);
+    const p99 = percentile(metric.times, 99);
+
+    metrics.push({
+      sql: metric.sql,
+      count: metric.count,
+      avgTime: avgTime.toFixed(2),
+      minTime: metric.minTime === Infinity ? 0 : metric.minTime.toFixed(2),
+      maxTime: metric.maxTime.toFixed(2),
+      p50: p50.toFixed(2),
+      p95: p95.toFixed(2),
+      p99: p99.toFixed(2),
+      totalTime: metric.totalTime.toFixed(2)
+    });
+  }
+
+  return metrics.sort((a, b) => parseFloat(b.totalTime) - parseFloat(a.totalTime));
+};
+
+export const getSlowQueries = () => {
+  return slowQueries.slice().reverse();
+};
+
+export const getSummary = () => {
+  const metrics = getMetrics();
+  const totalQueries = metrics.reduce((sum, m) => sum + m.count, 0);
+  const totalTime = metrics.reduce((sum, m) => sum + parseFloat(m.totalTime), 0);
+  const allTimes = [];
+
+  for (const [, metric] of queryMetrics) {
+    allTimes.push(...metric.times);
+  }
+
+  return {
+    totalQueries,
+    uniqueQueries: metrics.length,
+    totalTime: totalTime.toFixed(2),
+    avgTime: totalQueries > 0 ? (totalTime / totalQueries).toFixed(2) : '0',
+    p50: percentile(allTimes, 50).toFixed(2),
+    p95: percentile(allTimes, 95).toFixed(2),
+    p99: percentile(allTimes, 99).toFixed(2),
+    slowQueries: slowQueries.length,
+    slowQueryThreshold: SLOW_QUERY_THRESHOLD
+  };
+};
+
+export const clearMetrics = () => {
+  queryMetrics.clear();
+  slowQueries.length = 0;
+};
+
+export const withPerfTracking = (db, sql, params, executor) => {
+  const start = performance.now();
+  try {
+    const result = executor();
+    const duration = performance.now() - start;
+    trackQuery(sql, duration, params);
+    return result;
+  } catch (e) {
+    const duration = performance.now() - start;
+    trackQuery(sql, duration, params);
+    throw e;
+  }
+};

package/src/lib/query-string-adapter.js

@@ -0,0 +1,75 @@
+/**
+ * Query String Adapter - Parse URL query parameters into typed objects
+ * Adapted from moonlanding/src/lib/query-string-adapter.js
+ */
+
+/**
+ * Parse query string from URL
+ * @param {object} request - Fetch Request or NextRequest-like
+ * @returns {Promise<{q?: string, page?: number, pageSize?: number, filters?: object, sort?: object}>}
+ */
+export async function parseQuery(request) {
+  const url = request.url || (request._url ? `http://localhost${request._url}` : 'http://localhost/');
+  const searchParams = new URL(url).searchParams;
+
+  const result = {
+    q: searchParams.get('q') || null,
+    page: parseInt(searchParams.get('page') || '1', 10),
+    pageSize: parseInt(searchParams.get('pageSize') || searchParams.get('page_size') || '50', 10),
+    filters: {},
+    sort: null,
+  };
+
+  // Parse filters (filter[key]=value)
+  for (const [key, value] of searchParams.entries()) {
+    if (key.startsWith('filter_') || key.startsWith('filters[')) {
+      const filterKey = key.replace('filter_', '').replace(/filters\[(\w+)\]/, '$1');
+      result.filters[filterKey] = coerceValue(value);
+    }
+  }
+
+  // Parse sort
+  const sortBy = searchParams.get('sort') || searchParams.get('sortBy');
+  const sortDir = searchParams.get('dir') || searchParams.get('direction') || 'asc';
+  if (sortBy) {
+    result.sort = {
+      field: sortBy,
+      dir: sortDir.toLowerCase() === 'desc' ? 'desc' : 'asc',
+    };
+  }
+
+  return result;
+}
+
+/**
+ * Coerce string to appropriate type
+ * @param {string} value
+ * @returns {any}
+ */
+function coerceValue(value) {
+  // Lowercase boolean strings
+  if (value === 'true') return true;
+  if (value === 'false') return false;
+  if (value === 'null' || value === '') return null;
+
+  // Number?
+  const num = Number(value);
+  if (!isNaN(num) && value.trim() !== '') return num;
+
+  return value;
+}
+
+/**
+ * Get default value for config key
+ * @param {string} key
+ * @returns {any}
+ */
+export function getDefault(key) {
+  const defaults = {
+    page: 1,
+    pageSize: 50,
+    q: null,
+    filters: {},
+  };
+  return defaults[key];
+}

package/src/lib/realtime-server.js

@@ -0,0 +1,67 @@
+/**
+ * Realtime Server - WebSocket/Server-Sent Events for live updates
+ * Optional component - can be disabled if not needed
+ */
+
+let subscribers = new Map(); // channel -> Set of callbacks
+let enabled = false;
+
+/**
+ * Initialize realtime (optional)
+ * @param {object} server - HTTP server to attach WS endpoint
+ */
+export function initRealtime(server) {
+  if (!server) return;
+
+  // Can be extended with ws library for full WebSocket support
+  // For now, provides a polling-based notification system
+  enabled = true;
+  console.log('[Realtime] Initialized (polling mode)');
+}
+
+/**
+ * Broadcast update to all subscribers of a channel
+ * @param {string} channel
+ * @param {string} event
+ * @param {any} data
+ */
+export function broadcastUpdate(channel, event, data) {
+  if (!enabled) return;
+
+  const channelSubscribers = subscribers.get(channel);
+  if (channelSubscribers) {
+    const payload = { event, data, timestamp: Date.now() };
+    channelSubscribers.forEach(cb => {
+      try { cb(payload); } catch {}
+    });
+  }
+}
+
+/**
+ * Subscribe to updates for a channel
+ * @param {string} channel
+ * @param {Function} callback
+ * @returns {Function} Unsubscribe function
+ */
+export function subscribe(channel, callback) {
+  if (!subscribers.has(channel)) {
+    subscribers.set(channel, new Set());
+  }
+  subscribers.get(channel).add(callback);
+
+  // Return unsubscribe function
+  return () => {
+    subscribers.get(channel)?.delete(callback);
+  };
+}
+
+/**
+ * Get update stats
+ * @returns {object}
+ */
+export function getStats() {
+  return {
+    channels: subscribers.size,
+    totalSubscribers: Array.from(subscribers.values()).reduce((sum, set) => sum + set.size, 0),
+  };
+}

package/src/lib/render-cache.js

@@ -0,0 +1,61 @@
+const cache = new Map()
+const maxSize = 500
+const ttl = 60000
+let hits = 0
+let misses = 0
+
+function cacheKey(fn, args) {
+  return `${fn.name}:${JSON.stringify(args)}`
+}
+
+export function memoize(fn) {
+  return function(...args) {
+    const key = cacheKey(fn, args)
+    const cached = cache.get(key)
+    if (cached && Date.now() - cached.ts < ttl) {
+      hits++
+      return cached.value
+    }
+    misses++
+    const value = fn(...args)
+    cache.set(key, { value, ts: Date.now() })
+    if (cache.size > maxSize) {
+      const first = cache.keys().next().value
+      cache.delete(first)
+    }
+    return value
+  }
+}
+
+export function invalidate(pattern) {
+  if (!pattern) {
+    cache.clear()
+    return cache.size
+  }
+  let count = 0
+  for (const key of cache.keys()) {
+    if (key.includes(pattern)) {
+      cache.delete(key)
+      count++
+    }
+  }
+  return count
+}
+
+export function getCacheStats() {
+  const total = hits + misses
+  return {
+    size: cache.size,
+    hits,
+    misses,
+    hitRate: total > 0 ? (hits / total * 100).toFixed(2) + '%' : '0%',
+    maxSize,
+    ttl
+  }
+}
+
+export function clearCache() {
+  cache.clear()
+  hits = 0
+  misses = 0
+}