webspresso 0.0.101 → 0.0.103

package/plugins/site-analytics/tracking.js

@@ -11,6 +11,66 @@ const { detectCountry } = require('./geo');
 const STATIC_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map|webp|avif|mp4|webm)$/i;
 const DEFAULT_EXCLUDE = ['/_admin', '/api/', '/health', '/_next', '/__'];
 
+/**
+ * Permanent / Fatal DB error codes that cannot be recovered by immediate retries.
+ */
+const PERMANENT_ERROR_CODES = new Set([
+  'SQLITE_FULL',
+  'SQLITE_IOERR',
+  'SQLITE_CORRUPT',
+  'SQLITE_CANTOPEN',
+  'SQLITE_NOTADB',
+  'ER_DISK_FULL',
+  'ENOSPC',
+]);
+
+/**
+ * Patterns matching fatal disk full or corrupted database errors.
+ */
+const PERMANENT_ERROR_PATTERNS = [
+  /database or disk is full/i,
+  /disk full/i,
+  /no space left on device/i,
+  /disk i\/o error/i,
+  /database disk image is malformed/i,
+  /file is not a database/i,
+];
+
+/**
+ * Sanitize and format database error messages to prevent raw SQL query leakage in logs.
+ * Knex appends executed queries ("insert into ... - <actual error>") which can be 50-100 KB.
+ * @param {Error|Object|string} e
+ * @returns {string}
+ */
+function formatDbError(e) {
+  if (!e) return 'Unknown error';
+
+  let msg;
+  if (typeof e.message === 'string' && e.message.includes(' - ')) {
+    msg = e.message.split(' - ').pop().trim();
+  } else if (typeof e.message === 'string') {
+    msg = e.message.trim();
+  } else if (typeof e === 'string') {
+    msg = e.trim();
+  } else {
+    msg = String(e.code || 'Database error');
+  }
+
+  return msg.replace(/[\r\n]+/g, ' ').trim().slice(0, 150);
+}
+
+/**
+ * Detect whether a database error is permanent/fatal (e.g. disk full, corruption).
+ * @param {Error|Object|string} e
+ * @returns {boolean}
+ */
+function isPermanentDbError(e) {
+  if (!e) return false;
+  if (e.code && PERMANENT_ERROR_CODES.has(e.code)) return true;
+  const msg = typeof e.message === 'string' ? e.message : (typeof e === 'string' ? e : '');
+  return PERMANENT_ERROR_PATTERNS.some((pattern) => pattern.test(msg));
+}
+
 /**
  * Hash a string with SHA-256 (truncated to 16 hex chars for compactness)
  */
@@ -38,6 +98,10 @@ function getClientIp(req) {
  * @param {string} [options.tableName='analytics_page_views'] - DB table name
  * @param {number} [options.batchSize=20] - Flush when queue reaches this size
  * @param {number} [options.flushIntervalMs=3000] - Flush interval for low traffic
+ * @param {number} [options.maxRetries=3] - Maximum retry attempts per batch item before dropping
+ * @param {number} [options.maxBackoffMs=60000] - Max exponential backoff cooldown
+ * @param {number} [options.maxQueueSize=5000] - Queue memory cap to prevent OOM
+ * @param {number} [options.errorLogIntervalMs=60000] - Rate limit window for batch failure logs
  */
 function createTrackingMiddleware(options) {
   const {
@@ -47,12 +111,26 @@ function createTrackingMiddleware(options) {
     tableName = 'analytics_page_views',
     batchSize = 20,
     flushIntervalMs = 3000,
+    maxRetries = 3,
+    maxBackoffMs = 60000,
+    maxQueueSize = 5000,
+    errorLogIntervalMs = 60000,
   } = options;
 
   const allExcludes = [...DEFAULT_EXCLUDE, ...excludePaths];
   let tableReady = false;
   const queue = [];
-  let flushScheduled = false;
+
+  // Circuit breaker & backoff state
+  let failureCount = 0;
+  let nextAllowedFlushTime = 0;
+  let isFlushing = false;
+  let flushPromise = null;
+  let flushTimeout = null;
+
+  // Log throttling state
+  let lastErrorLogTime = 0;
+  let suppressedErrorCount = 0;
 
   async function ensureTable() {
     if (tableReady) return;
@@ -75,7 +153,26 @@ function createTrackingMiddleware(options) {
       }
       tableReady = true;
     } catch (e) {
-      console.error('[site-analytics] Failed to ensure table:', e.message);
+      logError(e, 'Failed to ensure table');
+      throw e;
+    }
+  }
+
+  function logError(e, prefix = 'Batch insert failed') {
+    const now = Date.now();
+    const formatted = formatDbError(e);
+
+    if (lastErrorLogTime === 0 || now - lastErrorLogTime >= errorLogIntervalMs) {
+      const elapsedSec = lastErrorLogTime > 0 ? Math.round((now - lastErrorLogTime) / 1000) : 0;
+      if (suppressedErrorCount > 0) {
+        console.error(`[site-analytics] ${prefix} (suppressed ${suppressedErrorCount} occurrences in last ${elapsedSec}s): ${formatted}`);
+      } else {
+        console.error(`[site-analytics] ${prefix}: ${formatted}`);
+      }
+      lastErrorLogTime = now;
+      suppressedErrorCount = 0;
+    } else {
+      suppressedErrorCount++;
     }
   }
 
@@ -83,17 +180,17 @@ function createTrackingMiddleware(options) {
   const sessionMap = new Map();
   const SESSION_TIMEOUT = 30 * 60 * 1000; // 30 minutes
   const MAX_SESSIONS = 10000;
-  const MAX_QUEUE_SIZE = 5000;
 
   // Periodic session cleanup
-  setInterval(() => {
+  const sessionCleanupInterval = setInterval(() => {
     const now = Date.now();
     for (const [vid, data] of sessionMap) {
       if (now - data.lastSeen > SESSION_TIMEOUT * 2) {
         sessionMap.delete(vid);
       }
     }
-  }, 5 * 60 * 1000).unref();
+  }, 5 * 60 * 1000);
+  if (sessionCleanupInterval.unref) sessionCleanupInterval.unref();
 
   function getSessionId(visitorId) {
     const now = Date.now();
@@ -115,44 +212,93 @@ function createTrackingMiddleware(options) {
   }
 
   async function flushQueue() {
+    if (flushPromise) return flushPromise;
     if (queue.length === 0) return;
-    const batch = queue.splice(0, queue.length);
 
-    try {
-      await ensureTable();
-      for (const row of batch) {
-        row.created_at = knex.fn.now();
-      }
-      await knex(tableName).insert(batch);
-    } catch (e) {
-      console.error('[site-analytics] Batch insert failed:', e.message);
-      // Re-queue failed items with max size limit to prevent memory leak
-      if (queue.length + batch.length <= MAX_QUEUE_SIZE) {
-        queue.unshift(...batch);
-      } else {
-        const available = Math.max(0, MAX_QUEUE_SIZE - queue.length);
-        if (available > 0) {
-          queue.unshift(...batch.slice(batch.length - available));
+    const now = Date.now();
+    if (now < nextAllowedFlushTime) {
+      scheduleFlush(nextAllowedFlushTime - now);
+      return;
+    }
+
+    isFlushing = true;
+    flushPromise = (async () => {
+      const batch = queue.splice(0, queue.length);
+
+      try {
+        await ensureTable();
+        const dbBatch = batch.map(({ _retries, ...row }) => ({
+          ...row,
+          created_at: knex.fn.now(),
+        }));
+        await knex(tableName).insert(dbBatch);
+
+        // Successfully inserted: reset circuit breaker & failure counters
+        failureCount = 0;
+        nextAllowedFlushTime = 0;
+        lastErrorLogTime = 0;
+        suppressedErrorCount = 0;
+      } catch (e) {
+        failureCount++;
+        const backoffDelay = Math.min(flushIntervalMs * Math.pow(2, failureCount), maxBackoffMs);
+        nextAllowedFlushTime = Date.now() + backoffDelay;
+
+        // Rate-limited sanitized logging (no raw SQL query leakage)
+        logError(e, 'Batch insert failed');
+
+        // Schedule next attempt respecting backoff
+        scheduleFlush(backoffDelay);
+
+        // Drop policy: If fatal/permanent error (disk full, corrupted DB, etc.), drop entire batch
+        if (!isPermanentDbError(e)) {
+          const retryable = [];
+          for (const item of batch) {
+            item._retries = (item._retries || 0) + 1;
+            if (item._retries < maxRetries) {
+              retryable.push(item);
+            }
+          }
+
+          // Re-queue items that haven't exceeded retry limits, respecting maxQueueSize
+          if (retryable.length > 0) {
+            const available = Math.max(0, maxQueueSize - queue.length);
+            if (available > 0) {
+              queue.unshift(...retryable.slice(0, available));
+            }
+          }
         }
+      } finally {
+        isFlushing = false;
+        flushPromise = null;
       }
-    }
+    })();
+
+    return flushPromise;
   }
 
-  function scheduleFlush() {
-    if (flushScheduled || queue.length === 0) return;
-    flushScheduled = true;
-    setTimeout(() => {
-      flushScheduled = false;
+  function scheduleFlush(customDelay) {
+    if (flushTimeout || queue.length === 0) return;
+    const now = Date.now();
+    const delay = customDelay !== undefined
+      ? customDelay
+      : Math.max(flushIntervalMs, nextAllowedFlushTime - now, 0);
+
+    flushTimeout = setTimeout(() => {
+      flushTimeout = null;
       flushQueue().catch(() => {});
-    }, flushIntervalMs);
+    }, delay);
+    if (flushTimeout.unref) flushTimeout.unref();
   }
 
   // Periodic flush (catches low-traffic case)
-  setInterval(() => {
-    if (queue.length > 0) flushQueue().catch(() => {});
-  }, flushIntervalMs).unref();
+  const periodicFlushInterval = setInterval(() => {
+    if (queue.length > 0 && Date.now() >= nextAllowedFlushTime && !isFlushing) {
+      flushQueue().catch(() => {});
+    }
+  }, flushIntervalMs);
+  if (periodicFlushInterval.unref) periodicFlushInterval.unref();
 
-  return async function trackingMiddleware(req, res, next) {
+  async function trackingMiddleware(req, res, next) {
     next();
 
     if (req.method !== 'GET') return;
@@ -168,33 +314,86 @@ function createTrackingMiddleware(options) {
 
       if (isBot && !trackBots) return;
 
+      // Protect queue under pressure or during active database failures:
+      // Drop bots immediately to preserve queue capacity and avoid wasting DB I/O
+      if (isBot && (failureCount > 0 || queue.length >= maxQueueSize * 0.7)) {
+        return;
+      }
+
       const ipHash = quickHash(ip);
       const visitorId = quickHash(ip + '|' + userAgent);
-      const sessionId = getSessionId(visitorId);
+
+      // Do not thrash human sessionMap with bot/crawler visits
+      const sessionId = isBot
+        ? quickHash('bot-' + (botName || 'crawler') + '-' + visitorId)
+        : getSessionId(visitorId);
+
       const country = detectCountry(req);
       const referrer = req.headers['referer'] || req.headers['referrer'] || null;
 
-      queue.push({
-        session_id: sessionId,
-        visitor_id: visitorId,
-        path,
-        referrer: referrer ? referrer.slice(0, 500) : null,
-        user_agent: userAgent.slice(0, 1000),
-        ip_hash: ipHash,
-        country,
-        is_bot: isBot,
-        bot_name: botName,
-      });
+      if (queue.length < maxQueueSize) {
+        queue.push({
+          session_id: sessionId,
+          visitor_id: visitorId,
+          path,
+          referrer: referrer ? referrer.slice(0, 500) : null,
+          user_agent: userAgent.slice(0, 1000),
+          ip_hash: ipHash,
+          country,
+          is_bot: isBot,
+          bot_name: botName,
+        });
+      }
 
       if (queue.length >= batchSize) {
-        flushQueue().catch(() => {});
+        if (Date.now() >= nextAllowedFlushTime) {
+          flushQueue().catch(() => {});
+        } else {
+          scheduleFlush();
+        }
       } else {
         scheduleFlush();
       }
     } catch (e) {
       // Non-blocking: don't let tracking errors affect the app
     }
+  }
+
+  // Diagnostic / testing accessors
+  trackingMiddleware.flush = () => flushQueue();
+  trackingMiddleware.getQueue = () => queue;
+  trackingMiddleware.getFailureCount = () => failureCount;
+  trackingMiddleware.getNextAllowedFlushTime = () => nextAllowedFlushTime;
+  trackingMiddleware.resetState = (opts = {}) => {
+    failureCount = 0;
+    nextAllowedFlushTime = 0;
+    const resetLogs = typeof opts === 'boolean' ? opts : !!opts.resetLogs;
+    if (resetLogs) {
+      lastErrorLogTime = 0;
+      suppressedErrorCount = 0;
+    }
+    queue.length = 0;
+    if (flushTimeout) {
+      clearTimeout(flushTimeout);
+      flushTimeout = null;
+    }
   };
+  trackingMiddleware.stop = () => {
+    if (flushTimeout) {
+      clearTimeout(flushTimeout);
+      flushTimeout = null;
+    }
+    clearInterval(sessionCleanupInterval);
+    clearInterval(periodicFlushInterval);
+  };
+
+  return trackingMiddleware;
 }
 
-module.exports = { createTrackingMiddleware };
+module.exports = {
+  createTrackingMiddleware,
+  formatDbError,
+  isPermanentDbError,
+  PERMANENT_ERROR_CODES,
+  PERMANENT_ERROR_PATTERNS,
+};

package/src/api/api-loader.js

@@ -6,9 +6,11 @@
  * @module src/api/api-loader
  */
 
+const path = require('path');
 const { ZodError } = require('zod');
 const { z: zForApi } = require('../../core/validation');
 const { resolveMiddlewares } = require('../file-router');
+const { translateValidationIssues, getTranslatorForRequest } = require('./validation-i18n');
 
 /**
  * Compiles a schema definition from an API module
@@ -93,7 +95,7 @@ function createApiHandler(descriptor, context) {
       // 1. Reload module in development
       if (isDev) {
         try {
-          const resolvedPath = require.resolve(descriptor.file);
+          const resolvedPath = descriptor._resolvedPath || (descriptor._resolvedPath = require.resolve(descriptor.file));
           if (require.cache[resolvedPath]) {
             delete require.cache[resolvedPath];
             cachedMiddlewares = null;
@@ -147,9 +149,17 @@ function createApiHandler(descriptor, context) {
           validateRequestInput(req, compiledSchema);
         } catch (err) {
           if (err instanceof ZodError || err.name === 'ZodError' || Array.isArray(err?.issues)) {
+            const pagesDir = context.pagesDir || appOptions.pagesDir || 'pages';
+            const routeDir = descriptor.file ? path.dirname(descriptor.file) : '';
+            const translatedIssues = translateValidationIssues(err.issues, {
+              req,
+              pagesDir,
+              routeDir,
+              isDev,
+            });
             return res.status(400).json({
               error: 'Validation Error',
-              issues: err.issues,
+              issues: translatedIssues,
             });
           }
           throw err;
@@ -185,6 +195,10 @@ function createApiHandler(descriptor, context) {
       }
 
       // 5. Execute handler
+      const pagesDir = context.pagesDir || appOptions.pagesDir || 'pages';
+      const routeDir = descriptor.file ? path.dirname(descriptor.file) : '';
+      const tr = getTranslatorForRequest(req, { pagesDir, routeDir, isDev });
+
       const apiCtx = {
         req,
         res,
@@ -192,6 +206,8 @@ function createApiHandler(descriptor, context) {
         query: req.query || {},
         body: req.body || {},
         input: req.input,
+        locale: tr.locale,
+        t: tr.t,
         service: (name, input, opts) =>
           serviceRegistry ? serviceRegistry.call(name, input, { req, res, db }, opts) : null,
         module: descriptor.module || null,

package/src/api/validation-i18n.js

@@ -0,0 +1,231 @@
+'use strict';
+
+/**
+ * Webspresso API Validation i18n & Error Translator
+ * Automatically localizes Zod validation error messages and keys based on request locale
+ * @module src/api/validation-i18n
+ */
+
+const path = require('path');
+
+const EMPTY_OBJECT = Object.freeze({});
+
+let _fileRouter = null;
+function getFileRouter() {
+  if (!_fileRouter) {
+    _fileRouter = require('../file-router');
+  }
+  return _fileRouter;
+}
+
+/**
+ * Resolves or creates a cached translator function for the given request and directory context
+ * @param {import('express').Request} req - Express request
+ * @param {Object} [options]
+ * @param {string} [options.pagesDir] - Pages directory path
+ * @param {string} [options.routeDir] - Route-specific directory
+ * @param {boolean} [options.isDev] - Is development environment
+ * @returns {{ t: Function, locale: string }}
+ */
+function getTranslatorForRequest(req, options = {}) {
+  if (req && req.t && req.locale) {
+    return { t: req.t, locale: req.locale };
+  }
+
+  const {
+    loadI18n,
+    createTranslator,
+    detectLocale,
+    translatorCache,
+  } = getFileRouter();
+
+  const defaultLocale = process.env.DEFAULT_LOCALE || 'en';
+  const locale = (req && req.locale) ? req.locale : (req ? detectLocale(req) : defaultLocale);
+
+  const rawPagesDir = options.pagesDir || (req?.app?.get?.('pagesDir')) || 'pages';
+  const pagesDir = path.resolve(rawPagesDir);
+  const routeDir = options.routeDir ? path.resolve(options.routeDir) : '';
+  const isDev = options.isDev !== undefined ? options.isDev : process.env.NODE_ENV !== 'production';
+
+  let t;
+  if (!isDev && translatorCache) {
+    const transKey = `${pagesDir}::${routeDir}::${locale}::${defaultLocale}`;
+    t = translatorCache.get(transKey);
+    if (!t) {
+      const translations = loadI18n(pagesDir, routeDir, locale, false);
+      const fallbackTranslations = (locale !== defaultLocale)
+        ? loadI18n(pagesDir, routeDir, defaultLocale, false)
+        : EMPTY_OBJECT;
+      t = createTranslator(translations, {
+        locale,
+        fallbackTranslations,
+        fallbackLocale: defaultLocale,
+      });
+      translatorCache.set(transKey, t);
+    }
+  } else {
+    const translations = loadI18n(pagesDir, routeDir, locale, true);
+    const fallbackTranslations = (locale !== defaultLocale)
+      ? loadI18n(pagesDir, routeDir, defaultLocale, true)
+      : EMPTY_OBJECT;
+    t = createTranslator(translations, {
+      locale,
+      fallbackTranslations,
+      fallbackLocale: defaultLocale,
+    });
+  }
+
+  if (req) {
+    req.locale = locale;
+    req.t = t;
+  }
+
+  return { t, locale };
+}
+
+/**
+ * Translates a single Zod issue using candidate keys and parameter interpolation
+ * @param {Object} issue - Zod issue
+ * @param {Function} t - Translator function
+ * @returns {Object} Translated issue clone
+ */
+function translateSingleIssue(issue, t) {
+  if (!issue || typeof issue !== 'object') return issue;
+
+  const pathKey = Array.isArray(issue.path) && issue.path.length > 0 ? issue.path.join('.') : '';
+  const field = Array.isArray(issue.path) && issue.path.length > 0 ? issue.path[issue.path.length - 1] : '';
+  const isRequired = (issue.code === 'invalid_type' && (issue.received === 'undefined' || issue.message === 'Required'))
+    || issue.message === 'Required';
+
+  const params = Object.assign(
+    {
+      path: pathKey,
+      field,
+      min: issue.minimum !== undefined ? issue.minimum : undefined,
+      max: issue.maximum !== undefined ? issue.maximum : undefined,
+      minimum: issue.minimum !== undefined ? issue.minimum : undefined,
+      maximum: issue.maximum !== undefined ? issue.maximum : undefined,
+      limit: issue.minimum !== undefined ? issue.minimum : (issue.maximum !== undefined ? issue.maximum : undefined),
+      expected: issue.expected,
+      received: issue.received,
+      type: issue.type,
+      validation: issue.validation,
+    },
+    issue
+  );
+
+  const candidates = [];
+
+  // 1. Explicit key in custom message (e.g. z.string({ message: 'auth.email_required' }))
+  if (typeof issue.message === 'string' && typeof t.has === 'function' && t.has(issue.message)) {
+    candidates.push(issue.message);
+  } else if (typeof issue.message === 'string' && issue.message.includes('{{')) {
+    const interpolated = issue.message.replace(/{{\s*(\w+)\s*}}/g, (_, k) =>
+      params[k] !== undefined ? String(params[k]) : ''
+    );
+    return Object.assign({}, issue, { message: interpolated });
+  }
+
+  // 2. Field-specific overrides (e.g. validation.fields.email.required or validation.email.required)
+  if (pathKey) {
+    if (isRequired) {
+      candidates.push(`validation.fields.${pathKey}.required`);
+      candidates.push(`validation.${pathKey}.required`);
+      candidates.push(`validation.fields.${pathKey}`);
+      candidates.push(`validation.${pathKey}`);
+    } else {
+      if (issue.validation) {
+        candidates.push(`validation.fields.${pathKey}.${issue.validation}`);
+        candidates.push(`validation.${pathKey}.${issue.validation}`);
+      }
+      candidates.push(`validation.fields.${pathKey}.${issue.code}`);
+      candidates.push(`validation.${pathKey}.${issue.code}`);
+      candidates.push(`validation.fields.${pathKey}`);
+      candidates.push(`validation.${pathKey}`);
+    }
+  }
+
+  // 3. General validation code translations
+  if (isRequired) {
+    candidates.push('validation.required');
+    candidates.push('zod.required');
+    candidates.push('validation.invalid_type');
+    candidates.push('zod.invalid_type');
+  } else if (issue.code === 'invalid_string' && issue.validation) {
+    candidates.push(`validation.invalid_string.${issue.validation}`);
+    candidates.push(`validation.${issue.validation}`);
+    candidates.push(`zod.invalid_string.${issue.validation}`);
+    candidates.push(`zod.${issue.validation}`);
+    candidates.push('validation.invalid_string');
+    candidates.push('zod.invalid_string');
+  } else {
+    candidates.push(`validation.${issue.code}`);
+    candidates.push(`zod.${issue.code}`);
+  }
+
+  let matchedKey = null;
+  if (typeof t.has === 'function') {
+    for (const key of candidates) {
+      if (t.has(key)) {
+        matchedKey = key;
+        break;
+      }
+    }
+  }
+
+  if (matchedKey) {
+    return Object.assign({}, issue, {
+      message: t(matchedKey, params),
+      key: matchedKey,
+    });
+  }
+
+  // If no translation found, interpolate params in custom message if it contains placeholders
+  if (typeof issue.message === 'string' && issue.message.includes('{{')) {
+    const interpolated = issue.message.replace(/{{\s*(\w+)\s*}}/g, (_, k) =>
+      params[k] !== undefined ? String(params[k]) : ''
+    );
+    return Object.assign({}, issue, { message: interpolated });
+  }
+
+  return Object.assign({}, issue);
+}
+
+/**
+ * Translates an array of Zod validation issues using the active request locale
+ * @param {Array<Object>} issues - Zod error issues array
+ * @param {Object} [options]
+ * @param {import('express').Request} [options.req] - Express request
+ * @param {Function} [options.t] - Optional direct translator
+ * @param {string} [options.locale] - Optional direct locale
+ * @param {string} [options.pagesDir] - Pages directory path
+ * @param {string} [options.routeDir] - Route directory path
+ * @param {boolean} [options.isDev] - Is development environment
+ * @returns {Array<Object>} Translated issues
+ */
+function translateValidationIssues(issues, options = {}) {
+  if (!Array.isArray(issues) || issues.length === 0) {
+    return issues;
+  }
+
+  let t = options.t || options.req?.t;
+  let locale = options.locale || options.req?.locale;
+
+  if (!t && options.req) {
+    const tr = getTranslatorForRequest(options.req, options);
+    t = tr.t;
+    locale = tr.locale;
+  }
+
+  if (!t) {
+    return issues;
+  }
+
+  return issues.map((issue) => translateSingleIssue(issue, t));
+}
+
+module.exports = {
+  getTranslatorForRequest,
+  translateSingleIssue,
+  translateValidationIssues,
+};