webspresso 0.0.102 → 0.0.104

package/plugins/site-analytics/tracking.js

@@ -5,12 +5,73 @@
  */
 
 const crypto = require('crypto');
+const { getClientIp: resolveClientIp } = require('../../core/utils/ip');
 const { detectBot } = require('./bot-patterns');
 const { detectCountry } = require('./geo');
 
 const STATIC_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map|webp|avif|mp4|webm)$/i;
 const DEFAULT_EXCLUDE = ['/_admin', '/api/', '/health', '/_next', '/__'];
 
+/**
+ * Permanent / Fatal DB error codes that cannot be recovered by immediate retries.
+ */
+const PERMANENT_ERROR_CODES = new Set([
+  'SQLITE_FULL',
+  'SQLITE_IOERR',
+  'SQLITE_CORRUPT',
+  'SQLITE_CANTOPEN',
+  'SQLITE_NOTADB',
+  'ER_DISK_FULL',
+  'ENOSPC',
+]);
+
+/**
+ * Patterns matching fatal disk full or corrupted database errors.
+ */
+const PERMANENT_ERROR_PATTERNS = [
+  /database or disk is full/i,
+  /disk full/i,
+  /no space left on device/i,
+  /disk i\/o error/i,
+  /database disk image is malformed/i,
+  /file is not a database/i,
+];
+
+/**
+ * Sanitize and format database error messages to prevent raw SQL query leakage in logs.
+ * Knex appends executed queries ("insert into ... - <actual error>") which can be 50-100 KB.
+ * @param {Error|Object|string} e
+ * @returns {string}
+ */
+function formatDbError(e) {
+  if (!e) return 'Unknown error';
+
+  let msg;
+  if (typeof e.message === 'string' && e.message.includes(' - ')) {
+    msg = e.message.split(' - ').pop().trim();
+  } else if (typeof e.message === 'string') {
+    msg = e.message.trim();
+  } else if (typeof e === 'string') {
+    msg = e.trim();
+  } else {
+    msg = String(e.code || 'Database error');
+  }
+
+  return msg.replace(/[\r\n]+/g, ' ').trim().slice(0, 150);
+}
+
+/**
+ * Detect whether a database error is permanent/fatal (e.g. disk full, corruption).
+ * @param {Error|Object|string} e
+ * @returns {boolean}
+ */
+function isPermanentDbError(e) {
+  if (!e) return false;
+  if (e.code && PERMANENT_ERROR_CODES.has(e.code)) return true;
+  const msg = typeof e.message === 'string' ? e.message : (typeof e === 'string' ? e : '');
+  return PERMANENT_ERROR_PATTERNS.some((pattern) => pattern.test(msg));
+}
+
 /**
  * Hash a string with SHA-256 (truncated to 16 hex chars for compactness)
  */
@@ -22,24 +83,25 @@ function quickHash(str) {
  * Get client IP from request, handling proxies
  */
 function getClientIp(req) {
-  const forwarded = req.headers['x-forwarded-for'];
-  if (forwarded) {
-    return forwarded.split(',')[0].trim();
-  }
-  return req.headers['x-real-ip'] || req.ip || req.connection?.remoteAddress || '';
+  return resolveClientIp(req, '');
 }
 
 /**
  * Create tracking middleware
- * @param {Object} options
- * @param {Object} options.knex - Knex instance
+ * @param {Object} [options={}]
+ * @param {Object} [options.knex] - Knex instance
  * @param {string[]} [options.excludePaths] - Paths to exclude from tracking
  * @param {boolean} [options.trackBots=true] - Whether to record bot visits
  * @param {string} [options.tableName='analytics_page_views'] - DB table name
  * @param {number} [options.batchSize=20] - Flush when queue reaches this size
  * @param {number} [options.flushIntervalMs=3000] - Flush interval for low traffic
+ * @param {number} [options.maxRetries=3] - Maximum retry attempts per batch item before dropping
+ * @param {number} [options.maxBackoffMs=60000] - Max exponential backoff cooldown
+ * @param {number} [options.maxQueueSize=5000] - Queue memory cap to prevent OOM
+ * @param {number} [options.errorLogIntervalMs=60000] - Rate limit window for batch failure logs
  */
-function createTrackingMiddleware(options) {
+function createTrackingMiddleware(options = {}) {
+  const opts = options || {};
   const {
     knex,
     excludePaths = [],
@@ -47,12 +109,26 @@ function createTrackingMiddleware(options) {
     tableName = 'analytics_page_views',
     batchSize = 20,
     flushIntervalMs = 3000,
-  } = options;
+    maxRetries = 3,
+    maxBackoffMs = 60000,
+    maxQueueSize = 5000,
+    errorLogIntervalMs = 60000,
+  } = opts;
 
   const allExcludes = [...DEFAULT_EXCLUDE, ...excludePaths];
   let tableReady = false;
   const queue = [];
-  let flushScheduled = false;
+
+  // Circuit breaker & backoff state
+  let failureCount = 0;
+  let nextAllowedFlushTime = 0;
+  let isFlushing = false;
+  let flushPromise = null;
+  let flushTimeout = null;
+
+  // Log throttling state
+  let lastErrorLogTime = 0;
+  let suppressedErrorCount = 0;
 
   async function ensureTable() {
     if (tableReady) return;
@@ -75,7 +151,26 @@ function createTrackingMiddleware(options) {
       }
       tableReady = true;
     } catch (e) {
-      console.error('[site-analytics] Failed to ensure table:', e.message);
+      logError(e, 'Failed to ensure table');
+      throw e;
+    }
+  }
+
+  function logError(e, prefix = 'Batch insert failed') {
+    const now = Date.now();
+    const formatted = formatDbError(e);
+
+    if (lastErrorLogTime === 0 || now - lastErrorLogTime >= errorLogIntervalMs) {
+      const elapsedSec = lastErrorLogTime > 0 ? Math.round((now - lastErrorLogTime) / 1000) : 0;
+      if (suppressedErrorCount > 0) {
+        console.error(`[site-analytics] ${prefix} (suppressed ${suppressedErrorCount} occurrences in last ${elapsedSec}s): ${formatted}`);
+      } else {
+        console.error(`[site-analytics] ${prefix}: ${formatted}`);
+      }
+      lastErrorLogTime = now;
+      suppressedErrorCount = 0;
+    } else {
+      suppressedErrorCount++;
     }
   }
 
@@ -83,17 +178,17 @@ function createTrackingMiddleware(options) {
   const sessionMap = new Map();
   const SESSION_TIMEOUT = 30 * 60 * 1000; // 30 minutes
   const MAX_SESSIONS = 10000;
-  const MAX_QUEUE_SIZE = 5000;
 
   // Periodic session cleanup
-  setInterval(() => {
+  const sessionCleanupInterval = setInterval(() => {
     const now = Date.now();
     for (const [vid, data] of sessionMap) {
       if (now - data.lastSeen > SESSION_TIMEOUT * 2) {
         sessionMap.delete(vid);
       }
     }
-  }, 5 * 60 * 1000).unref();
+  }, 5 * 60 * 1000);
+  if (sessionCleanupInterval.unref) sessionCleanupInterval.unref();
 
   function getSessionId(visitorId) {
     const now = Date.now();
@@ -115,86 +210,193 @@ function createTrackingMiddleware(options) {
   }
 
   async function flushQueue() {
-    if (queue.length === 0) return;
-    const batch = queue.splice(0, queue.length);
+    if (flushPromise) return flushPromise;
+    if (queue.length === 0 || !knex) return;
 
-    try {
-      await ensureTable();
-      for (const row of batch) {
-        row.created_at = knex.fn.now();
-      }
-      await knex(tableName).insert(batch);
-    } catch (e) {
-      console.error('[site-analytics] Batch insert failed:', e.message);
-      // Re-queue failed items with max size limit to prevent memory leak
-      if (queue.length + batch.length <= MAX_QUEUE_SIZE) {
-        queue.unshift(...batch);
-      } else {
-        const available = Math.max(0, MAX_QUEUE_SIZE - queue.length);
-        if (available > 0) {
-          queue.unshift(...batch.slice(batch.length - available));
+    const now = Date.now();
+    if (now < nextAllowedFlushTime) {
+      scheduleFlush(nextAllowedFlushTime - now);
+      return;
+    }
+
+    isFlushing = true;
+    flushPromise = (async () => {
+      const batch = queue.splice(0, queue.length);
+
+      try {
+        await ensureTable();
+        const dbBatch = batch.map(({ _retries, ...row }) => ({
+          ...row,
+          created_at: knex.fn.now(),
+        }));
+        await knex(tableName).insert(dbBatch);
+
+        // Successfully inserted: reset circuit breaker & failure counters
+        failureCount = 0;
+        nextAllowedFlushTime = 0;
+        lastErrorLogTime = 0;
+        suppressedErrorCount = 0;
+      } catch (e) {
+        failureCount++;
+        const backoffDelay = Math.min(flushIntervalMs * Math.pow(2, failureCount), maxBackoffMs);
+        nextAllowedFlushTime = Date.now() + backoffDelay;
+
+        // Rate-limited sanitized logging (no raw SQL query leakage)
+        logError(e, 'Batch insert failed');
+
+        // Schedule next attempt respecting backoff
+        scheduleFlush(backoffDelay);
+
+        // Drop policy: If fatal/permanent error (disk full, corrupted DB, etc.), drop entire batch
+        if (!isPermanentDbError(e)) {
+          const retryable = [];
+          for (const item of batch) {
+            item._retries = (item._retries || 0) + 1;
+            if (item._retries < maxRetries) {
+              retryable.push(item);
+            }
+          }
+
+          // Re-queue items that haven't exceeded retry limits, respecting maxQueueSize
+          if (retryable.length > 0) {
+            const available = Math.max(0, maxQueueSize - queue.length);
+            if (available > 0) {
+              queue.unshift(...retryable.slice(0, available));
+            }
+          }
         }
+      } finally {
+        isFlushing = false;
+        flushPromise = null;
       }
-    }
+    })();
+
+    return flushPromise;
   }
 
-  function scheduleFlush() {
-    if (flushScheduled || queue.length === 0) return;
-    flushScheduled = true;
-    setTimeout(() => {
-      flushScheduled = false;
+  function scheduleFlush(customDelay) {
+    if (flushTimeout || queue.length === 0) return;
+    const now = Date.now();
+    const delay = customDelay !== undefined
+      ? customDelay
+      : Math.max(flushIntervalMs, nextAllowedFlushTime - now, 0);
+
+    flushTimeout = setTimeout(() => {
+      flushTimeout = null;
       flushQueue().catch(() => {});
-    }, flushIntervalMs);
+    }, delay);
+    if (flushTimeout.unref) flushTimeout.unref();
   }
 
   // Periodic flush (catches low-traffic case)
-  setInterval(() => {
-    if (queue.length > 0) flushQueue().catch(() => {});
-  }, flushIntervalMs).unref();
+  const periodicFlushInterval = setInterval(() => {
+    if (queue.length > 0 && Date.now() >= nextAllowedFlushTime && !isFlushing) {
+      flushQueue().catch(() => {});
+    }
+  }, flushIntervalMs);
+  if (periodicFlushInterval.unref) periodicFlushInterval.unref();
 
-  return async function trackingMiddleware(req, res, next) {
-    next();
+  async function trackingMiddleware(req, res, next) {
+    if (typeof next === 'function') next();
 
-    if (req.method !== 'GET') return;
+    if (!req || req.method !== 'GET') return;
 
-    const path = req.path;
-    if (STATIC_EXT.test(path)) return;
+    const path = req.path || req.url || '';
+    if (!path || STATIC_EXT.test(path)) return;
     if (allExcludes.some(prefix => path.startsWith(prefix))) return;
 
     try {
-      const userAgent = req.headers['user-agent'] || '';
+      const headers = req.headers || {};
+      const userAgent = headers['user-agent'] || '';
       const ip = getClientIp(req);
       const { isBot, botName } = detectBot(userAgent);
 
       if (isBot && !trackBots) return;
 
+      // Protect queue under pressure or during active database failures:
+      // Drop bots immediately to preserve queue capacity and avoid wasting DB I/O
+      if (isBot && (failureCount > 0 || queue.length >= maxQueueSize * 0.7)) {
+        return;
+      }
+
       const ipHash = quickHash(ip);
       const visitorId = quickHash(ip + '|' + userAgent);
-      const sessionId = getSessionId(visitorId);
+
+      // Do not thrash human sessionMap with bot/crawler visits
+      const sessionId = isBot
+        ? quickHash('bot-' + (botName || 'crawler') + '-' + visitorId)
+        : getSessionId(visitorId);
+
       const country = detectCountry(req);
-      const referrer = req.headers['referer'] || req.headers['referrer'] || null;
-
-      queue.push({
-        session_id: sessionId,
-        visitor_id: visitorId,
-        path,
-        referrer: referrer ? referrer.slice(0, 500) : null,
-        user_agent: userAgent.slice(0, 1000),
-        ip_hash: ipHash,
-        country,
-        is_bot: isBot,
-        bot_name: botName,
-      });
+      const referrer = headers['referer'] || headers['referrer'] || null;
+
+      if (queue.length < maxQueueSize) {
+        queue.push({
+          session_id: sessionId,
+          visitor_id: visitorId,
+          path,
+          referrer: referrer ? referrer.slice(0, 500) : null,
+          user_agent: userAgent.slice(0, 1000),
+          ip_hash: ipHash,
+          country,
+          is_bot: isBot,
+          bot_name: botName,
+        });
+      }
 
       if (queue.length >= batchSize) {
-        flushQueue().catch(() => {});
+        if (Date.now() >= nextAllowedFlushTime) {
+          flushQueue().catch(() => {});
+        } else {
+          scheduleFlush();
+        }
       } else {
         scheduleFlush();
       }
     } catch (e) {
       // Non-blocking: don't let tracking errors affect the app
     }
+  }
+
+  // Diagnostic / testing accessors
+  trackingMiddleware.flush = () => flushQueue();
+  trackingMiddleware.getQueue = () => queue;
+  trackingMiddleware.getFailureCount = () => failureCount;
+  trackingMiddleware.getNextAllowedFlushTime = () => nextAllowedFlushTime;
+  trackingMiddleware.resetState = (opts = {}) => {
+    failureCount = 0;
+    nextAllowedFlushTime = 0;
+    const resetLogs = typeof opts === 'boolean' ? opts : !!opts.resetLogs;
+    if (resetLogs) {
+      lastErrorLogTime = 0;
+      suppressedErrorCount = 0;
+    }
+    queue.length = 0;
+    if (flushTimeout) {
+      clearTimeout(flushTimeout);
+      flushTimeout = null;
+    }
   };
+  trackingMiddleware.stop = () => {
+    if (flushTimeout) {
+      clearTimeout(flushTimeout);
+      flushTimeout = null;
+    }
+    clearInterval(sessionCleanupInterval);
+    clearInterval(periodicFlushInterval);
+    if (queue.length > 0 && knex) {
+      return flushQueue().catch(() => {});
+    }
+    return Promise.resolve();
+  };
+
+  return trackingMiddleware;
 }
 
-module.exports = { createTrackingMiddleware };
+module.exports = {
+  createTrackingMiddleware,
+  formatDbError,
+  isPermanentDbError,
+  PERMANENT_ERROR_CODES,
+  PERMANENT_ERROR_PATTERNS,
+};

package/plugins/sitemap.js

@@ -157,8 +157,9 @@ function sitemapPlugin(options = {}) {
   const dynamicExclusions = [...exclude];
   const registeredSources = [...dynamicSources];
   
-  // Cached URLs (for dynamic sources)
+  // Cached URLs and pre-rendered XML (for dynamic sources)
   let cachedUrls = null;
+  let cachedXml = null;
   let cacheTime = null;
   const cacheMaxAge = options.cacheMaxAge || 5 * 60 * 1000; // 5 minutes default
   
@@ -326,7 +327,8 @@ function sitemapPlugin(options = {}) {
        */
       addUrl(path, opts = {}) {
         dynamicUrls.push({ path, ...opts });
-        cachedUrls = null; // Invalidate cache
+        cachedUrls = null;
+        cachedXml = null;
       },
       
       /**
@@ -335,7 +337,8 @@ function sitemapPlugin(options = {}) {
        */
       exclude(pattern) {
         dynamicExclusions.push(pattern);
-        cachedUrls = null; // Invalidate cache
+        cachedUrls = null;
+        cachedXml = null;
       },
       
       /**
@@ -347,7 +350,8 @@ function sitemapPlugin(options = {}) {
           throw new Error('urlPattern is required for dynamic source');
         }
         registeredSources.push(source);
-        cachedUrls = null; // Invalidate cache
+        cachedUrls = null;
+        cachedXml = null;
       },
       
       /**
@@ -369,6 +373,7 @@ function sitemapPlugin(options = {}) {
        */
       invalidateCache() {
         cachedUrls = null;
+        cachedXml = null;
         cacheTime = null;
       }
     },
@@ -384,28 +389,26 @@ function sitemapPlugin(options = {}) {
         try {
           // Check cache
           const now = Date.now();
-          if (cachedUrls && cacheTime && (now - cacheTime) < cacheMaxAge) {
+          if (cachedXml && cacheTime && (now - cacheTime) < cacheMaxAge) {
             res.type('application/xml');
-            return res.send(generateSitemapXml(cachedUrls, { 
-              hostname, 
-              defaultChangefreq: changefreq, 
-              defaultPriority: priority 
-            }));
+            return res.send(cachedXml);
           }
           
           // Build URLs
           const urls = await buildAllUrls(ctx.routes, dbInstance);
+          const xml = generateSitemapXml(urls, { 
+            hostname, 
+            defaultChangefreq: changefreq, 
+            defaultPriority: priority 
+          });
           
           // Cache results
           cachedUrls = urls;
+          cachedXml = xml;
           cacheTime = now;
           
           res.type('application/xml');
-          res.send(generateSitemapXml(urls, { 
-            hostname, 
-            defaultChangefreq: changefreq, 
-            defaultPriority: priority 
-          }));
+          res.send(xml);
         } catch (err) {
           console.error('Sitemap generation error:', err);
           res.status(500).send('Error generating sitemap');

package/plugins/swagger.js

@@ -11,20 +11,7 @@ const { buildOpenApiDocument } = require('../core/openapi/build-from-api-routes'
 
 const PKG = require(path.join(__dirname, '..', 'package.json'));
 
-/**
- * HTML Escaping
- * @param {string} str
- * @returns {string}
- */
-function escapeHtml(str) {
-  if (!str) return '';
-  return String(str)
-    .replace(/&/g, '&')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-    .replace(/'/g, '&#039;');
-}
+const { escapeHtml } = require('../core/utils/html');
 
 /**
  * Generates Swagger UI HTML page
@@ -258,10 +245,15 @@ function swaggerPlugin(options = {}) {
 
       const server = serverUrl || process.env.BASE_URL || 'http://localhost:3000';
 
+      let cachedDoc = null;
+
       // 1. OpenAPI 3 JSON Endpoint
       ctx.addRoute('get', jsonPath, (req, res) => {
         if (forbid(req, res)) return;
         try {
+          if (cachedDoc && process.env.NODE_ENV === 'production') {
+            return res.json(cachedDoc);
+          }
           const doc = buildOpenApiDocument({
             routes: ctx.routes || [],
             pagesDir: ctx.options.pagesDir,
@@ -274,6 +266,9 @@ function swaggerPlugin(options = {}) {
             },
             servers: [{ url: server.replace(/\/$/, '') }],
           });
+          if (process.env.NODE_ENV === 'production') {
+            cachedDoc = doc;
+          }
           res.json(doc);
         } catch (err) {
           console.warn('[swagger] OpenAPI generation failed:', err.message);

package/plugins/webhooks/crypto.js

@@ -20,24 +20,7 @@ function generateSecret(prefix = 'whsec_', byteLength = 24) {
   return `${prefix || ''}${bytes}`;
 }
 
-/**
- * Perform constant-time string comparison to prevent timing attacks
- * 
- * @param {string} a 
- * @param {string} b 
- * @returns {boolean}
- */
-function safeCompare(a, b) {
-  if (typeof a !== 'string' || typeof b !== 'string') {
-    return false;
-  }
-  const bufA = Buffer.from(a, 'utf8');
-  const bufB = Buffer.from(b, 'utf8');
-  if (bufA.length !== bufB.length) {
-    return false;
-  }
-  return crypto.timingSafeEqual(bufA, bufB);
-}
+const { safeCompare } = require('../../core/utils/crypto');
 
 /**
  * Sign a webhook payload using HMAC-SHA256

package/plugins/webhooks/dispatcher.js

@@ -461,9 +461,14 @@ class WebhookDispatcher extends EventEmitter {
             attempt: attempt + 1,
             autoRetry: true,
           }).catch((err) => {
-            this.emit('error', err);
+            if (this.listenerCount('error') > 0) {
+              this.emit('error', err);
+            } else {
+              console.error('[webhooks] Delivery retry failed:', err?.message || err);
+            }
           });
         }, delayMs);
+        if (timer.unref) timer.unref();
         this._activeTimers.add(timer);
       }
     } else {

package/plugins/webhooks/matcher.js

@@ -48,6 +48,25 @@ function normalizeEvents(events) {
   return ['*'];
 }
 
+const regexCache = new Map();
+const MAX_REGEX_CACHE = 1000;
+
+function getPatternRegex(pattern) {
+  let cached = regexCache.get(pattern);
+  if (!cached) {
+    if (regexCache.size >= MAX_REGEX_CACHE) {
+      const first = regexCache.keys().next().value;
+      if (first !== undefined) regexCache.delete(first);
+    }
+    const escaped = pattern
+      .replace(/[.+?^${}()|[\]\\]/g, '\\$&') // escape regex specials except *
+      .replace(/\*/g, '.*');
+    cached = new RegExp(`^${escaped}$`, 'i');
+    regexCache.set(pattern, cached);
+  }
+  return cached;
+}
+
 /**
  * Test whether an event pattern matches an actual event name
  * 
@@ -56,12 +75,15 @@ function normalizeEvents(events) {
  * @returns {boolean}
  */
 function matchEvent(pattern, eventName) {
-  if (!pattern || !eventName) {
+  if (typeof pattern !== 'string' || typeof eventName !== 'string') {
     return false;
   }
 
   const p = pattern.trim();
   const e = eventName.trim();
+  if (!p || !e) {
+    return false;
+  }
 
   // 1. Wildcard matches all
   if (p === '*') {
@@ -73,15 +95,8 @@ function matchEvent(pattern, eventName) {
     return true;
   }
 
-  // 3. Glob / Wildcard conversion
-  // Convert 'user.*' to regular expression '^user\.[^.]+$' or '^user\..+$'
-  // Support prefix, middle, or suffix wildcards safely
-  const escaped = p
-    .replace(/[.+?^${}()|[\]\\]/g, '\\$&') // escape regex specials except *
-    .replace(/\*/g, '.*');
-
-  const regex = new RegExp(`^${escaped}$`, 'i');
-  return regex.test(e);
+  // 3. Glob / Wildcard conversion with bounded cache
+  return getPatternRegex(p).test(e);
 }
 
 /**