webspresso 0.0.89 → 0.0.91

package/src/server.js

@@ -3,6 +3,9 @@
  * Express + Nunjucks SSR server with file-based routing
  */
 
+const fs = require('fs');
+const path = require('path');
+const { AsyncLocalStorage } = require('async_hooks');
 const express = require('express');
 const helmet = require('helmet');
 const nunjucks = require('nunjucks');
@@ -11,9 +14,116 @@ const timeout = require('connect-timeout');
 const { setAppContext } = require('./app-context');
 const { mountClientRuntime } = require('./client-runtime/mount');
 const { resolveClientRuntime } = require('./client-runtime/resolve');
-const { mountPages, detectLocale } = require('./file-router');
+const { mountPages, detectLocale, loadI18n, createTranslator } = require('./file-router');
 const { configureAssets, createHelpers, getScriptInjector } = require('./helpers');
 const { createPluginManager } = require('./plugin-manager');
+const { createServiceRegistry } = require('./services');
+const { ShutdownManager, NodeHttpAdapter } = require('../core/shutdown');
+const { createCompressionMiddleware } = require('../core/compression');
+const {
+  WebspressoError,
+  HttpError,
+  NotFoundError,
+  RouteNotFoundError,
+  RequestAbortedError,
+  ValidationError,
+  normalizeError,
+  toErrorResponseObject,
+  preferJsonErrorResponse,
+} = require('../core/errors');
+
+// Async storage for tracking template render call stacks (circular extends guard)
+const renderStackStorage = new AsyncLocalStorage();
+
+/**
+ * Configure Nunjucks environment with circular extends guard and loader protection
+ * @param {string|string[]} templateDirs - Template directory paths
+ * @param {Object} options - Nunjucks options
+ * @returns {nunjucks.Environment}
+ */
+function configureSafeNunjucks(templateDirs, options = {}) {
+  const env = nunjucks.configure(templateDirs, options);
+  const origGetTemplate = env.getTemplate.bind(env);
+  const origRender = env.render.bind(env);
+  const origRenderString = env.renderString.bind(env);
+
+  function wrapTemplateRoot(tmpl, name, store) {
+    if (!tmpl || tmpl._safeWrapped) return;
+    tmpl._safeWrapped = true;
+    const origRoot = tmpl.rootRenderFunc;
+    if (typeof origRoot !== 'function') return;
+
+    tmpl.rootRenderFunc = function(e, context, frame, runtime, renderCb) {
+      const normalizedName = String(name || tmpl.path || 'anonymous');
+      store.stack.push(normalizedName);
+      let finished = false;
+      const done = (err, out) => {
+        if (!finished) {
+          finished = true;
+          const idx = store.stack.lastIndexOf(normalizedName);
+          if (idx !== -1) store.stack.splice(idx, 1);
+        }
+        renderCb(err, out);
+      };
+      try {
+        return origRoot.call(this, e, context, frame, runtime, done);
+      } catch (ex) {
+        const idx = store.stack.lastIndexOf(normalizedName);
+        if (idx !== -1) store.stack.splice(idx, 1);
+        throw ex;
+      }
+    };
+  }
+
+  env.getTemplate = function(name, eagerCompile, parentName, ignoreMissing, cb) {
+    if (typeof parentName === 'function') {
+      cb = parentName;
+      parentName = null;
+    }
+    if (typeof eagerCompile === 'function') {
+      cb = eagerCompile;
+      eagerCompile = false;
+    }
+
+    const store = renderStackStorage.getStore();
+    if (store && name) {
+      const normalizedName = String(name);
+      if (store.stack.includes(normalizedName)) {
+        const cycle = [...store.stack, normalizedName].join(' -> ');
+        const err = new Error(`Circular template extension detected: ${cycle}`);
+        if (typeof cb === 'function') return cb(err);
+        throw err;
+      }
+    }
+
+    const wrappedCb = typeof cb === 'function' ? function(err, tmpl) {
+      if (tmpl && store) {
+        wrapTemplateRoot(tmpl, name, store);
+      }
+      cb(err, tmpl);
+    } : undefined;
+
+    const res = origGetTemplate(name, eagerCompile, parentName, ignoreMissing, wrappedCb);
+    if (res && store) {
+      wrapTemplateRoot(res, name, store);
+    }
+    return res;
+  };
+
+  env.render = function(name, ctx, cb) {
+    return renderStackStorage.run({ stack: [] }, () => {
+      return origRender(name, ctx, cb);
+    });
+  };
+
+  env.renderString = function(src, ctx, opts, cb) {
+    return renderStackStorage.run({ stack: [] }, () => {
+      return origRenderString(src, ctx, opts, cb);
+    });
+  };
+
+  return env;
+}
 
 /**
  * Get default Helmet configuration
@@ -57,15 +167,7 @@ function getDefaultHelmetConfig(isDev) {
   };
 }
 
-/**
- * Use JSON error responses for `pages/api/*` routes and clients that do not prefer HTML.
- * @param {import('express').Request} req
- * @returns {boolean}
- */
-function preferJsonErrorResponse(req) {
-  if (req.path && req.path.startsWith('/api')) return true;
-  return !req.accepts('html');
-}
+
 
 /**
  * Shared CSS for built-in HTML error pages (viewport-safe, fluid type, dark mode)
@@ -310,9 +412,141 @@ function createApp(options = {}) {
 
   const clientRuntime = resolveClientRuntime(options);
 
-  setAppContext({ db: options.db ?? null });
+  const shutdownConfig = {
+    enabled: true,
+    mode: 'graceful',
+    timeout: 10_000,
+    ...(options.shutdown || options.server?.shutdown || {}),
+  };
+
+  const shutdownManager = new ShutdownManager({
+    ...shutdownConfig,
+    logger: logging ? console : null,
+  });
+
+  // Services Layer initialization
+  const servicesDirOption = options.servicesDir ?? options.services?.dir;
+  const resolvedServicesDir = servicesDirOption
+    ? path.resolve(servicesDirOption)
+    : (fs.existsSync(path.join(process.cwd(), 'services'))
+      ? path.join(process.cwd(), 'services')
+      : null);
+
+  const serviceRegistry = createServiceRegistry({
+    servicesDir: resolvedServicesDir,
+    isDev,
+    logger: logging ? console : null,
+  });
+
+  // Auto-register built-in auth services if database is present and service is not custom-defined
+  if (options.db) {
+    const { createAuthServices } = require('./services/builtins/auth');
+    const authServices = createAuthServices({ db: options.db });
+    for (const [name, def] of Object.entries(authServices)) {
+      if (!serviceRegistry.has(name)) {
+        serviceRegistry.register(name, def);
+      }
+    }
+  }
+
+  // Auto-register built-in system services (health, cleanup, cache flush, info)
+  const { createSystemServices } = require('./services/builtins/system');
+  const systemServices = createSystemServices({
+    db: options.db ?? null,
+    serviceRegistry,
+    pluginManager,
+  });
+  for (const [name, def] of Object.entries(systemServices)) {
+    if (!serviceRegistry.has(name)) {
+      serviceRegistry.register(name, def);
+    }
+  }
+
+  setAppContext({ db: options.db ?? null, shutdownManager, serviceRegistry });
   
   const app = express();
+  app.serviceRegistry = serviceRegistry;
+
+  // Services caller middleware
+  app.use((req, res, next) => {
+    req.service = (name, input, opts) =>
+      serviceRegistry.call(name, input, { req, res, db: options.db ?? null, ...(req.context || {}) }, opts);
+    next();
+  });
+
+  // SSR Streaming Response Helper
+  const { renderStream } = require('../core/ssr/stream');
+  app.use((req, res, next) => {
+    res.renderStream = (templatePath, context = {}, renderOptions = {}) => {
+      return renderStream(res, templatePath, context, {
+        env: nunjucksEnv,
+        ...renderOptions,
+      });
+    };
+    next();
+  });
+
+  // Async handler wrapper helper for automatic promise rejection handling
+  function wrapAsync(fn) {
+    if (typeof fn !== 'function') return fn;
+    if (fn.length === 4) {
+      return function(err, req, res, next) {
+        try {
+          const ret = fn.call(this, err, req, res, next);
+          if (ret && typeof ret.catch === 'function') {
+            ret.catch(next);
+          }
+          return ret;
+        } catch (syncErr) {
+          return next(syncErr);
+        }
+      };
+    }
+    return function(req, res, next) {
+      try {
+        const ret = fn.call(this, req, res, next);
+        if (ret && typeof ret.catch === 'function') {
+          ret.catch(next);
+        }
+        return ret;
+      } catch (syncErr) {
+        return next(syncErr);
+      }
+    };
+  }
+
+  // Wrap routing methods on app to catch uncaught async errors
+  const HTTP_METHODS = ['get', 'post', 'put', 'delete', 'patch', 'options', 'head', 'all'];
+  for (const method of HTTP_METHODS) {
+    const origMethod = app[method];
+    if (typeof origMethod === 'function') {
+      app[method] = function(path, ...handlers) {
+        const wrappedHandlers = handlers.flat().map((h) => wrapAsync(h));
+        return origMethod.call(this, path, ...wrappedHandlers);
+      };
+    }
+  }
+
+  // Request draining check during graceful shutdown (zero overhead during normal operation)
+  app.use((req, res, next) => {
+    if (shutdownManager.isShuttingDown) {
+      res.set('Connection', 'close');
+      if (!res.headersSent) {
+        if (preferJsonErrorResponse(req)) {
+          return res.status(503).json({ error: 'Service Unavailable', message: 'Server is shutting down', status: 503 });
+        }
+        return res.status(503).send('Server is shutting down');
+      }
+      res.on('finish', () => {
+        try {
+          if (req.socket && !req.socket.destroyed) {
+            req.socket.end();
+          }
+        } catch (e) {}
+      });
+    }
+    next();
+  });
   
   // Security headers with Helmet
   if (helmetConfig !== false) {
@@ -359,19 +593,46 @@ function createApp(options = {}) {
     
     app.use(helmet(finalConfig));
   }
+
+  // HTTP Response Compression
+  const compressionOpt = options.compression ?? options.server?.compression ?? false;
+  if (compressionOpt) {
+    const compressionConfig = typeof compressionOpt === 'object' ? compressionOpt : {};
+    if (compressionConfig.enabled !== false) {
+      app.use(createCompressionMiddleware(compressionConfig));
+    }
+  }
   
   // Request timeout middleware
   if (timeoutConfig !== false) {
     app.use(timeout(timeoutConfig));
   }
   
-  // Trust proxy (for correct req.ip, req.protocol behind reverse proxy)
-  app.set('trust proxy', 1);
+  // Trust proxy (for correct req.ip, req.protocol behind reverse proxy; configurable via options.trustProxy)
+  const trustProxySetting = options.trustProxy !== undefined
+    ? options.trustProxy
+    : (options.server?.trustProxy !== undefined ? options.server.trustProxy : 1);
+  if (trustProxySetting !== false) {
+    app.set('trust proxy', trustProxySetting);
+  } else {
+    app.set('trust proxy', false);
+  }
+
+  // Ensure extended query parser for nested filter/sort query parameters (?filter[field]=val)
+  app.set('query parser', 'extended');
   
   // JSON body parser for API routes
   app.use(express.json());
   app.use(express.urlencoded({ extended: true }));
   
+  // Ensure default empty object for body on mutating requests if not populated
+  app.use((req, res, next) => {
+    if (req.body === undefined && (req.method === 'POST' || req.method === 'PUT' || req.method === 'PATCH')) {
+      req.body = {};
+    }
+    next();
+  });
+  
   // Halt processing if request has timed out (after body parsers)
   if (timeoutConfig !== false) {
     app.use(haltOnTimedout);
@@ -405,10 +666,10 @@ function createApp(options = {}) {
 
   mountClientRuntime(app, clientRuntime);
 
-  // Configure Nunjucks
-  const templateDirs = viewsDir ? [pagesDir, viewsDir] : [pagesDir];
+  // Configure Nunjucks with viewsDir priority and circular extension guard
+  const templateDirs = viewsDir ? [viewsDir, pagesDir] : [pagesDir];
   
-  const nunjucksEnv = nunjucks.configure(templateDirs, {
+  const nunjucksEnv = configureSafeNunjucks(templateDirs, {
     autoescape: true,
     express: app,
     watch: isDev && !isTest,
@@ -417,7 +678,21 @@ function createApp(options = {}) {
   
   // Add custom Nunjucks filters
   nunjucksEnv.addFilter('json', (obj) => {
-    return JSON.stringify(obj, null, 2);
+    const raw = JSON.stringify(obj, null, 2);
+    if (!raw) return '';
+    return raw
+      .replace(/</g, '\\u003c')
+      .replace(/>/g, '\\u003e')
+      .replace(/&/g, '\\u0026');
+  });
+
+  // Translation filter: {{ 'nav.home' | t }} or {{ 'cart.items' | t({ count: 5 }) }}
+  nunjucksEnv.addFilter('t', function(key, params, defaultVal) {
+    const tFunc = this.ctx?.t || this.ctx?.fsy?.t;
+    if (typeof tFunc === 'function') {
+      return tFunc(key, params, defaultVal);
+    }
+    return defaultVal !== undefined && defaultVal !== null ? defaultVal : key;
   });
   
   nunjucksEnv.addFilter('date', (date, format = 'short') => {
@@ -440,7 +715,7 @@ function createApp(options = {}) {
   });
   
   // Register plugins (sync) — middlewares is the same object later passed to mountPages
-  const pluginContext = { app, nunjucksEnv, options, middlewares };
+  const pluginContext = { app, nunjucksEnv, options, middlewares, shutdownManager };
   pluginManager.registerSync(plugins, pluginContext);
   
   // Request logging middleware
@@ -468,6 +743,7 @@ function createApp(options = {}) {
     db: options.db ?? null,
     clientRuntime,
     pageAssets: options.pageAssets,
+    serviceRegistry,
   });
 
   // Set route metadata in plugin manager
@@ -510,6 +786,7 @@ function createApp(options = {}) {
       pluginManager,
       options,
       clientRuntime,
+      middlewares,
     });
   }
 
@@ -528,10 +805,25 @@ function createApp(options = {}) {
     // Merge plugin helpers
     const pluginHelpers = pluginManager.getHelpers();
     Object.assign(fsy, pluginHelpers);
+
+    // Provide safe i18n translator helper (primary + fallback)
+    const defaultLocale = process.env.DEFAULT_LOCALE || 'en';
+    const translations = loadI18n(pagesDir, '', locale);
+    const fallbackTranslations = (locale !== defaultLocale)
+      ? loadI18n(pagesDir, '', defaultLocale)
+      : undefined;
+    const t = createTranslator(translations, {
+      locale,
+      fallbackTranslations,
+      fallbackLocale: defaultLocale,
+    });
     
     return {
       fsy,
       locale,
+      t,
+      service: (name, input, opts) =>
+        serviceRegistry.call(name, input, { req, res: {}, fsy, locale, t, db: options.db ?? null }, opts),
       isDev,
       url: req.url,
       method: req.method,
@@ -539,20 +831,63 @@ function createApp(options = {}) {
     };
   }
   
+  // Pre-resolve error templates and loaders once at startup to avoid runtime synchronous fs operations
+  const resolved404Template = typeof errorPages.notFound === 'string'
+    ? errorPages.notFound
+    : ((viewsDir && fs.existsSync(path.join(viewsDir, '404.njk'))) || fs.existsSync(path.join(pagesDir, '404.njk')))
+      ? '404.njk'
+      : null;
+
+  const config404Path = path.join(pagesDir, '404.js');
+  const has404DataLoader = fs.existsSync(config404Path);
+
+  const resolved500Template = typeof errorPages.serverError === 'string'
+    ? errorPages.serverError
+    : ((viewsDir && fs.existsSync(path.join(viewsDir, '500.njk'))) || fs.existsSync(path.join(pagesDir, '500.njk')))
+      ? '500.njk'
+      : null;
+
+  const resolved503Template = typeof errorPages.timeout === 'string'
+    ? errorPages.timeout
+    : ((viewsDir && fs.existsSync(path.join(viewsDir, '503.njk'))) || fs.existsSync(path.join(pagesDir, '503.njk')))
+      ? '503.njk'
+      : null;
+
   // 404 handler
-  app.use((req, res) => {
+  app.use(async (req, res) => {
     res.status(404);
     const ctx = createErrorContext(req);
     
-    // Custom handler function
+    // 1. Custom handler function
     if (typeof errorPages.notFound === 'function') {
       return errorPages.notFound(req, res, ctx);
     }
     
-    // Custom template
-    if (typeof errorPages.notFound === 'string') {
+    // 2. Custom or auto-discovered 404 template (pre-resolved at startup)
+    const notFoundTemplate = resolved404Template;
+    
+    if (notFoundTemplate && !preferJsonErrorResponse(req)) {
       try {
-        const html = nunjucksEnv.render(errorPages.notFound, ctx);
+        // If pages/404.js exists, execute load() data loader
+        if (has404DataLoader) {
+          try {
+            if (isDev && require.cache[require.resolve(config404Path)]) {
+              delete require.cache[require.resolve(config404Path)];
+            }
+            const config404 = require(config404Path);
+            const loadFn = typeof config404 === 'function' ? config404 : config404.load;
+            if (typeof loadFn === 'function') {
+              const loadData = await loadFn({ req, res, db: ctx.db || null, ctx });
+              if (loadData && typeof loadData === 'object') {
+                Object.assign(ctx, loadData);
+              }
+            }
+          } catch (loadErr) {
+            console.error('Error executing 404.js load():', loadErr);
+          }
+        }
+
+        const html = nunjucksEnv.render(notFoundTemplate, ctx);
         return res.send(html);
       } catch (e) {
         console.error('Error rendering 404 template:', e);
@@ -560,15 +895,35 @@ function createApp(options = {}) {
     }
     
     // Default response
-    if (req.accepts('html')) {
+    if (!preferJsonErrorResponse(req)) {
       res.send(default404Html());
     } else {
       res.json({ error: 'Not Found', status: 404 });
     }
   });
   
-  // Error handler
-  app.use((err, req, res, next) => {
+  let customErrorHandler = null;
+  app.setErrorHandler = function(handler) {
+    customErrorHandler = typeof handler === 'function' ? handler : null;
+    return app;
+  };
+
+  // Central Error Handler
+  app.use(async (err, req, res, next) => {
+    if (res.headersSent) {
+      return next(err);
+    }
+    
+    // Client connection aborted
+    if (err instanceof RequestAbortedError || req.aborted || (req.socket && req.socket.destroyed)) {
+      if (!res.headersSent && !res.writableEnded) {
+        try {
+          res.end();
+        } catch (_) {}
+      }
+      return;
+    }
+
     // Handle timeout errors
     if (req.timedout) {
       console.error('Request timed out:', req.method, req.url);
@@ -580,10 +935,12 @@ function createApp(options = {}) {
         return errorPages.timeout(req, res, ctx);
       }
       
-      // Custom timeout template
-      if (typeof errorPages.timeout === 'string' && !preferJsonErrorResponse(req)) {
+      // Custom or auto-discovered timeout template (pre-resolved at startup)
+      const timeoutTemplate = resolved503Template;
+
+      if (timeoutTemplate && !preferJsonErrorResponse(req)) {
         try {
-          const html = nunjucksEnv.render(errorPages.timeout, ctx);
+          const html = nunjucksEnv.render(timeoutTemplate, ctx);
           return res.send(html);
         } catch (e) {
           console.error('Error rendering timeout template:', e);
@@ -597,12 +954,41 @@ function createApp(options = {}) {
         return res.json({ error: 'Request Timeout', status: 503 });
       }
     }
-    
-    console.error('Server error:', err);
-    res.status(err.status || 500);
+
+    const normalized = normalizeError(err, isDev);
+    const status = normalized.status || 500;
+
+    // Attach custom headers from HttpError if present
+    if (normalized.headers && typeof normalized.headers === 'object') {
+      for (const [key, value] of Object.entries(normalized.headers)) {
+        res.setHeader(key, value);
+      }
+    }
+
+    // Custom Error Handler hook (app.setErrorHandler)
+    if (customErrorHandler) {
+      try {
+        const handled = await customErrorHandler(err, req, res, next);
+        if (res.headersSent || handled === true) {
+          return;
+        }
+      } catch (customErr) {
+        console.error('[webspresso] Error in custom error handler:', customErr);
+      }
+    }
+
+    // Log errors based on severity
+    if (status >= 500) {
+      console.error('Server error:', err);
+    } else if (isDev && status >= 400) {
+      console.warn(`[webspresso] HTTP ${status}:`, normalized.message);
+    }
+
+    res.status(status);
     const ctx = createErrorContext(req, {
-      error: isDev ? err : { message: 'Internal Server Error' },
-      status: err.status || 500
+      error: isDev ? err : { message: normalized.message },
+      status,
+      normalizedError: normalized,
     });
     
     // Custom handler function
@@ -610,10 +996,12 @@ function createApp(options = {}) {
       return errorPages.serverError(err, req, res, ctx);
     }
     
-    // Custom template (skipped for /api and JSON-preferring clients so they never get HTML)
-    if (typeof errorPages.serverError === 'string' && !preferJsonErrorResponse(req)) {
+    // Custom or auto-discovered 500 template (pre-resolved at startup)
+    const serverErrorTemplate = resolved500Template;
+
+    if (status >= 500 && serverErrorTemplate && !preferJsonErrorResponse(req)) {
       try {
-        const html = nunjucksEnv.render(errorPages.serverError, ctx);
+        const html = nunjucksEnv.render(serverErrorTemplate, ctx);
         return res.send(html);
       } catch (e) {
         console.error('Error rendering 500 template:', e);
@@ -622,17 +1010,55 @@ function createApp(options = {}) {
     
     // Default response
     if (!preferJsonErrorResponse(req)) {
-      res.send(default500Html(err, isDev));
+      res.send(default500Html(normalized, isDev));
     } else {
-      res.json({ 
-        error: 'Internal Server Error', 
-        status: err.status || 500,
-        ...(isDev && { message: err.message, stack: err.stack })
-      });
+      res.json(toErrorResponseObject(normalized, isDev));
     }
   });
+
+  // Decorate app with shutdown lifecycle helpers
+  app.shutdownManager = shutdownManager;
+  Object.defineProperty(app, 'isShuttingDown', {
+    get: () => shutdownManager.isShuttingDown,
+    configurable: true,
+    enumerable: true,
+  });
+
+  app.onShutdown = function(fn) {
+    shutdownManager.onShutdown(fn);
+    return app;
+  };
+
+  app.close = function(reason) {
+    return shutdownManager.close(reason);
+  };
+
+  app.enableShutdownHooks = function() {
+    shutdownManager.enableShutdownHooks();
+    return app;
+  };
+
+  app.disableShutdownHooks = function() {
+    shutdownManager.disableShutdownHooks();
+    return app;
+  };
+
+  // Wrap app.listen to register server adapter and signal hooks
+  const origListen = app.listen.bind(app);
+  app.listen = function(...args) {
+    const server = origListen(...args);
+    app.server = server;
+    const adapter = new NodeHttpAdapter(server);
+    shutdownManager.registerAdapter(adapter);
+
+    if (shutdownConfig.enabled !== false && !isTest) {
+      shutdownManager.enableShutdownHooks();
+    }
+
+    return server;
+  };
   
-  return { app, nunjucksEnv, pluginManager, authMiddleware };
+  return { app, nunjucksEnv, pluginManager, authMiddleware, shutdownManager };
 }
 
 // Export for use as library