webspresso 0.0.95 → 0.0.97

package/src/pages/page-loader.js

@@ -0,0 +1,214 @@
+'use strict';
+
+/**
+ * Webspresso Page Loader
+ * Handles execution of definePage() lifecycle and renders page output
+ * @module src/pages/page-loader
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { createHelpers } = require('../helpers');
+const { loadI18n, createTranslator, detectLocale, resolveMiddlewares } = require('../file-router');
+
+const EMPTY_ARRAY = Object.freeze([]);
+
+/**
+ * Resolves named and functional middlewares with module-level precedence
+ * @param {Array<string|Function>} middlewareList
+ * @param {Object} [globalMiddlewares={}]
+ * @param {Object} [moduleMiddlewares={}]
+ * @returns {Array<Function>}
+ */
+function resolveMiddlewaresWithModule(middlewareList, globalMiddlewares = {}, moduleMiddlewares = {}) {
+  const combined = Object.assign({}, globalMiddlewares, moduleMiddlewares);
+  return resolveMiddlewares(middlewareList, combined);
+}
+
+/**
+ * Creates an Express route handler for a discovered Page descriptor
+ * @param {Object} descriptor - Route descriptor
+ * @param {Object} context - Server execution context
+ * @returns {Function} Express request handler
+ */
+function createPageHandler(descriptor, context) {
+  const {
+    nunjucks,
+    middlewares = {},
+    serviceRegistry = null,
+    db = null,
+    options: appOptions = {},
+  } = context;
+
+  const isDev = process.env.NODE_ENV === 'development';
+  const pagesDir = path.dirname(descriptor.file);
+  const njkCandidate = descriptor.file.endsWith('.njk')
+    ? descriptor.file
+    : descriptor.file.replace(/\.js$/, '.njk');
+  
+  let hasNjkTemplate = !isDev ? fs.existsSync(njkCandidate) : null;
+  let cachedMiddlewares = null;
+  let cachedPageDef = null;
+
+  return async (req, res, next) => {
+    try {
+      // 1. Load page definition (with hot reload in dev)
+      if (isDev && descriptor.file.endsWith('.js')) {
+        try {
+          const resolvedPath = require.resolve(descriptor.file);
+          if (require.cache[resolvedPath]) {
+            delete require.cache[resolvedPath];
+            cachedMiddlewares = null;
+            cachedPageDef = null;
+          }
+        } catch {}
+      }
+
+      let pageDef = cachedPageDef;
+      if (!pageDef) {
+        let pageModule = {};
+        if (descriptor.file.endsWith('.js')) {
+          try {
+            pageModule = require(descriptor.file);
+          } catch (loadErr) {
+            return next(loadErr);
+          }
+        }
+
+        pageDef = typeof pageModule === 'function' ? { load: pageModule } : pageModule.default || pageModule;
+        if (!isDev) {
+          cachedPageDef = pageDef;
+        }
+      }
+
+      // 2. Resolve middlewares (module-local first, cached across requests)
+      if (!cachedMiddlewares) {
+        const moduleMiddlewares = descriptor.moduleConfig?.middlewares || {};
+        const combinedRegistry = Object.assign({}, middlewares, moduleMiddlewares);
+        cachedMiddlewares = resolveMiddlewares(pageDef.middleware, combinedRegistry);
+      }
+
+      // Run middlewares in sequence
+      for (const mw of cachedMiddlewares) {
+        if (typeof mw !== 'function') continue;
+        await new Promise((resolve, reject) => {
+          try {
+            mw(req, res, (err) => {
+              if (err) reject(err);
+              else resolve();
+            });
+          } catch (syncErr) {
+            reject(syncErr);
+          }
+        });
+        if (res.headersSent) return;
+      }
+
+      // 3. Build Page Context
+      const baseUrl = process.env.BASE_URL || `http://localhost:${process.env.PORT || 3000}`;
+      const locale = detectLocale(req);
+      const fsy = createHelpers({ req, res, baseUrl, locale });
+
+      // i18n
+      const defaultLocale = process.env.DEFAULT_LOCALE || 'en';
+      const translations = loadI18n(pagesDir, '', locale, isDev);
+      const fallbackTranslations = locale !== defaultLocale ? loadI18n(pagesDir, '', defaultLocale, isDev) : undefined;
+      const t = createTranslator(translations, {
+        locale,
+        fallbackTranslations,
+        fallbackLocale: defaultLocale,
+      });
+
+      const pageCtx = {
+        req,
+        res,
+        params: req.params || {},
+        query: req.query || {},
+        service: (name, input, opts) =>
+          serviceRegistry ? serviceRegistry.call(name, input, { req, res, db }, opts) : null,
+        module: descriptor.module || null,
+        config: appOptions,
+        logger: console,
+        redirect: (url, status = 302) => {
+          res.redirect(status, url);
+          return { __redirected: true };
+        },
+        error: (status = 500, message = 'Page Error') => {
+          const err = new Error(message);
+          err.status = status;
+          throw err;
+        },
+        fsy,
+        locale,
+        t,
+        db,
+      };
+
+      // 4. Run load() lifecycle
+      let loadedData = {};
+      if (typeof pageDef.load === 'function') {
+        loadedData = (await pageDef.load(pageCtx)) || {};
+        if (loadedData.__redirected || res.headersSent) {
+          return;
+        }
+      }
+
+      // 5. Run head() lifecycle
+      let headData = {};
+      if (typeof pageDef.head === 'function') {
+        headData = (await pageDef.head(loadedData, pageCtx)) || {};
+      }
+
+      // 6. Check for companion Nunjucks template (cached in production)
+      const templateExists = hasNjkTemplate !== null ? hasNjkTemplate : fs.existsSync(njkCandidate);
+
+      if (templateExists && nunjucks) {
+        const templateData = {
+          ...loadedData,
+          head: headData,
+          pageHead: headData,
+          fsy,
+          locale,
+          t,
+          params: req.params,
+          query: req.query,
+        };
+        return nunjucks.render(njkCandidate, templateData, (err, html) => {
+          if (err) return next(err);
+          res.setHeader('Content-Type', 'text/html; charset=utf-8');
+          return res.send(html);
+        });
+      }
+
+      // 7. Render via definePage render() method
+      if (typeof pageDef.render === 'function') {
+        const html = await pageDef.render(loadedData, pageCtx);
+        res.setHeader('Content-Type', 'text/html; charset=utf-8');
+        return res.send(html);
+      }
+
+      // 8. Fallback: if json or data
+      if (typeof loadedData === 'object' && loadedData !== null) {
+        return res.json(loadedData);
+      }
+
+      return res.send(String(loadedData));
+    } catch (err) {
+      if (err && typeof err === 'object') {
+        if (!err.route) err.route = descriptor.path;
+        if (!err.method) err.method = descriptor.method ? descriptor.method.toUpperCase() : (req.method || 'GET');
+        if (!err.source) err.source = descriptor.source || descriptor.file;
+        if (!err.file) err.file = descriptor.file;
+        if (!err.module) err.module = descriptor.module || null;
+        if (!err.phase) err.phase = 'page';
+        if (!err.requestId) err.requestId = req.id || (req.headers && req.headers['x-request-id']) || null;
+      }
+      return next(err);
+    }
+  };
+}
+
+module.exports = {
+  createPageHandler,
+  resolveMiddlewaresWithModule,
+};

package/src/routing/mount-discovered-routes.js

@@ -0,0 +1,78 @@
+'use strict';
+
+/**
+ * Webspresso Route Mounting Layer
+ * Binds compiled RouteTable descriptors to Express routes with static/dynamic phase separation
+ * @module src/routing/mount-discovered-routes
+ */
+
+const { createPageHandler } = require('../pages/page-loader');
+const { createApiHandler } = require('../api/api-loader');
+const { getRouteSortMeta } = require('./route-table');
+
+const NOOP = () => {};
+
+/**
+ * Mounts discovered routes onto the Express application
+ * @param {import('express').Application} app
+ * @param {import('./route-table').RouteTable} routeTable
+ * @param {Object} context - Server context
+ * @returns {{ routeMetadata: Array, registerDynamicDiscoveredRoutes: Function }}
+ */
+function mountDiscoveredRoutes(app, routeTable, context) {
+  const { silent = false } = context;
+  const log = silent ? NOOP : console.log.bind(console);
+
+  const staticRoutes = [];
+  const dynamicRoutes = [];
+  const routeMetadata = [];
+
+  for (const descriptor of routeTable.routes) {
+    const sortMeta = descriptor._sortMeta || getRouteSortMeta(descriptor.path);
+    if (sortMeta.tier === 0) {
+      staticRoutes.push(descriptor);
+    } else {
+      dynamicRoutes.push(descriptor);
+    }
+
+    routeMetadata.push({
+      file: descriptor.source || descriptor.file,
+      method: descriptor.method.toUpperCase(),
+      path: descriptor.path,
+      type: descriptor.type,
+      module: descriptor.module || null,
+    });
+  }
+
+  function bindRoute(descriptor) {
+    const method = descriptor.method.toLowerCase();
+    const handler =
+      descriptor.type === 'page'
+        ? createPageHandler(descriptor, context)
+        : createApiHandler(descriptor, context);
+
+    app[method](descriptor.path, handler);
+    log(`  ${descriptor.method.toUpperCase().padEnd(6)} ${descriptor.path} -> ${descriptor.source || descriptor.file}`);
+  }
+
+  // Phase 1: Mount static routes immediately
+  for (const r of staticRoutes) {
+    bindRoute(r);
+  }
+
+  // Phase 2: Deferred function to mount dynamic and catch-all routes after plugins
+  function registerDynamicDiscoveredRoutes() {
+    for (const r of dynamicRoutes) {
+      bindRoute(r);
+    }
+  }
+
+  return {
+    routeMetadata,
+    registerDynamicDiscoveredRoutes,
+  };
+}
+
+module.exports = {
+  mountDiscoveredRoutes,
+};

package/src/routing/route-table.js

@@ -0,0 +1,191 @@
+'use strict';
+
+/**
+ * Webspresso Route Table & Compiler
+ * Compiles route descriptors, performs conflict checks, sorts deterministically, and provides introspection
+ * @module src/routing/route-table
+ */
+
+/**
+ * Computes route sorting priority metadata
+ * @param {string} routePath
+ * @returns {{ tier: number, literalSegCount: number, paramSegCount: number, depth: number, routePath: string }}
+ */
+function getRouteSortMeta(routePath) {
+  let hasStar = false;
+  let hasColon = false;
+  let depth = 0;
+  let literalSegCount = 0;
+  let paramSegCount = 0;
+
+  const clean = routePath.split('?')[0];
+  const segments = clean.split('/').filter(Boolean);
+
+  for (const seg of segments) {
+    depth++;
+    if (seg.includes('*')) {
+      hasStar = true;
+    } else if (seg.includes(':')) {
+      hasColon = true;
+      paramSegCount++;
+    } else {
+      literalSegCount++;
+    }
+  }
+
+  // Tier 0: Pure static (/about, /users)
+  // Tier 1: Parameterized (/users/:id)
+  // Tier 2: Catch-all (/docs/*)
+  let tier = 0;
+  if (hasStar) tier = 2;
+  else if (hasColon) tier = 1;
+
+  return {
+    tier,
+    literalSegCount,
+    paramSegCount,
+    depth,
+    routePath,
+  };
+}
+
+/**
+ * Comparator for deterministic route registration order
+ * @param {Object} a - Route descriptor
+ * @param {Object} b - Route descriptor
+ * @returns {number}
+ */
+function compareRouteOrder(a, b) {
+  const ma = a._sortMeta || getRouteSortMeta(a.path);
+  const mb = b._sortMeta || getRouteSortMeta(b.path);
+
+  if (ma.tier !== mb.tier) return ma.tier - mb.tier;
+  if (ma.literalSegCount !== mb.literalSegCount) return mb.literalSegCount - ma.literalSegCount;
+  if (ma.depth !== mb.depth) return mb.depth - ma.depth;
+  if (ma.paramSegCount !== mb.paramSegCount) return ma.paramSegCount - mb.paramSegCount;
+  return ma.routePath.localeCompare(mb.routePath);
+}
+
+/**
+ * Immutable Route Table representing all compiled application routes
+ */
+class RouteTable {
+  /**
+   * @param {Array<Object>} routes - Sorted, validated route descriptors
+   */
+  constructor(routes = []) {
+    this.routes = Object.freeze([...routes]);
+    this._map = new Map();
+
+    for (const r of this.routes) {
+      const key = `${r.method.toUpperCase()} ${r.path}`;
+      this._map.set(key, r);
+    }
+  }
+
+  /**
+   * Returns a copy of route introspection metadata
+   * @returns {Array<{ method: string, path: string, type: string, module: string|null, source: string, description?: string, tags?: string[] }>}
+   */
+  list() {
+    return this.routes.map((r) => ({
+      method: r.method.toUpperCase(),
+      path: r.path,
+      type: r.type,
+      module: r.module || null,
+      source: r.source || r.file,
+      description: r.description,
+      tags: r.tags,
+    }));
+  }
+
+  /**
+   * Retrieves a route descriptor by method and path
+   * @param {string} method
+   * @param {string} path
+   * @returns {Object|undefined}
+   */
+  get(method, path) {
+    return this._map.get(`${method.toUpperCase()} ${path}`);
+  }
+
+  /**
+   * Checks if a route exists in the table
+   * @param {string} method
+   * @param {string} path
+   * @returns {boolean}
+   */
+  has(method, path) {
+    return this._map.has(`${method.toUpperCase()} ${path}`);
+  }
+
+  /**
+   * Returns the count of routes
+   * @returns {number}
+   */
+  get size() {
+    return this.routes.length;
+  }
+}
+
+/**
+ * Compiles discovered route descriptors into an immutable RouteTable
+ * @param {Array<Object>} descriptors - Discovered route descriptors
+ * @param {Object} [options]
+ * @param {Array<Object>} [options.explicitRoutes=[]] - Manual user-configured routes
+ * @param {boolean} [options.isDev=false] - Development mode flag
+ * @returns {RouteTable}
+ */
+function compileRouteTable(descriptors = [], options = {}) {
+  const { explicitRoutes = [], isDev = process.env.NODE_ENV !== 'production' } = options;
+
+  const routeMap = new Map();
+  const explicitKeys = new Set(
+    explicitRoutes.map((r) => `${(r.method || 'GET').toUpperCase()} ${r.path}`)
+  );
+
+  const safeDescriptors = Array.isArray(descriptors) ? descriptors : [];
+  for (const descriptor of safeDescriptors) {
+    if (!descriptor || typeof descriptor !== 'object' || !descriptor.path || !descriptor.method) continue;
+
+    const method = descriptor.method.toUpperCase();
+    const key = `${method} ${descriptor.path}`;
+
+    // 1. Check if overridden by explicit route
+    if (explicitKeys.has(key)) {
+      if (isDev) {
+        console.warn(`[webspresso] Generated route ${key} was overridden by an explicit route.`);
+      }
+      continue;
+    }
+
+    // 2. Conflict detection: check for duplicate route registrations
+    if (routeMap.has(key)) {
+      const existing = routeMap.get(key);
+      const existingSrc = existing.source || existing.file;
+      const currentSrc = descriptor.source || descriptor.file;
+
+      throw new Error(
+        `Route conflict detected:\n\n` +
+          `${key}\n\n` +
+          `  ${existingSrc}\n` +
+          `  ${currentSrc}\n`
+      );
+    }
+
+    descriptor._sortMeta = getRouteSortMeta(descriptor.path);
+    routeMap.set(key, descriptor);
+  }
+
+  // Sort routes deterministically
+  const sortedRoutes = Array.from(routeMap.values()).sort(compareRouteOrder);
+
+  return new RouteTable(sortedRoutes);
+}
+
+module.exports = {
+  RouteTable,
+  compileRouteTable,
+  compareRouteOrder,
+  getRouteSortMeta,
+};

package/src/server.js

@@ -5,6 +5,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const crypto = require('crypto');
 const { AsyncLocalStorage } = require('async_hooks');
 const express = require('express');
 const helmet = require('helmet');
@@ -18,6 +19,10 @@ const { mountPages, detectLocale, loadI18n, createTranslator } = require('./file
 const { configureAssets, createHelpers, getScriptInjector } = require('./helpers');
 const { createPluginManager } = require('./plugin-manager');
 const { createServiceRegistry } = require('./services');
+const { scanRoutes } = require('./discovery/scan-routes');
+const { compileRouteTable } = require('./routing/route-table');
+const { mountDiscoveredRoutes } = require('./routing/mount-discovered-routes');
+const { discoverModuleDetails } = require('./modules/module-discovery');
 const { ShutdownManager, NodeHttpAdapter } = require('../core/shutdown');
 const { createCompressionMiddleware } = require('../core/compression');
 const {
@@ -390,7 +395,7 @@ function createApp(options = {}) {
   const isTest = NODE_ENV === 'test';
   
   const {
-    pagesDir,
+    pagesDir: initialPagesDir,
     viewsDir,
     publicDir,
     logging = isDev && !isTest,
@@ -404,6 +409,26 @@ function createApp(options = {}) {
     setupRoutes,
   } = options;
   
+  if (!initialPagesDir && !options.pages && !options.api && !options.modules && !options.routes && !options.rootDir) {
+    throw new Error('pagesDir is required');
+  }
+
+  const rootDir = options.rootDir || process.cwd();
+  let pagesDir = initialPagesDir;
+  if (!pagesDir) {
+    if (typeof options.pages === 'string') {
+      pagesDir = path.isAbsolute(options.pages) ? options.pages : path.join(rootDir, options.pages);
+    } else if (typeof options.pages === 'object' && options.pages?.dir) {
+      pagesDir = path.isAbsolute(options.pages.dir) ? options.pages.dir : path.join(rootDir, options.pages.dir);
+    } else if (fs.existsSync(path.join(rootDir, 'src/pages'))) {
+      pagesDir = path.join(rootDir, 'src/pages');
+    } else if (fs.existsSync(path.join(rootDir, 'pages'))) {
+      pagesDir = path.join(rootDir, 'pages');
+    } else {
+      pagesDir = path.join(rootDir, 'pages');
+    }
+  }
+
   // Create plugin manager
   const pluginManager = createPluginManager();
   
@@ -413,10 +438,6 @@ function createApp(options = {}) {
     ...assetsConfig
   });
   
-  if (!pagesDir) {
-    throw new Error('pagesDir is required');
-  }
-
   const clientRuntime = resolveClientRuntime(options);
 
   const shutdownConfig = {
@@ -474,6 +495,14 @@ function createApp(options = {}) {
   const app = express();
   app.serviceRegistry = serviceRegistry;
 
+  // Correlation Request ID middleware
+  app.use((req, res, next) => {
+    const reqId = (req.headers && req.headers['x-request-id']) ? req.headers['x-request-id'] : crypto.randomUUID();
+    req.id = reqId;
+    res.setHeader('X-Request-Id', reqId);
+    next();
+  });
+
   // Request context & services caller middleware
   app.use((req, res, next) => {
     if (options.db) {
@@ -685,7 +714,10 @@ function createApp(options = {}) {
   mountClientRuntime(app, clientRuntime);
 
   // Configure Nunjucks with viewsDir priority and circular extension guard
-  const templateDirs = viewsDir ? [viewsDir, pagesDir] : [pagesDir];
+  const templateDirs = [viewsDir, pagesDir].filter(Boolean);
+  if (templateDirs.length === 0) {
+    templateDirs.push(path.join(rootDir, 'pages'));
+  }
   
   const nunjucksEnv = configureSafeNunjucks(templateDirs, {
     autoescape: true,
@@ -731,7 +763,33 @@ function createApp(options = {}) {
     }
     return d.toString();
   });
-  
+
+  // Auto-discover module services and middlewares if modules directory exists
+  const modulesDirCandidate = options.modules?.dir
+    ? (path.isAbsolute(options.modules.dir) ? options.modules.dir : path.join(rootDir, options.modules.dir))
+    : (fs.existsSync(path.join(rootDir, 'src/modules')) ? path.join(rootDir, 'src/modules') : (fs.existsSync(path.join(rootDir, 'modules')) ? path.join(rootDir, 'modules') : null));
+
+  if (modulesDirCandidate && fs.existsSync(modulesDirCandidate) && options.modules !== false) {
+    try {
+      const entries = fs.readdirSync(modulesDirCandidate, { withFileTypes: true });
+      for (const entry of entries) {
+        if (entry.isDirectory() && !entry.name.startsWith('_') && !entry.name.startsWith('.')) {
+          const details = discoverModuleDetails(path.join(modulesDirCandidate, entry.name), entry.name);
+          for (const [sName, sDef] of Object.entries(details.services)) {
+            if (!serviceRegistry.has(sName)) {
+              serviceRegistry.register(sName, sDef);
+            }
+          }
+          for (const [mName, mFn] of Object.entries(details.middlewares)) {
+            middlewares[`${entry.name}.${mName}`] = mFn;
+          }
+        }
+      }
+    } catch (mErr) {
+      console.warn('[webspresso] Warning discovering modules:', mErr.message);
+    }
+  }
+
   // Register plugins (sync) — middlewares is the same object later passed to mountPages
   const pluginContext = { app, nunjucksEnv, options, middlewares, shutdownManager };
   pluginManager.registerSync(plugins, pluginContext);
@@ -748,12 +806,29 @@ function createApp(options = {}) {
     });
   }
   
-  // Mount file-based routes
-  if (!isTest) {
+  // 1. Scan and compile discovered routes (src/pages, src/api, modules/*)
+  const discoveredDescriptors = scanRoutes({
+    rootDir,
+    pages: options.pages !== undefined ? options.pages : (options.pagesDir ? { dir: options.pagesDir } : true),
+    api: options.api,
+    modules: options.modules,
+  });
+
+  const routeTable = compileRouteTable(discoveredDescriptors, {
+    explicitRoutes: options.routes || [],
+    isDev,
+  });
+
+  // Expose RouteTable on app instance
+  app.routes = routeTable;
+  app.routeTable = routeTable;
+
+  if (!isTest && routeTable.size > 0) {
     console.log('\nMounting routes:');
   }
-  const { routeMetadata, registerDynamicFileRoutes } = mountPages(app, {
-    pagesDir,
+
+  // Mount discovered routes
+  const { routeMetadata: discoveredMeta, registerDynamicDiscoveredRoutes } = mountDiscoveredRoutes(app, routeTable, {
     nunjucks: nunjucksEnv,
     middlewares,
     pluginManager,
@@ -762,10 +837,32 @@ function createApp(options = {}) {
     clientRuntime,
     pageAssets: options.pageAssets,
     serviceRegistry,
+    options,
   });
 
-  // Set route metadata in plugin manager
-  pluginManager.setRoutes(routeMetadata);
+  // 2. Mount classic file-based routes for backwards compatibility
+  let classicRouteMeta = [];
+  let registerDynamicFileRoutes = () => {};
+  if (pagesDir && fs.existsSync(path.resolve(pagesDir))) {
+    const classicMount = mountPages(app, {
+      pagesDir,
+      nunjucks: nunjucksEnv,
+      middlewares,
+      pluginManager,
+      silent: isTest,
+      db: options.db ?? null,
+      clientRuntime,
+      pageAssets: options.pageAssets,
+      serviceRegistry,
+      skipExistingRoutes: routeTable,
+    });
+    classicRouteMeta = classicMount.routeMetadata || [];
+    registerDynamicFileRoutes = classicMount.registerDynamicFileRoutes || (() => {});
+  }
+
+  // Combine route metadata for plugin manager
+  const allRouteMeta = [...discoveredMeta, ...classicRouteMeta];
+  pluginManager.setRoutes(allRouteMeta);
   
   // Call onRoutesReady hook synchronously (plugins should not be async in this phase)
   // and mount any custom routes added by plugins
@@ -808,9 +905,9 @@ function createApp(options = {}) {
     });
   }
 
-  // Dynamic / catch-all file routes after plugins and setupRoutes so paths like /_admin
-  // or custom /login are not shadowed by pages/[slug].njk (/:slug).
+  // Dynamic / catch-all file routes after plugins and setupRoutes
   registerDynamicFileRoutes();
+  registerDynamicDiscoveredRoutes();
 
   // Helper to create error page context with fsy
   function createErrorContext(req, extraData = {}) {