webspresso 0.0.91 → 0.0.94

package/core/orm/eager-loader.js

@@ -49,6 +49,35 @@ async function loadRelations(records, relationNames, model, knex, scopeContext)
   return records;
 }
 
+/**
+ * Execute whereIn query in chunks to prevent parameter limits and query planner degradation
+ * @param {import('knex').Knex} knex
+ * @param {string} table
+ * @param {string} column
+ * @param {Array<string|number>} values
+ * @param {import('./types').ScopeContext} scopeContext
+ * @param {import('./types').ModelDefinition} model
+ * @param {number} [chunkSize=500]
+ * @returns {Promise<Object[]>}
+ */
+async function queryInChunks(knex, table, column, values, scopeContext, model, chunkSize = 500) {
+  if (values.length <= chunkSize) {
+    let qb = knex(table).whereIn(column, values);
+    qb = applyScopes(qb, scopeContext, model);
+    return await qb;
+  }
+
+  const results = [];
+  for (let i = 0; i < values.length; i += chunkSize) {
+    const slice = values.slice(i, i + chunkSize);
+    let qb = knex(table).whereIn(column, slice);
+    qb = applyScopes(qb, scopeContext, model);
+    const chunkResults = await qb;
+    results.push(...chunkResults);
+  }
+  return results;
+}
+
 /**
  * Load belongsTo relation
  * Parent record has foreign key pointing to related record's primary key
@@ -78,9 +107,14 @@ async function loadBelongsTo(records, relationName, localKey, foreignKey, relate
   }
 
   const foreignKeyValues = Array.from(fkSet);
-  let qb = knex(relatedModel.table).whereIn(relatedModel.primaryKey, foreignKeyValues);
-  qb = applyScopes(qb, scopeContext, relatedModel);
-  const relatedRecords = await qb;
+  const relatedRecords = await queryInChunks(
+    knex,
+    relatedModel.table,
+    relatedModel.primaryKey,
+    foreignKeyValues,
+    scopeContext,
+    relatedModel
+  );
 
   const relatedMap = new Map();
   const rLen = relatedRecords.length;
@@ -128,9 +162,14 @@ async function loadHasMany(records, relationName, localKey, foreignKey, relatedM
   }
 
   const primaryKeyValues = Array.from(pkSet);
-  let qb = knex(relatedModel.table).whereIn(foreignKey, primaryKeyValues);
-  qb = applyScopes(qb, scopeContext, relatedModel);
-  const relatedRecords = await qb;
+  const relatedRecords = await queryInChunks(
+    knex,
+    relatedModel.table,
+    foreignKey,
+    primaryKeyValues,
+    scopeContext,
+    relatedModel
+  );
 
   const relatedGroups = new Map();
   const rLen = relatedRecords.length;
@@ -181,9 +220,14 @@ async function loadHasOne(records, relationName, localKey, foreignKey, relatedMo
   }
 
   const primaryKeyValues = Array.from(pkSet);
-  let qb = knex(relatedModel.table).whereIn(foreignKey, primaryKeyValues);
-  qb = applyScopes(qb, scopeContext, relatedModel);
-  const relatedRecords = await qb;
+  const relatedRecords = await queryInChunks(
+    knex,
+    relatedModel.table,
+    foreignKey,
+    primaryKeyValues,
+    scopeContext,
+    relatedModel
+  );
 
   const relatedMap = new Map();
   const rLen = relatedRecords.length;
@@ -222,5 +266,6 @@ module.exports = {
   loadBelongsTo,
   loadHasMany,
   loadHasOne,
+  queryInChunks,
 };
 

package/core/orm/index.js

@@ -74,10 +74,46 @@ function createDatabase(config) {
     }
   }
 
-  // Create Knex instance
+  // Create Knex instance (default useNullAsDefault: true for SQLite to prevent unneeded warnings)
+  const isSqlite = client === 'better-sqlite3' || client === 'sqlite3';
+  const knexConfig = {
+    ...(isSqlite && config.useNullAsDefault === undefined ? { useNullAsDefault: true } : {}),
+    ...config,
+  };
+
+  if (isSqlite) {
+    const userAfterCreate = knexConfig.pool?.afterCreate;
+    knexConfig.pool = {
+      ...(knexConfig.pool || {}),
+      afterCreate: (conn, cb) => {
+        try {
+          if (typeof conn.pragma === 'function') {
+            const isMemory =
+              knexConfig.connection === ':memory:' ||
+              knexConfig.connection?.filename === ':memory:' ||
+              knexConfig.connection?.database === ':memory:';
+
+            if (!isMemory) {
+              try { conn.pragma('journal_mode = WAL'); } catch (e) {}
+            }
+            try { conn.pragma('synchronous = NORMAL'); } catch (e) {}
+            try { conn.pragma('busy_timeout = 5000'); } catch (e) {}
+            try { conn.pragma('foreign_keys = ON'); } catch (e) {}
+          }
+        } catch (e) {}
+
+        if (typeof userAfterCreate === 'function') {
+          userAfterCreate(conn, cb);
+        } else {
+          cb(null, conn);
+        }
+      },
+    };
+  }
+
   let knexInstance;
   try {
-    knexInstance = knex(config);
+    knexInstance = knex(knexConfig);
   } catch (e) {
     // Provide helpful error message
     if (e.message && (e.message.includes('Cannot find module') || e.message.includes('npm install'))) {
@@ -239,7 +275,10 @@ function createDatabase(config) {
    * @returns {Promise<*>} Result of callback
    */
   async function transaction(callback, scopeContext) {
-    return knexInstance.transaction(async (trx) => {
+    const ambientTrx = getAmbientTransaction();
+    const runner = (ambientTrx && typeof ambientTrx.transaction === 'function') ? ambientTrx : knexInstance;
+
+    return runner.transaction(async (trx) => {
       const trxContext = {
         trx,
         getRepository(modelName, sContext) {

package/core/orm/model.js

@@ -46,13 +46,22 @@ function defineModel(options) {
     throw new Error('Model schema is required and must be a Zod schema');
   }
 
+  let normalizedSchema = schema;
+  if (schema && schema.shape) {
+    for (const [key, val] of Object.entries(schema.shape)) {
+      if (val && typeof val._finalize === 'function') {
+        schema.shape[key] = val._finalize();
+      }
+    }
+  }
+
   // Check for duplicate registration
   if (modelRegistry.has(name)) {
     throw new Error(`Model "${name}" is already defined`);
   }
 
   // Extract column metadata from schema
-  const columns = extractColumnsFromSchema(schema);
+  const columns = extractColumnsFromSchema(normalizedSchema);
 
   // Validate relations
   const normalizedRelations = {};
@@ -80,7 +89,7 @@ function defineModel(options) {
   const model = {
     name,
     table,
-    schema,
+    schema: normalizedSchema,
     primaryKey,
     relations: normalizedRelations,
     scopes: {

package/core/orm/transaction.js

@@ -87,7 +87,9 @@ function createTransactionContext(trx, scopeContext) {
  * @returns {Promise<*>} Result of callback
  */
 async function runTransaction(knex, callback, scopeContext) {
-  return knex.transaction(async (trx) => {
+  const ambientTrx = getAmbientTransaction();
+  const runner = (ambientTrx && typeof ambientTrx.transaction === 'function') ? ambientTrx : knex;
+  return runner.transaction(async (trx) => {
     const ctx = createTransactionContext(trx, scopeContext);
     return runWithAmbientTransaction(trx, () => callback(ctx));
   });

package/core/queue/adapters/database.js

@@ -46,6 +46,7 @@ class DatabaseQueueAdapter {
         table.dateTime('run_at').notNullable().index();
         table.dateTime('created_at').notNullable();
         table.dateTime('updated_at').notNullable();
+        table.index(['status', 'run_at', 'priority']);
       });
     }
 
@@ -88,8 +89,20 @@ class DatabaseQueueAdapter {
    * @returns {Promise<Job|null>}
    */
   async dequeue(jobNames = []) {
+    const jobs = await this.dequeueMany(jobNames, 1);
+    return jobs.length > 0 ? jobs[0] : null;
+  }
+
+  /**
+   * Dequeue multiple pending jobs up to limit in a single transaction
+   * @param {string[]} [jobNames]
+   * @param {number} [limit=1]
+   * @returns {Promise<Job[]>}
+   */
+  async dequeueMany(jobNames = [], limit = 1) {
     await this.ensureTable();
     const now = new Date();
+    const take = Math.max(1, limit || 1);
 
     return this.knex.transaction(async (trx) => {
       let query = trx(this.tableName)
@@ -97,14 +110,14 @@ class DatabaseQueueAdapter {
         .where('run_at', '<=', now)
         .orderBy('priority', 'desc')
         .orderBy('run_at', 'asc')
-        .limit(1);
+        .limit(take);
 
       if (jobNames && jobNames.length > 0) {
         query = query.whereIn('name', jobNames);
       }
 
       // If Postgres or MySQL support forUpdate / skipLocked
-      const client = this.knex.client.config.client;
+      const client = this.knex.client?.config?.client;
       if (client === 'pg' || client === 'postgres' || client === 'mysql' || client === 'mysql2') {
         try {
           query = query.forUpdate().skipLocked();
@@ -112,26 +125,46 @@ class DatabaseQueueAdapter {
       }
 
       const rows = await query;
-      if (!rows || rows.length === 0) return null;
+      if (!rows || rows.length === 0) return [];
 
-      const row = rows[0];
-      const nextAttempts = Number(row.attempts) + 1;
+      const ids = rows.map((r) => r.id);
       const updatedAt = new Date();
 
+      if (take === 1) {
+        const row = rows[0];
+        const nextAttempts = Number(row.attempts) + 1;
+        await trx(this.tableName)
+          .where('id', row.id)
+          .update({
+            status: 'running',
+            attempts: nextAttempts,
+            updated_at: updatedAt,
+          });
+
+        return [this._rowToJob({
+          ...row,
+          status: 'running',
+          attempts: nextAttempts,
+          updated_at: updatedAt,
+        })];
+      }
+
       await trx(this.tableName)
-        .where('id', row.id)
+        .whereIn('id', ids)
         .update({
           status: 'running',
-          attempts: nextAttempts,
+          attempts: trx.raw('attempts + 1'),
           updated_at: updatedAt,
         });
 
-      return this._rowToJob({
-        ...row,
-        status: 'running',
-        attempts: nextAttempts,
-        updated_at: updatedAt,
-      });
+      return rows.map((row) =>
+        this._rowToJob({
+          ...row,
+          status: 'running',
+          attempts: Number(row.attempts) + 1,
+          updated_at: updatedAt,
+        })
+      );
     });
   }
 

package/core/queue/adapters/memory.js

@@ -60,6 +60,27 @@ class MemoryQueueAdapter {
     return null;
   }
 
+  /**
+   * Dequeue multiple pending jobs up to limit
+   * @param {string[]} [jobNames]
+   * @param {number} [limit=1]
+   * @returns {Promise<Job[]>}
+   */
+  async dequeueMany(jobNames = [], limit = 1) {
+    if (limit <= 1) {
+      const single = await this.dequeue(jobNames);
+      return single ? [single] : [];
+    }
+
+    const jobs = [];
+    for (let i = 0; i < limit; i++) {
+      const job = await this.dequeue(jobNames);
+      if (!job) break;
+      jobs.push(job);
+    }
+    return jobs;
+  }
+
   /**
    * Mark job as completed
    * @param {string|number} jobId

package/core/queue/adapters/redis.js

@@ -157,6 +157,27 @@ class RedisQueueAdapter {
     return null;
   }
 
+  /**
+   * Dequeue multiple pending jobs up to limit
+   * @param {string[]} [jobNames]
+   * @param {number} [limit=1]
+   * @returns {Promise<Job[]>}
+   */
+  async dequeueMany(jobNames = [], limit = 1) {
+    if (limit <= 1) {
+      const single = await this.dequeue(jobNames);
+      return single ? [single] : [];
+    }
+
+    const jobs = [];
+    for (let i = 0; i < limit; i++) {
+      const job = await this.dequeue(jobNames);
+      if (!job) break;
+      jobs.push(job);
+    }
+    return jobs;
+  }
+
   /**
    * Mark job as completed
    * @param {string|number} jobId

package/core/queue/manager.js

@@ -26,6 +26,8 @@ class QueueManager extends EventEmitter {
     this.adapter = options.adapter || new MemoryQueueAdapter();
     this.concurrency = Math.max(1, options.concurrency || 2);
     this.pollInterval = Math.max(50, options.pollInterval || 500);
+    this.maxPollInterval = Math.max(this.pollInterval, options.maxPollInterval || (this.pollInterval * 5));
+    this._currentPollInterval = this.pollInterval;
     this.jobsDir = options.jobsDir || null;
 
     /** @type {Map<string, { handler: Function, options: Object }>} */
@@ -125,7 +127,8 @@ class QueueManager extends EventEmitter {
     const enqueued = await this.adapter.enqueue(job);
     this.emit('job:queued', enqueued);
 
-    // Trigger immediate poll if capacity is available
+    // Reset polling backoff and trigger immediate poll if capacity is available
+    this._currentPollInterval = this.pollInterval;
     if (this.isRunning && !this.isPaused && this.runningCount < this.concurrency) {
       setImmediate(() => this._tick());
     }
@@ -140,6 +143,7 @@ class QueueManager extends EventEmitter {
     if (this.isRunning) return this;
     this.isRunning = true;
     this.isPaused = false;
+    this._currentPollInterval = this.pollInterval;
     this._scheduleNextTick(0);
     this.emit('start');
     return this;
@@ -164,6 +168,7 @@ class QueueManager extends EventEmitter {
   resume() {
     if (!this.isRunning) return this.start();
     this.isPaused = false;
+    this._currentPollInterval = this.pollInterval;
     this._scheduleNextTick(0);
     this.emit('resume');
     return this;
@@ -268,7 +273,7 @@ class QueueManager extends EventEmitter {
    * Schedule next tick
    * @private
    */
-  _scheduleNextTick(delay = this.pollInterval) {
+  _scheduleNextTick(delay = this._currentPollInterval) {
     if (!this.isRunning || this.isPaused || this.isDraining) return;
     if (this._timer) clearTimeout(this._timer);
     this._timer = setTimeout(() => this._tick(), delay);
@@ -281,28 +286,45 @@ class QueueManager extends EventEmitter {
   async _tick() {
     if (!this.isRunning || this.isPaused || this.isDraining) return;
 
-    while (this.runningCount < this.concurrency) {
-      if (this.isPaused || this.isDraining) break;
+    const availableSlots = this.concurrency - this.runningCount;
+    if (availableSlots <= 0) return;
 
-      const registeredNames = Array.from(this.handlers.keys());
-      if (registeredNames.length === 0) {
-        break;
-      }
+    const registeredNames = Array.from(this.handlers.keys());
+    if (registeredNames.length === 0) {
+      this._scheduleNextTick(this._currentPollInterval);
+      return;
+    }
 
-      let job = null;
-      try {
-        job = await this.adapter.dequeue(registeredNames);
-      } catch (err) {
-        this.emit('error', err);
-        break;
+    let jobs = [];
+    try {
+      if (typeof this.adapter.dequeueMany === 'function' && availableSlots > 1) {
+        jobs = await this.adapter.dequeueMany(registeredNames, availableSlots);
+      } else {
+        const single = await this.adapter.dequeue(registeredNames);
+        if (single) jobs = [single];
       }
+    } catch (err) {
+      this.emit('error', err);
+      this._scheduleNextTick(this._currentPollInterval);
+      return;
+    }
 
-      if (!job) {
-        // No jobs ready
-        break;
-      }
+    if (!jobs || jobs.length === 0) {
+      // No jobs ready -> back off polling interval
+      this._currentPollInterval = Math.min(
+        this.maxPollInterval,
+        Math.round(this._currentPollInterval * 1.5)
+      );
+      this._scheduleNextTick(this._currentPollInterval);
+      return;
+    }
+
+    // Found job(s) -> reset polling interval
+    this._currentPollInterval = this.pollInterval;
+
+    for (const job of jobs) {
+      if (this.isPaused || this.isDraining) break;
 
-      // Found a job -> process it
       this.runningCount++;
       this.activeJobs.set(job.id, job);
 
@@ -320,7 +342,11 @@ class QueueManager extends EventEmitter {
       });
     }
 
-    this._scheduleNextTick(this.pollInterval);
+    if (this.runningCount < this.concurrency) {
+      this._scheduleNextTick(0);
+    } else {
+      this._scheduleNextTick(this.pollInterval);
+    }
   }
 
   /**

package/core/ssr/stream.js

@@ -17,7 +17,9 @@ const { Readable } = require('stream');
  * @returns {Readable}
  */
 function createHtmlStream(options = {}) {
-  const { env, templatePath, context = {}, defer = {} } = options;
+  const { env, templatePath, context = {}, defer = {}, nonce: optionNonce } = options;
+  const nonce = optionNonce || context.nonce || context.cspNonce || '';
+  const scriptTag = nonce ? `<script nonce="${nonce}">` : '<script>';
 
   if (!env) {
     throw new Error('createHtmlStream requires a valid Nunjucks environment in options.env');
@@ -63,6 +65,9 @@ function createHtmlStream(options = {}) {
 
         try {
           const resolvedData = await promise;
+          if (stream.destroyed) {
+            return;
+          }
           let slotHtml = '';
           if (slotTemplate) {
             slotHtml = env.render(slotTemplate, { ...context, [key]: resolvedData });
@@ -73,7 +78,7 @@ function createHtmlStream(options = {}) {
           // Push an out-of-order streaming chunk with inline replacement script or comment
           const slotChunk = `
 <template data-stream-slot="${key}">${slotHtml}</template>
-<script>
+${scriptTag}
 (function(){
   var t = document.querySelector('template[data-stream-slot="${key}"]');
   var target = document.querySelector('[data-stream-target="${key}"]') || document.getElementById('stream-target-${key}');
@@ -83,18 +88,27 @@ function createHtmlStream(options = {}) {
   }
 })();
 </script>`;
-          stream.push(slotChunk);
+          if (!stream.destroyed) {
+            stream.push(slotChunk);
+          }
         } catch (slotErr) {
+          if (stream.destroyed) {
+            return;
+          }
           const errorChunk = `
 <template data-stream-slot="${key}">
   <!-- Slot Error (${key}): ${slotErr.message} -->
 </template>`;
-          stream.push(errorChunk);
+          if (!stream.destroyed) {
+            stream.push(errorChunk);
+          }
         }
       });
 
       await Promise.all(resolutionPromises);
-      stream.push(null); // End of stream
+      if (!stream.destroyed) {
+        stream.push(null); // End of stream
+      }
     } catch (err) {
       stream.destroy(err);
     }
@@ -122,8 +136,17 @@ function renderStream(res, templatePath, context = {}, options = {}) {
       defer = {},
       status = 200,
       flushImmediately = true,
+      nonce: optionNonce,
     } = options;
 
+    const nonce =
+      optionNonce ||
+      res?.locals?.cspNonce ||
+      res?.locals?.nonce ||
+      context?.cspNonce ||
+      context?.nonce ||
+      '';
+
     if (!env) {
       return reject(new Error('renderStream requires a valid Nunjucks environment in options.env'));
     }
@@ -145,25 +168,46 @@ function renderStream(res, templatePath, context = {}, options = {}) {
       templatePath,
       context,
       defer,
+      nonce,
     });
 
+    let finished = false;
+    const cleanup = () => {
+      if (finished) return;
+      finished = true;
+      if (!stream.destroyed) {
+        stream.destroy();
+      }
+      resolve();
+    };
+
+    res.on('close', cleanup);
+
     stream.on('data', (chunk) => {
-      res.write(chunk);
+      if (!finished && !res.writableEnded) {
+        res.write(chunk);
+      }
     });
 
     stream.on('end', () => {
-      res.end();
-      resolve();
+      if (!finished) {
+        finished = true;
+        res.end();
+        resolve();
+      }
     });
 
     stream.on('error', (err) => {
-      if (!res.headersSent) {
-        res.status(500).send('Streaming Render Error');
-      } else {
-        res.write(`<!-- Streaming Error: ${err.message} -->`);
-        res.end();
+      if (!finished) {
+        finished = true;
+        if (!res.headersSent) {
+          res.status(500).send('Streaming Render Error');
+        } else if (!res.writableEnded) {
+          res.write(`<!-- Streaming Error: ${err.message} -->`);
+          res.end();
+        }
+        reject(err);
       }
-      reject(err);
     });
   });
 }

package/core/validation/index.js

@@ -0,0 +1,70 @@
+/**
+ * Webspresso Unified Validation Core
+ * Centralized Zod extensions, prototype pollution defense, and schema utilities
+ * @module core/validation
+ */
+
+const z = require('zod');
+const { extendZ, generateNanoid, zodNanoid } = require('../orm/utils/nanoid');
+
+/**
+ * Standard Zod instance extended with z.nanoid() for framework-wide validation.
+ */
+const zExtended = extendZ(z);
+
+/**
+ * Check if an object contains dangerous prototype pollution keys
+ * @param {*} obj
+ * @returns {boolean}
+ */
+function hasDangerousKeys(obj) {
+  if (obj === null || typeof obj !== 'object') {
+    return false;
+  }
+  if (Array.isArray(obj)) {
+    return obj.some(item => hasDangerousKeys(item));
+  }
+  for (const key of Object.keys(obj)) {
+    if (key === '__proto__' || key === 'constructor' || key === 'prototype') {
+      return true;
+    }
+    if (typeof obj[key] === 'object' && obj[key] !== null && hasDangerousKeys(obj[key])) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Sanitize input payload against prototype pollution
+ * @param {*} input
+ * @returns {*} Sanitized input
+ */
+function sanitizeInput(input) {
+  if (input === null || typeof input !== 'object') {
+    return input;
+  }
+  if (!hasDangerousKeys(input)) {
+    return input;
+  }
+  if (Array.isArray(input)) {
+    return input.map(item => sanitizeInput(item));
+  }
+  const clean = {};
+  for (const key of Object.keys(input)) {
+    if (key === '__proto__' || key === 'constructor' || key === 'prototype') {
+      continue;
+    }
+    clean[key] = sanitizeInput(input[key]);
+  }
+  return clean;
+}
+
+module.exports = {
+  z: zExtended,
+  extendZ,
+  generateNanoid,
+  zodNanoid,
+  hasDangerousKeys,
+  sanitizeInput,
+};