sqlcache 2.9.0 → 2.9.1

package/dbConnector.js

@@ -1,6 +1,8 @@
 const crypto = require('crypto');
+const { performance } = require('perf_hooks');
 const db = require('mysql2/promise');
-const { getArrayItem, addArrayItem, delPrefixKeyItem } = require('./redis.Connector');
+const redisConnector = require('./redis.Connector');
+const { getArrayItem, addArrayItem, delPrefixKeyItem } = redisConnector;
 const { generateCacheKey, isAutoKeyEnabled } = require('./core/autoKey');
 const { determineInvalidationPatterns, mergeInvalidationPatterns } = require('./core/autoInvalidate');
 require('dotenv').config();
@@ -9,6 +11,14 @@ const env = process.env;
 // Check if Redis is enabled (default: true)
 const REDIS_ENABLED = env.REDIS_ENABLED !== 'false';
 
+/**
+ * Parses a positive integer env value.
+ */
+function readPositive(value, fallback) {
+    const parsed = parseInt(value, 10);
+    return parsed > 0 ? parsed : fallback;
+}
+
 /**
  * Validates required environment variables and provides helpful error messages
  * @throws {Error} If any required configuration is missing
@@ -137,8 +147,9 @@ async function selectDatabase(connection, database) {
 }
 
 // SQL that may switch the default database by itself (a USE statement, possibly inside a
-// multi-statement query). Matching is deliberately broad: a false positive only costs one USE.
-const MAY_SWITCH_DATABASE = /\buse\b/i;
+// multi-statement query, or DROP DATABASE, which unsets it when it drops the session's database).
+// Matching is deliberately broad: a false positive only costs one USE.
+const MAY_SWITCH_DATABASE = /\b(?:use|drop\s+(?:database|schema))\b/i;
 
 /**
  * Called before running caller-supplied SQL: if it could change the connection's database,
@@ -164,23 +175,60 @@ async function runQuery(database, sql, parameters, attempt = null) {
         // running the query now would only add load (typically to a saturated pool)
         if (attempt && attempt.timedOut) return undefined;
         await selectDatabase(connection, database);
+        // ... or while the USE round trip was on its way
+        if (attempt && attempt.timedOut) return undefined;
         trackCallerSql(connection, sql);
         if (attempt) attempt.connection = connection;
         const [data] = await connection.query(sql, parameters);
         return data;
     } finally {
         if (attempt) attempt.connection = null;
-        connection.release();
+        if (attempt && attempt.kill) {
+            // A KILL QUERY for this connection's thread was issued and may still be on its way:
+            // the connection is dropped instead of being handed to another caller, whose
+            // statement the KILL would interrupt (the pool opens a new connection instead)
+            if (typeof connection.destroy === 'function') {
+                connection.destroy();
+            } else {
+                connection.release();
+            }
+        } else {
+            connection.release();
+        }
     }
 }
 
+// KILL QUERY connections in progress, by thread id
+const MAX_CONCURRENT_KILLS = 4;
+const pendingKills = new Map();
+
 /**
- * Stops the statement running on a pooled connection (KILL QUERY). Uses its own short-lived
- * connection, because the pool may be exhausted exactly when queries time out.
+ * Stops the statement running on a pooled connection (KILL QUERY), at most once per thread and
+ * for at most MAX_CONCURRENT_KILLS threads at a time, so a burst of timeouts cannot open a
+ * burst of connections. Returns the kill's promise (it never rejects), or undefined when no
+ * kill was issued.
  */
-async function killRunningQuery(connection) {
+function killRunningQuery(connection) {
     const threadId = connection && connection.threadId;
-    if (!threadId) return;
+    if (!threadId) return undefined;
+    if (pendingKills.has(threadId)) return pendingKills.get(threadId);
+    if (pendingKills.size >= MAX_CONCURRENT_KILLS) {
+        console.error('Could not cancel timed-out query: too many cancellations in progress');
+        return undefined;
+    }
+    const done = () => {
+        pendingKills.delete(threadId);
+    };
+    const kill = sendKillQuery(threadId).then(done, done);
+    pendingKills.set(threadId, kill);
+    return kill;
+}
+
+/**
+ * Sends KILL QUERY for a thread. Uses its own short-lived connection, because the pool may be
+ * exhausted exactly when queries time out.
+ */
+async function sendKillQuery(threadId) {
     let killer;
     try {
         killer = await db.createConnection({
@@ -206,6 +254,19 @@ function invalidate(patterns) {
     return Promise.all(mergeInvalidationPatterns(patterns).map(pattern => delPrefixKeyItem(pattern)));
 }
 
+/**
+ * Invalidation after a write that failed but may have been applied. A problem here is only
+ * logged, so the caller gets the error of the write itself.
+ */
+async function invalidateAfterFailure(patterns) {
+    if (patterns.length === 0) return;
+    try {
+        await invalidate(patterns);
+    } catch (err) {
+        console.error('Cache invalidation error:', err);
+    }
+}
+
 /**
  * Auto-generated cache key. The same SQL returns different rows in different databases
  * (e.g. one database per tenant), so a call with `database` gets its own key; the suffix keeps
@@ -216,6 +277,116 @@ function autoCacheKey(sql, parameters, database) {
     return database ? `${key}@${database}` : key;
 }
 
+// Functions that already logged being called without a cacheName (logged once per function)
+const loggedWithoutCacheName = new Set();
+
+/**
+ * Cache key for getCacheQueryPagination and getCacheQueryWithTimeout: the cacheName, else an
+ * automatic key when auto keys are on, else null and the call runs without the cache (a key
+ * shared by every call without a cacheName would serve one query's rows for another).
+ */
+function cacheKeyFor(functionName, cacheName, sql, parameters, database) {
+    if (cacheName !== null && cacheName !== undefined && cacheName !== '') return cacheName;
+    if (!REDIS_ENABLED) return null; // nothing is cached anyway
+    if (isAutoKeyEnabled()) return autoCacheKey(sql, parameters, database);
+    if (!loggedWithoutCacheName.has(functionName)) {
+        loggedWithoutCacheName.add(functionName);
+        console.warn(`sqlcache: ${functionName}() was called without a cacheName, so its result is not cached. Pass a cacheName or enable automatic keys (CORE_AUTO_FEATURES=true).`);
+    }
+    return null;
+}
+
+/**
+ * Whether a read-through result is served from the cache and stored in it: only rows. Anything
+ * else (an empty array, a non-array value) is a miss, and is not stored, as it would never be served.
+ */
+function hasRows(value) {
+    return Array.isArray(value) && value.length > 0;
+}
+
+/**
+ * Whether a cached pagination result has the shape getCacheQueryPagination stores
+ */
+function isPageResult(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value) && Array.isArray(value.detail);
+}
+
+// ── Stale-fill guard ────────────────────────────────────────────
+// redis.Connector skips a read-through cache fill when a delete recorded since its SELECT
+// started covers its key (see "Stale-fill guard" there). Its fill() writes the cache itself, so
+// it is only used together with that connector's own addArrayItem: a replaced addArrayItem (a
+// test double, for instance) keeps receiving the fills, unguarded, as before.
+const fillGuard = redisConnector._fillGuard && redisConnector._fillGuard.addArrayItem === addArrayItem
+    ? redisConnector._fillGuard
+    : null;
+
+/**
+ * Delete sequence number, captured right before a read-through SELECT starts
+ */
+function fillSequence() {
+    return fillGuard ? fillGuard.sequence() : 0;
+}
+
+/**
+ * Stores the rows of a read-through SELECT that started at `sequence`
+ */
+function fillCache(key, data, sequence) {
+    return fillGuard ? fillGuard.fill(key, data, sequence) : addArrayItem(key, data);
+}
+
+// ── bulkInsert identifiers ──────────────────────────────────────
+// bulkInsert puts the table and column names into the statement as given, so only names of the
+// form ^\s*P(?:\.P){0,2}\s*$ are accepted, where P is a bare name (0-9, A-Z, a-z, $, _ and
+// U+0080 to U+FFFF) or a backtick-quoted one, as in IDENTIFIER_PART below. That regex is cubic
+// on runs of whitespace that are also name characters (U+00A0, U+3000, ...), so it is evaluated
+// in linear time instead: the name without surrounding whitespace is tested against the
+// unambiguous IDENTIFIER_BODY. Only a leading or trailing '.' needs the whitespace character next
+// to it, which can only be part of the name as a one-character part when it is a name character.
+const IDENTIFIER_PART = '(?:[0-9A-Za-z$_\\u0080-\\uFFFF]+|`(?:[^`]|``)+`)';
+const IDENTIFIER_BODY = new RegExp(`^${IDENTIFIER_PART}(?:\\.${IDENTIFIER_PART}){0,2}$`);
+const NAME_CHARACTER_FROM = 0x80; // whitespace characters from here on are name characters
+
+/**
+ * Same result as /^\s*P(?:\.P){0,2}\s*$/.test(name), see above
+ */
+function isValidIdentifier(name) {
+    if (typeof name !== 'string') return false;
+    const start = name.length - name.trimStart().length;
+    const end = name.trimEnd().length;
+    if (start >= end) {
+        // only whitespace: valid as a single name part made of whitespace name characters
+        for (let i = 0; i < name.length; i++) {
+            if (name.charCodeAt(i) >= NAME_CHARACTER_FROM) return true;
+        }
+        return false;
+    }
+    let body = name.slice(start, end);
+    if (body[0] === '.') {
+        if (!(start > 0 && name.charCodeAt(start - 1) >= NAME_CHARACTER_FROM)) return false;
+        body = `x${body}`;
+    }
+    if (body[body.length - 1] === '.') {
+        if (!(end < name.length && name.charCodeAt(end) >= NAME_CHARACTER_FROM)) return false;
+        body = `${body}x`;
+    }
+    return IDENTIFIER_BODY.test(body);
+}
+
+// Errors that are never retried: the statement may have taken effect
+const NO_RETRY = Symbol('no retry');
+
+/**
+ * Marks the error of a failed COMMIT, whose outcome is unknown (e.g. only its reply was lost):
+ * it is not retried, which could apply the transaction twice, and err.commitOutcomeUnknown is set.
+ */
+function markCommitOutcomeUnknown(err) {
+    if (err !== null && typeof err === 'object' && Object.isExtensible(err)) {
+        err.commitOutcomeUnknown = true;
+        Object.defineProperty(err, NO_RETRY, { value: true });
+    }
+    return err;
+}
+
 /**
  * Identifies one paginated query (database, SQL and parameters) for DB_PAGINATION_REUSE_TOTAL.
  */
@@ -237,15 +408,26 @@ const COALESCING = env.DB_COALESCE_QUERIES !== 'false';
 // Opt-in: reuse a page's COUNT(*) total for the other pages of the same query (until the cache
 // name is invalidated), instead of counting again on every page miss
 const REUSE_PAGINATION_TOTAL = env.DB_PAGINATION_REUSE_TOTAL === 'true';
+// A caller joins a shared load for at most DB_COALESCE_MAX_WAIT ms after it started (a load can
+// hang, e.g. on a connection that was silently dropped); after that it shares a fresh one
+const COALESCE_MAX_WAIT = Math.min(readPositive(env.DB_COALESCE_MAX_WAIT, 10000), 2147483647); // setTimeout limit
+// Shared loads in progress, by coalescing key: { promise, startedAt, overdue, whenOverdue, timer }
 const inflightLoads = new Map();
 
-// Only single, plain SELECT statements are shared; anything that might write runs per caller
-const PLAIN_SELECT = /^\s*\(?\s*SELECT\b/i;
+// Only single, plain SELECT statements are shared; anything that might write runs per caller.
+// Tested on the statement without its leading whitespace: a leading \s* next to the \s* after
+// the optional '(' would make long whitespace runs quadratic.
+const PLAIN_SELECT = /^\(?\s*SELECT\b/i;
+const WHITESPACE = /\s/;
 
 function isPlainRead(sql) {
     if (typeof sql !== 'string') return false;
-    const body = sql.replace(/[\s;]+$/, '');
-    return PLAIN_SELECT.test(body) && body.indexOf(';') === -1;
+    // Without trailing whitespace and semicolons (a loop: /[\s;]+$/ is quadratic on long runs
+    // of whitespace inside the statement)
+    let end = sql.length;
+    while (end > 0 && (sql.charCodeAt(end - 1) === 59 /* ';' */ || WHITESPACE.test(sql[end - 1]))) end--;
+    const body = sql.slice(0, end);
+    return PLAIN_SELECT.test(body.trimStart()) && body.indexOf(';') === -1;
 }
 
 /**
@@ -286,36 +468,114 @@ function cloneResult(value) {
     if (Buffer.isBuffer(value)) return Buffer.from(value);
     const copy = Object.create(Object.getPrototypeOf(value));
     for (const key of Object.keys(value)) {
-        copy[key] = cloneResult(value[key]);
+        if (key === '__proto__') {
+            // An own "__proto__" key (e.g. from a JSON column) is data: assigning it would set
+            // the copy's prototype instead
+            Object.defineProperty(copy, key, { value: cloneResult(value[key]), writable: true, enumerable: true, configurable: true });
+        } else {
+            copy[key] = cloneResult(value[key]);
+        }
     }
     return copy;
 }
 
+// MAX_EXECUTION_TIME / max_execution_time exceeded: the same statement would only time out again
+function isQueryTimeout(err) {
+    return Boolean(err) && (err.errno === 3024 || err.code === 'ER_QUERY_TIMEOUT');
+}
+
+/**
+ * Starts `load` as the shared load for `key`, replacing any entry there
+ */
+function startSharedLoad(key, load) {
+    const entry = { promise: null, startedAt: performance.now(), overdue: false, whenOverdue: null, timer: null };
+    entry.promise = load();
+    inflightLoads.set(key, entry);
+    const settled = () => {
+        clearTimeout(entry.timer);
+        if (inflightLoads.get(key) === entry) inflightLoads.delete(key);
+    };
+    entry.promise.then(settled, settled);
+    return entry;
+}
+
+function isOverdue(entry) {
+    return entry.overdue || performance.now() - entry.startedAt >= COALESCE_MAX_WAIT;
+}
+
+/**
+ * Resolves once the entry has run for COALESCE_MAX_WAIT. One timer per entry, started for its
+ * first waiting caller, and cleared when the load settles.
+ */
+function whenOverdue(entry) {
+    if (!entry.whenOverdue) {
+        entry.whenOverdue = new Promise((resolve) => {
+            entry.timer = setTimeout(() => {
+                entry.overdue = true;
+                resolve();
+            }, Math.max(0, COALESCE_MAX_WAIT - (performance.now() - entry.startedAt)));
+            if (entry.timer.unref) entry.timer.unref();
+        });
+    }
+    return entry.whenOverdue;
+}
+
+/**
+ * Settles with the first of two shared loads to fulfil (a copy, unless the value is from the
+ * caller's own load), or rejects with the fresh load's error once both have rejected.
+ */
+function firstFulfilled(overdue, fresh, freshIsOwn) {
+    return new Promise((resolve, reject) => {
+        let done = false;
+        let failures = 0;
+        let freshError;
+        const fulfil = own => (value) => {
+            if (done) return;
+            done = true;
+            resolve(own ? value : cloneResult(value));
+        };
+        const fail = () => {
+            if (++failures === 2) reject(freshError);
+        };
+        overdue.then(fulfil(false), fail);
+        fresh.then(fulfil(freshIsOwn), (err) => {
+            freshError = err;
+            fail();
+        });
+    });
+}
+
 /**
  * Runs `load` once per key at a time. Callers that arrive while it runs wait for it and get
- * their own copy; if it fails they run `load` themselves, exactly as they would have before.
+ * their own copy; if it fails they run `load` themselves, exactly as they would have before
+ * (except for ER_QUERY_TIMEOUT, which they get as well). A caller waits for a load at most
+ * until it has run for COALESCE_MAX_WAIT: then it shares a fresh load (started by the first
+ * caller that finds the load overdue, and joined by later callers instead of the overdue one)
+ * and gets whichever of the two fulfils first.
  */
 async function loadOnce(key, load) {
     if (key === null) return load();
 
     const inflight = inflightLoads.get(key);
-    if (inflight) {
-        let shared;
-        try {
-            shared = await inflight;
-        } catch (_) {
+    if (!inflight) return startSharedLoad(key, load).promise;
+
+    if (!isOverdue(inflight)) {
+        const outcome = await Promise.race([
+            // Copied in the load's first reaction, before the first caller can change the rows
+            inflight.promise.then(value => ({ value: cloneResult(value) }), err => ({ err })),
+            whenOverdue(inflight)
+        ]);
+        if (outcome) {
+            if (!('err' in outcome)) return outcome.value;
+            if (isQueryTimeout(outcome.err)) throw outcome.err;
             return load();
         }
-        return cloneResult(shared);
     }
 
-    const pending = load();
-    inflightLoads.set(key, pending);
-    try {
-        return await pending;
-    } finally {
-        if (inflightLoads.get(key) === pending) inflightLoads.delete(key);
-    }
+    let fresh = inflightLoads.get(key);
+    const own = !fresh || fresh === inflight || isOverdue(fresh);
+    if (own) fresh = startSharedLoad(key, load);
+    return firstFulfilled(inflight.promise, fresh.promise, own);
 }
 
 // Errors MySQL raises because of the COUNT(*) wrapper itself (the same SQL just ran as the page
@@ -327,7 +587,11 @@ const WRAPPER_ERRORS = new Set([
     'ER_WRONG_USAGE', 'ER_NOT_SUPPORTED_YET'
 ]);
 const UNCOUNTABLE_SQL_LIMIT = 500;
-const uncountableSql = new Set();
+const uncountableSql = new Set(); // SHA-1 of the SQL text: bounded memory, whatever the SQL size
+
+function sqlHash(sql) {
+    return crypto.createHash('sha1').update(sql).digest('hex');
+}
 
 // A MAX_EXECUTION_TIME optimizer hint only applies to a top-level SELECT, so it is copied onto
 // the COUNT query to keep the caller's time limit
@@ -346,7 +610,7 @@ function countSql(cleanSql) {
  * fetching the full result and counting it, exactly as earlier versions always did.
  */
 async function countRows(connection, cleanSql, parameters) {
-    if (!uncountableSql.has(cleanSql)) {
+    if (uncountableSql.size === 0 || !uncountableSql.has(sqlHash(cleanSql))) {
         try {
             const [rows] = await connection.query(countSql(cleanSql), parameters);
             return Number(rows[0].total);
@@ -355,7 +619,7 @@ async function countRows(connection, cleanSql, parameters) {
             if (uncountableSql.size >= UNCOUNTABLE_SQL_LIMIT) {
                 uncountableSql.delete(uncountableSql.values().next().value);
             }
-            uncountableSql.add(cleanSql);
+            uncountableSql.add(sqlHash(cleanSql));
         }
     }
 
@@ -403,6 +667,8 @@ async function executeWithRetry(fn, retries = 3, delay = 1000, database = null,
             return await fn(database);
         } catch (error) {
             if (i === retries - 1) throw error;
+            // Never retried: the statement may have taken effect (see markCommitOutcomeUnknown)
+            if (error && error[NO_RETRY]) throw error;
 
             // Retry only on connection errors
             if (error.code === 'ECONNREFUSED' ||
@@ -485,16 +751,17 @@ module.exports = {
         return executeWithRetry(async (db) => {
             if (REDIS_ENABLED) {
                 const cachedData = await getArrayItem(finalCacheName);
-                if (cachedData.length > 0) {
+                if (hasRows(cachedData)) {
                     return cachedData;
                 }
             }
 
             return loadOnce(coalescingKey(finalCacheName, db, sql, parameters), async () => {
+                const sequence = fillSequence(); // before the SELECT can read anything
                 const data = await runQuery(db, sql, parameters);
 
-                if (REDIS_ENABLED) {
-                    await addArrayItem(finalCacheName, data);
+                if (REDIS_ENABLED && hasRows(data)) {
+                    await fillCache(finalCacheName, data, sequence);
                 }
 
                 return data;
@@ -510,6 +777,7 @@ module.exports = {
      * Uses server-side pagination (SQL LIMIT) for better performance.
      * The total count is computed with SELECT COUNT(*) over the query instead of fetching every row.
      * Handles semicolons at the end of SQL queries properly.
+     * Without a cacheName the result is not cached, unless auto keys are on (CORE_AUTO_FEATURES).
      *
      * @param {string} sql - The SQL query to execute.
      * @param {Array} parameters - The parameters to be used in the SQL query.
@@ -521,6 +789,9 @@ module.exports = {
      * @throws {Error} - If an error occurs during the execution of the function.
      */
     async getCacheQueryPagination(sql, parameters, cacheName, page, pageSize = 30, database = null) {
+        const cacheKey = cacheKeyFor('getCacheQueryPagination', cacheName, sql, parameters, database);
+        const useCache = REDIS_ENABLED && cacheKey !== null;
+
         return executeWithRetry(async (db) => {
             // Validate and normalize pageSize parameter
             let validPageSize = parseInt(pageSize);
@@ -534,27 +805,35 @@ module.exports = {
                 validPage = 0;
             }
 
-            // Create unique cache key including page and pageSize
-            const uniqueCacheKey = `${cacheName}:page${validPage}:size${validPageSize}`;
+            // Create unique cache key including page and pageSize (also the coalescing key of
+            // a call without cache)
+            const uniqueCacheKey = `${cacheKey === null ? '' : cacheKey}:page${validPage}:size${validPageSize}`;
 
-            if (REDIS_ENABLED) {
+            if (useCache) {
                 const cachedData = await getArrayItem(uniqueCacheKey);
-                if (typeof cachedData === 'object' && !Array.isArray(cachedData) && cachedData !== null) {
+                if (isPageResult(cachedData)) {
                     return cachedData;
                 }
             }
 
-            // Prepare SQL: remove trailing semicolon(s) and whitespace
-            const cleanSql = sql.trim().replace(/;+\s*$/, '');
+            // Prepare SQL: remove trailing semicolon(s) and whitespace. After trim() that is the
+            // trailing run of semicolons (a loop: /;+\s*$/ is quadratic on long runs of
+            // semicolons inside the SQL)
+            const trimmedSql = sql.trim();
+            let cleanEnd = trimmedSql.length;
+            while (cleanEnd > 0 && trimmedSql.charCodeAt(cleanEnd - 1) === 59 /* ';' */) cleanEnd--;
+            const cleanSql = trimmedSql.slice(0, cleanEnd);
 
             // Calculate offset for pagination
             const offset = validPage * validPageSize;
 
             return loadOnce(coalescingKey(uniqueCacheKey, db, cleanSql, parameters), async () => {
+                const sequence = fillSequence(); // before anything is read, the stored total included
+
                 // DB_PAGINATION_REUSE_TOTAL: total stored under the cache name, so invalidating
                 // the cache name clears it together with the pages
-                const totalKey = REDIS_ENABLED && REUSE_PAGINATION_TOTAL
-                    ? `${cacheName}:total:${paginationTotalHash(db, cleanSql, parameters)}`
+                const totalKey = useCache && REUSE_PAGINATION_TOTAL
+                    ? `${cacheKey}:total:${paginationTotalHash(db, cleanSql, parameters)}`
                     : null;
                 let storedTotal = null;
                 if (totalKey) {
@@ -571,8 +850,9 @@ module.exports = {
                     await selectDatabase(connection, db);
                     trackCallerSql(connection, cleanSql);
 
-                    // Get paginated data with parameterized LIMIT to prevent SQL injection
-                    const paginatedSql = `${cleanSql} LIMIT ?, ?`;
+                    // Get paginated data with parameterized LIMIT to prevent SQL injection. On its
+                    // own line, so that a trailing -- comment in the SQL cannot swallow it
+                    const paginatedSql = `${cleanSql}\nLIMIT ?, ?`;
                     [data] = await connection.query(paginatedSql, [...parameters, offset, validPageSize]);
 
                     // A short page is the last one, so it already gives the exact total and the
@@ -595,10 +875,10 @@ module.exports = {
                     detail: data
                 };
 
-                if (REDIS_ENABLED) {
-                    await addArrayItem(uniqueCacheKey, result);
+                if (useCache) {
+                    await fillCache(uniqueCacheKey, result, sequence);
                     if (totalKey && totalCount !== storedTotal) {
-                        await addArrayItem(totalKey, [totalCount]);
+                        await fillCache(totalKey, [totalCount], sequence);
                     }
                 }
 
@@ -614,6 +894,12 @@ module.exports = {
      * was not acknowledged instead of inserting the already committed chunks again.
      * With `transaction: true` all chunks are committed together (all or nothing), which is also
      * faster for large inserts; a retry then starts over, as the failed attempt was rolled back.
+     * A failed COMMIT is not retried (it may have been applied): its error is thrown with
+     * `commitOutcomeUnknown: true`.
+     * The table and column names are put into the statement as given; a name that is not a
+     * plain (schema.)table or column name is rejected with a TypeError before anything is sent.
+     * The cache is invalidated with resetCacheName, else by auto-invalidation (if enabled), also
+     * when the call fails after rows may have been written.
      *
      * @param {string} table - The table name to insert into.
      * @param {Array} records - Array of objects with column-value pairs.
@@ -629,13 +915,28 @@ module.exports = {
         }
 
         const columns = Object.keys(records[0]);
+        // Anything but a plain name could only be a syntax error or SQL injection (e.g. through
+        // record keys taken from a request body)
+        if (!isValidIdentifier(table)) {
+            throw new TypeError('bulkInsert: invalid table name ' + JSON.stringify(String(table)));
+        }
+        for (const column of columns) {
+            if (!isValidIdentifier(column)) {
+                throw new TypeError('bulkInsert: invalid column name ' + JSON.stringify(String(column)));
+            }
+        }
         const rowPlaceholder = `(${columns.map(() => '?').join(',')})`;
         const insertPrefix = `INSERT INTO ${table} (${columns.join(',')}) VALUES `;
+        // The patterns QuaryCache would use for this INSERT: resetCacheName, else auto-invalidation
+        const patterns = REDIS_ENABLED ? determineInvalidationPatterns(insertPrefix, resetCacheName) : [];
 
         // Progress survives retries: only chunks whose INSERT was acknowledged are skipped
         let nextIndex = 0;
         let totalInserted = 0;
         let chunks = 0;
+        // Rows may be in MySQL even if the call fails: an INSERT was sent (without a transaction),
+        // or the COMMIT failed with an unknown outcome
+        let mayHaveWritten = false;
 
         await executeWithRetry(async (db) => {
             const connection = await con.getConnection();
@@ -661,12 +962,20 @@ module.exports = {
 
                     const sql = insertPrefix + chunk.map(() => rowPlaceholder).join(',');
 
+                    if (!transaction) mayHaveWritten = true;
                     const [result] = await connection.query(sql, flatValues);
                     totalInserted += result.affectedRows;
                     chunks++;
                 }
 
-                if (transaction) await connection.commit();
+                if (transaction) {
+                    try {
+                        await connection.commit();
+                    } catch (err) {
+                        mayHaveWritten = true;
+                        throw markCommitOutcomeUnknown(err);
+                    }
+                }
             } catch (err) {
                 if (transaction) {
                     try {
@@ -679,11 +988,17 @@ module.exports = {
             } finally {
                 connection.release();
             }
-        }, 3, 1000, database);
+        }, 3, 1000, database).catch(async (err) => {
+            if (mayHaveWritten) {
+                forgetInflightReads();
+                await invalidateAfterFailure(patterns);
+            }
+            throw err;
+        });
         forgetInflightReads();
 
-        if (resetCacheName && REDIS_ENABLED) {
-            await delPrefixKeyItem(resetCacheName);
+        if (patterns.length > 0) {
+            await invalidate(patterns);
         }
 
         return { insertedRows: totalInserted, chunks };
@@ -692,6 +1007,7 @@ module.exports = {
     /**
      * Executes a query with timeout protection.
      * Prevents long-running queries from blocking the application.
+     * Without a cacheName the result is not cached, unless auto keys are on (CORE_AUTO_FEATURES).
      *
      * @param {string} sql - The SQL query to execute.
      * @param {Array} parameters - The parameters to be passed to the SQL query.
@@ -702,29 +1018,32 @@ module.exports = {
      */
     async getCacheQueryWithTimeout(sql, parameters, cacheName, options = {}) {
         const { timeout = 30000, database = null, cancelOnTimeout = false } = options;
+        const cacheKey = cacheKeyFor('getCacheQueryWithTimeout', cacheName, sql, parameters, database);
+        const useCache = REDIS_ENABLED && cacheKey !== null;
         let attempt = null;
         const onTimeout = () => {
             attempt.timedOut = true;
             if (cancelOnTimeout && attempt.connection) {
-                killRunningQuery(attempt.connection);
+                attempt.kill = killRunningQuery(attempt.connection);
             }
         };
         return executeWithRetry(async (db) => {
-            attempt = { timedOut: false, connection: null };
+            attempt = { timedOut: false, connection: null, kill: undefined };
             const current = attempt;
 
-            if (REDIS_ENABLED) {
-                const cachedData = await getArrayItem(cacheName);
-                if (cachedData.length > 0) {
+            if (useCache) {
+                const cachedData = await getArrayItem(cacheKey);
+                if (hasRows(cachedData)) {
                     return cachedData;
                 }
             }
 
+            const sequence = fillSequence(); // before the SELECT can read anything
             const data = await runQuery(db, sql, parameters, current);
 
             // A query skipped after the timeout has nothing to cache; a cancelled one either
-            if (REDIS_ENABLED && data !== undefined && !(current.timedOut && cancelOnTimeout)) {
-                await addArrayItem(cacheName, data);
+            if (useCache && hasRows(data) && !(current.timedOut && cancelOnTimeout)) {
+                await fillCache(cacheKey, data, sequence);
             }
 
             return data;
@@ -767,7 +1086,9 @@ module.exports = {
     /**
      * Executes queries within a database transaction.
      * Automatically commits on success or rolls back on error.
-     * Cache invalidation is buffered and applied only on successful commit.
+     * Cache invalidation is buffered and applied only on successful commit (or when the COMMIT
+     * fails, as it may have been applied). Cache fills of tx.getCacheQuery are buffered too and
+     * written only after a successful commit.
      *
      * @param {Function} callback - Async function that receives transaction context (tx)
      * @param {Object} options - Transaction options
@@ -787,6 +1108,14 @@ module.exports = {
         let connection;
         let result;
         const invalidationBuffer = []; // Buffer cache patterns for commit
+        // Cache fills of tx.getCacheQuery misses (the last one per key), written after COMMIT:
+        // until then the rows may include this transaction's own uncommitted writes
+        const fillBuffer = new Map();
+        // The fills are checked against deletes since the transaction started (its snapshot can
+        // be older than the read that fills)
+        const sequence = fillSequence();
+        let commitFailed = false;
+        let commitError;
 
         try {
             if (isShuttingDown) {
@@ -827,7 +1156,8 @@ module.exports = {
 
                 /**
                  * Execute a cached read query within the transaction
-                 * Note: Reads from cache, but doesn't guarantee transaction isolation
+                 * Note: Reads from cache, but doesn't guarantee transaction isolation. A miss is
+                 * cached only after the transaction committed.
                  * @param {string} sql - SQL query
                  * @param {Array} parameters - Query parameters
                  * @param {string|null} cacheName - Cache key (optional if auto-key enabled)
@@ -849,16 +1179,17 @@ module.exports = {
                     // Check cache first
                     if (REDIS_ENABLED) {
                         const cachedData = await getArrayItem(finalCacheName);
-                        if (cachedData.length > 0) {
+                        if (hasRows(cachedData)) {
                             return cachedData;
                         }
                     }
 
-                    // Execute in transaction and cache result
+                    // Execute in transaction; its result is cached after COMMIT (a copy, as the
+                    // caller may change the rows it gets meanwhile)
                     trackCallerSql(connection, sql);
                     const [data] = await connection.query(sql, parameters);
-                    if (REDIS_ENABLED) {
-                        await addArrayItem(finalCacheName, data);
+                    if (REDIS_ENABLED && hasRows(data)) {
+                        fillBuffer.set(finalCacheName, cloneResult(data));
                     }
 
                     return data;
@@ -880,7 +1211,13 @@ module.exports = {
             result = await callback(tx);
 
             // Commit transaction
-            await connection.commit();
+            try {
+                await connection.commit();
+            } catch (error) {
+                commitFailed = true;
+                commitError = error;
+                throw error;
+            }
             forgetInflightReads();
         } catch (error) {
             // Rollback on error
@@ -891,13 +1228,23 @@ module.exports = {
                     console.error('Rollback error:', rollbackError);
                 }
             }
-            throw error;
+            if (!commitFailed) throw error;
         } finally {
             if (connection) {
                 connection.release();
             }
         }
 
+        if (commitFailed) {
+            // The COMMIT may have been applied (e.g. only its reply was lost): invalidate as after
+            // a successful one. The buffered fills are dropped.
+            forgetInflightReads();
+            if (REDIS_ENABLED) {
+                await invalidateAfterFailure([...new Set(invalidationBuffer)]);
+            }
+            throw commitError;
+        }
+
         // Apply buffered cache invalidations on successful commit, after the connection is
         // back in the pool (a SCAN-based invalidation can take a while on large keyspaces)
         if (REDIS_ENABLED && invalidationBuffer.length > 0) {
@@ -906,6 +1253,12 @@ module.exports = {
             await invalidate(uniquePatterns);
         }
 
+        // Then the buffered fills, through the stale-fill guard: a fill is skipped when its key was
+        // invalidated (by this transaction or anything else) since the transaction started
+        if (fillBuffer.size > 0) {
+            await Promise.all([...fillBuffer].map(([key, data]) => fillCache(key, data, sequence)));
+        }
+
         return result;
     }
 };