sqlcache 2.9.0 → 2.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +31 -17
- package/{readme.md → README.md} +106 -28
- package/core/autoInvalidate.js +14 -4
- package/core/autoKey.js +44 -8
- package/dbConnector.js +418 -65
- package/docs/API.md +161 -61
- package/docs/CONFIGURATION.md +155 -47
- package/docs/CONTRIBUTING.md +39 -29
- package/docs/EXAMPLES.md +70 -29
- package/docs/MIGRATION.md +141 -42
- package/docs/README.tr.md +75 -27
- package/docs/USE-CASE-BULK.md +48 -17
- package/docs/USE-CASE-ECOMMERCE.md +8 -2
- package/docs/USE-CASE-REALTIME.md +18 -6
- package/docs/USE-CASE-SAAS.md +24 -15
- package/index.d.ts +163 -50
- package/index.js +50 -23
- package/package.json +11 -5
- package/redis.Connector.js +98 -5
package/.env.example
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Template for sqlcache settings. Copy it to .env in your app's working directory
|
|
2
|
+
# (in an installed project: cp node_modules/sqlcache/.env.example .env).
|
|
3
|
+
# Only .env is loaded, and ${VAR} references are not expanded.
|
|
4
|
+
|
|
1
5
|
# ============================================
|
|
2
6
|
# MySQL Database Configuration (REQUIRED)
|
|
3
7
|
# ============================================
|
|
@@ -21,10 +25,10 @@ DB_NAME=my_database
|
|
|
21
25
|
# Connection Pool Settings (OPTIONAL)
|
|
22
26
|
# ============================================
|
|
23
27
|
|
|
24
|
-
# Maximum number of connections in pool (default:
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
DB_CONNECTION_LIMIT=
|
|
28
|
+
# Maximum number of connections in the pool, per process (default: 151)
|
|
29
|
+
# Keep instances x workers x DB_CONNECTION_LIMIT, plus other clients, below MySQL's
|
|
30
|
+
# max_connections (MySQL's default is also 151)
|
|
31
|
+
DB_CONNECTION_LIMIT=151
|
|
28
32
|
|
|
29
33
|
# Maximum number of queued connection requests (default: 0 = unlimited)
|
|
30
34
|
# 0 = unlimited queue (recommended - waits when pool is full)
|
|
@@ -45,14 +49,19 @@ DB_IDLE_TIMEOUT=60000
|
|
|
45
49
|
# Time to wait for initial database connection before throwing error
|
|
46
50
|
DB_CONNECT_TIMEOUT=10000
|
|
47
51
|
|
|
48
|
-
# Allow multiple SQL statements in a single query (default: true for backward
|
|
49
|
-
#
|
|
50
|
-
|
|
52
|
+
# Allow multiple SQL statements in a single query (default when unset: true, for backward
|
|
53
|
+
# compatibility). false is recommended for new projects: with true, an SQL injection can run
|
|
54
|
+
# additional statements. Set true only if you send several statements in one query.
|
|
55
|
+
DB_MULTIPLE_STATEMENTS=false
|
|
51
56
|
|
|
52
57
|
# Concurrent cache misses for the same key, database, SQL and parameters share one
|
|
53
58
|
# database query; each caller still gets its own copy of the rows (default: true)
|
|
54
59
|
# DB_COALESCE_QUERIES=true
|
|
55
60
|
|
|
61
|
+
# Max time (ms) a caller waits for such a shared query before running its own, which protects
|
|
62
|
+
# against a stuck connection; it still takes the shared result if that arrives first (default: 10000)
|
|
63
|
+
# DB_COALESCE_MAX_WAIT=10000
|
|
64
|
+
|
|
56
65
|
# getCacheQueryPagination: reuse the COUNT(*) total of one page for the other pages of the same
|
|
57
66
|
# query until its cache name is invalidated, instead of counting on every page miss. Totals can
|
|
58
67
|
# then lag behind writes that do not invalidate the cache name (default: false)
|
|
@@ -73,12 +82,14 @@ REDIS_SERVER=localhost
|
|
|
73
82
|
REDIS_PORT=6379
|
|
74
83
|
|
|
75
84
|
# Redis password (leave empty if no password)
|
|
85
|
+
# Cached rows are served as stored, so treat Redis as a trusted store: use AUTH and a private network
|
|
76
86
|
REDIS_PASSWORD=
|
|
77
87
|
|
|
78
88
|
# Redis virtual host / namespace prefix
|
|
79
|
-
# All cache keys will be prefixed with this value
|
|
80
|
-
# Useful
|
|
81
|
-
#
|
|
89
|
+
# All cache keys will be prefixed with this value and a colon
|
|
90
|
+
# Useful to separate applications or environments that share one Redis
|
|
91
|
+
# Set it once per process; do not change it at runtime (for example per tenant request)
|
|
92
|
+
# Example: REDIS_VHOST=app1 will create keys like "app1:users-cache"
|
|
82
93
|
REDIS_VHOST=
|
|
83
94
|
|
|
84
95
|
# ============================================
|
|
@@ -117,12 +128,13 @@ TIMEZONE=+00:00
|
|
|
117
128
|
# v2.6.0 Smart Features (OPTIONAL) 🚀
|
|
118
129
|
# ============================================
|
|
119
130
|
|
|
120
|
-
#
|
|
121
|
-
# This enables
|
|
131
|
+
# Automatic cache keys: generate a cache key when cacheName is omitted
|
|
132
|
+
# This enables auto keys only, NOT auto invalidation (set CORE_AUTO_INVALIDATION for that)
|
|
122
133
|
CORE_AUTO_FEATURES=false
|
|
123
134
|
|
|
124
|
-
#
|
|
125
|
-
#
|
|
135
|
+
# Automatic invalidation: after INSERT/UPDATE/DELETE/REPLACE (and, since 2.9.1, bulkInsert)
|
|
136
|
+
# without a resetCacheName, delete the table's <table>_* and <table>:* keys
|
|
137
|
+
CORE_AUTO_INVALIDATION=false
|
|
126
138
|
|
|
127
139
|
# ============================================
|
|
128
140
|
# Example Configurations for Different Environments
|
|
@@ -138,13 +150,15 @@ CORE_AUTO_FEATURES=false
|
|
|
138
150
|
# DB_HOST=mysql-prod.example.com
|
|
139
151
|
# DB_USERNAME=prod_user
|
|
140
152
|
# DB_PASSWORD=strong_password_here
|
|
141
|
-
# DB_CONNECTION_LIMIT=
|
|
153
|
+
# DB_CONNECTION_LIMIT=50 # Per process: instances x workers x limit < max_connections
|
|
142
154
|
# DB_QUEUE_LIMIT=0 # Unlimited queue (recommended)
|
|
143
155
|
# DB_CONNECT_TIMEOUT=10000 # 10 seconds
|
|
156
|
+
# DB_MULTIPLE_STATEMENTS=false
|
|
144
157
|
# REDIS_SERVER=redis-prod.example.com
|
|
145
158
|
# REDIS_PASSWORD=redis_password_here
|
|
146
|
-
# REDIS_VHOST=prod
|
|
147
|
-
# CORE_AUTO_FEATURES=true #
|
|
159
|
+
# REDIS_VHOST=prod
|
|
160
|
+
# CORE_AUTO_FEATURES=true # Automatic cache keys
|
|
161
|
+
# CORE_AUTO_INVALIDATION=true # Automatic invalidation
|
|
148
162
|
|
|
149
163
|
# Docker Environment:
|
|
150
164
|
# DB_HOST=mysql
|
package/{readme.md → README.md}
RENAMED
|
@@ -38,7 +38,9 @@ MySQL fail.
|
|
|
38
38
|
|
|
39
39
|
## Requirements
|
|
40
40
|
|
|
41
|
-
- Node.js
|
|
41
|
+
- Node.js 14 or newer (tested on Node 18–26)
|
|
42
|
+
- mysql2 3.23.1 or newer, installed as a dependency (older versions have security advisories,
|
|
43
|
+
including critical ones)
|
|
42
44
|
- MySQL 5.7 or 8.x (tested with 8.0)
|
|
43
45
|
- Redis 2.8 or newer for `SCAN` (tested with 7); optional, see `REDIS_ENABLED`
|
|
44
46
|
|
|
@@ -52,7 +54,8 @@ npm install sqlcache
|
|
|
52
54
|
|
|
53
55
|
The package is configured with environment variables. It loads a `.env` file from the working
|
|
54
56
|
directory and validates the settings when it is first required, with an error that lists what is
|
|
55
|
-
missing.
|
|
57
|
+
missing. Only `.env` is read (not `.env.production` and the like), and `${VAR}` references in it are
|
|
58
|
+
not expanded. A commented template of every setting is in `node_modules/sqlcache/.env.example`.
|
|
56
59
|
|
|
57
60
|
```bash
|
|
58
61
|
# .env
|
|
@@ -74,6 +77,7 @@ await QuaryCache('UPDATE users SET name = ? WHERE id = ?', ['Ada', 42], 'user:')
|
|
|
74
77
|
```
|
|
75
78
|
|
|
76
79
|
> `QuaryCache` is the function's published name (with this spelling) and is kept for compatibility.
|
|
80
|
+
> The examples use `await`; in CommonJS, run them inside an `async` function.
|
|
77
81
|
|
|
78
82
|
## Usage
|
|
79
83
|
|
|
@@ -86,10 +90,15 @@ const rows = await getCacheQuery(sql, params, cacheName, database);
|
|
|
86
90
|
- **Cache hit:** the rows come from Redis.
|
|
87
91
|
- **Cache miss:** the query runs on MySQL, and the rows are cached for 40,000 s (about 11 h) before
|
|
88
92
|
the promise resolves. Keep cached data fresh through invalidation; there is no per-call TTL.
|
|
89
|
-
- **Empty results** are not served from the cache
|
|
90
|
-
the next call.
|
|
93
|
+
- **Empty results** are not served from the cache (and since 2.9.1 not written to it either), so a
|
|
94
|
+
query that returned no rows runs again on the next call.
|
|
91
95
|
- **Concurrent misses** for the same key, database, SQL and parameters share one query, and each
|
|
92
|
-
caller receives its own copy of the rows. Set `DB_COALESCE_QUERIES=false` to turn this off.
|
|
96
|
+
caller receives its own copy of the rows. Set `DB_COALESCE_QUERIES=false` to turn this off. A caller
|
|
97
|
+
that has waited `DB_COALESCE_MAX_WAIT` ms (default 10000) for the shared query starts its own.
|
|
98
|
+
- **Cache hits are JSON round trips:** a `Date` comes back as an ISO string and a `Buffer` as
|
|
99
|
+
`{ type, data }`, while a miss returns mysql2's types. Normalize such columns in your code.
|
|
100
|
+
- **Stale reads:** since 2.9.1 a slow read does not re-cache data that was invalidated while it ran in
|
|
101
|
+
the same process. Across processes a short window remains.
|
|
93
102
|
|
|
94
103
|
### Writing and invalidation
|
|
95
104
|
|
|
@@ -98,9 +107,9 @@ const result = await QuaryCache(sql, params, resetCacheName, database);
|
|
|
98
107
|
// result: rows for SELECT, or { insertId, affectedRows, ... } for writes
|
|
99
108
|
```
|
|
100
109
|
|
|
101
|
-
After the statement succeeds, every key that starts with
|
|
102
|
-
`resetCacheName` can be a string or an array of prefixes. The call
|
|
103
|
-
has finished.
|
|
110
|
+
`QuaryCache` never reads the cache. After the statement succeeds, every key that starts with
|
|
111
|
+
`resetCacheName` is deleted. `resetCacheName` can be a string or an array of prefixes. The call
|
|
112
|
+
resolves once the invalidation has finished.
|
|
104
113
|
|
|
105
114
|
### Pagination
|
|
106
115
|
|
|
@@ -116,8 +125,8 @@ const { totalCount, pageCount, detail } = await getCacheQueryPagination(
|
|
|
116
125
|
|
|
117
126
|
- Each page is cached as `<cacheName>:page<n>:size<m>`, so `QuaryCache(..., 'orders:customer:7')`
|
|
118
127
|
clears every page.
|
|
119
|
-
- The total is counted with `SELECT COUNT(*)` only when the page is full
|
|
120
|
-
one and already gives the total.
|
|
128
|
+
- The total is counted with `SELECT COUNT(*)` only when the page is full or is an empty page past the
|
|
129
|
+
end; a short page is the last one and already gives the total.
|
|
121
130
|
- With `DB_PAGINATION_REUSE_TOTAL=true`, the total is counted once and reused for the other pages
|
|
122
131
|
until the cache name is invalidated.
|
|
123
132
|
|
|
@@ -134,8 +143,11 @@ const orderId = await withTransaction(async (tx) => {
|
|
|
134
143
|
```
|
|
135
144
|
|
|
136
145
|
The transaction commits when the callback resolves and rolls back when it throws. Cache invalidations
|
|
137
|
-
from `tx.query` are buffered and applied only after a successful commit. `tx.getCacheQuery`
|
|
138
|
-
|
|
146
|
+
from `tx.query` are buffered and applied only after a successful commit. `tx.getCacheQuery` checks
|
|
147
|
+
the cache first and otherwise reads on the transaction's connection; since 2.9.1 it writes what it
|
|
148
|
+
read to the cache only after the commit, so nothing is cached from a rolled-back transaction. A cache
|
|
149
|
+
hit does not reflect the transaction's own changes, so use `tx.query` for reads that decide a write
|
|
150
|
+
(for example `SELECT ... FOR UPDATE`). `tx.getConnection()` returns the raw mysql2 connection.
|
|
139
151
|
|
|
140
152
|
### Bulk insert
|
|
141
153
|
|
|
@@ -148,7 +160,16 @@ const { insertedRows, chunks } = await bulkInsert('events', records, {
|
|
|
148
160
|
```
|
|
149
161
|
|
|
150
162
|
Without `transaction`, each chunk commits on its own. If a connection error interrupts the insert,
|
|
151
|
-
the retry continues from the first chunk that was not acknowledged.
|
|
163
|
+
the retry continues from the first chunk that was not acknowledged. With `transaction: true`, an error
|
|
164
|
+
from `COMMIT` is not retried and carries `err.commitOutcomeUnknown = true`: the rows may or may not
|
|
165
|
+
have been written.
|
|
166
|
+
|
|
167
|
+
- **Columns come from the first record**, and the table and column names go into the SQL as-is.
|
|
168
|
+
Since 2.9.1 an invalid name is rejected with a `TypeError` before anything is sent. Never pass
|
|
169
|
+
untrusted keys: map request or CSV input onto a fixed list of columns.
|
|
170
|
+
- **Invalidation:** `resetCacheName` prefixes are deleted after the insert. Since 2.9.1, without
|
|
171
|
+
`resetCacheName` auto-invalidation (if enabled) clears the table's keys as a `QuaryCache` write would,
|
|
172
|
+
and invalidation also runs when the insert fails after rows may have been written.
|
|
152
173
|
|
|
153
174
|
### Query timeouts
|
|
154
175
|
|
|
@@ -161,7 +182,9 @@ const rows = await getCacheQueryWithTimeout(sql, params, 'report:daily', {
|
|
|
161
182
|
|
|
162
183
|
The call rejects with `Query timeout exceeded`. If it timed out before its query started (for example
|
|
163
184
|
while waiting for a pool connection), the query is not run. Without `cancelOnTimeout`, a query that
|
|
164
|
-
was already running finishes on the server and its result is cached.
|
|
185
|
+
was already running finishes on the server and its result is cached. With it, since 2.9.1 the
|
|
186
|
+
connection whose query was killed is discarded instead of being returned to the pool, and at most 4
|
|
187
|
+
cancellations run at once.
|
|
165
188
|
|
|
166
189
|
### Several databases
|
|
167
190
|
|
|
@@ -172,7 +195,13 @@ await getCacheQuery('SELECT * FROM users', [], 'tenant7:users', 'tenant_7');
|
|
|
172
195
|
```
|
|
173
196
|
|
|
174
197
|
Pass the database per call. Do not change `process.env` per request: environment variables are
|
|
175
|
-
shared by all concurrent requests
|
|
198
|
+
shared by all concurrent requests, so a per-request `REDIS_VHOST` mixes tenants' cache entries
|
|
199
|
+
(2.9.1 logs a warning when it changes at runtime). For multi-tenant apps:
|
|
200
|
+
|
|
201
|
+
- Take the tenant from the authenticated session and check it against an allowlist, never from a raw
|
|
202
|
+
request header.
|
|
203
|
+
- Pass the tenant's database per call; automatic keys then end with `@<database>`. Put the tenant in
|
|
204
|
+
explicit cache names too, such as `tenant7:users`.
|
|
176
205
|
|
|
177
206
|
### Redis helpers
|
|
178
207
|
|
|
@@ -190,22 +219,35 @@ None of these throw when Redis is unavailable: reads return `[]`, writes return
|
|
|
190
219
|
caching it, and deletes are retried when Redis is back. `getRedisClient()` returns the underlying
|
|
191
220
|
node-redis v3 client.
|
|
192
221
|
|
|
222
|
+
Treat Redis as a trusted store: cached rows are served without checks, so protect it with AUTH and a
|
|
223
|
+
private network, and give the app its own `REDIS_VHOST` or Redis database.
|
|
224
|
+
|
|
193
225
|
### Health checks and shutdown
|
|
194
226
|
|
|
195
227
|
```javascript
|
|
228
|
+
const { isRedisConnected, getPoolStats, closeConnections, getRedisClient } = require('sqlcache');
|
|
229
|
+
|
|
196
230
|
app.get('/health', (req, res) => res.json({ redis: isRedisConnected(), mysql: getPoolStats() }));
|
|
197
231
|
// getPoolStats(): { totalConnections, activeConnections, freeConnections, queuedRequests }
|
|
198
232
|
|
|
199
233
|
process.on('SIGTERM', async () => {
|
|
200
234
|
await closeConnections(); // closes the MySQL pool; new queries are rejected afterwards
|
|
235
|
+
if (process.env.REDIS_ENABLED !== 'false') getRedisClient().quit(); // Redis is not closed by closeConnections()
|
|
201
236
|
});
|
|
202
237
|
```
|
|
203
238
|
|
|
239
|
+
Without the `quit()`, the open Redis connection keeps the process alive. With `REDIS_ENABLED=false`,
|
|
240
|
+
`isRedisConnected()` returns `false` without opening a connection (since 2.9.1).
|
|
241
|
+
|
|
204
242
|
## Cache keys and invalidation
|
|
205
243
|
|
|
206
244
|
All keys are prefixed with `REDIS_VHOST:` when `REDIS_VHOST` is set. Invalidation always works by
|
|
207
245
|
prefix, so give related keys a common prefix, such as `user:42` and `user:list`.
|
|
208
246
|
|
|
247
|
+
Prefixes passed to `resetCacheName` or `delPrefixKeyItem` are Redis glob patterns: escape `*`, `?`,
|
|
248
|
+
`[`, `]` and `\` in any user-supplied part. An empty prefix without `REDIS_VHOST` deletes every key in
|
|
249
|
+
that Redis database.
|
|
250
|
+
|
|
209
251
|
### Automatic keys and invalidation (opt-in)
|
|
210
252
|
|
|
211
253
|
```bash
|
|
@@ -220,18 +262,33 @@ const { configure } = require('sqlcache');
|
|
|
220
262
|
|
|
221
263
|
configure({
|
|
222
264
|
autoKey: { enabled: true },
|
|
223
|
-
autoInvalidation: { enabled: true, tables: { users: ['users:*', 'profiles:*'] } } // optional custom rules
|
|
265
|
+
autoInvalidation: { enabled: true, tables: { users: ['users:*', 'users_*', 'profiles:*'] } } // optional custom rules
|
|
224
266
|
});
|
|
225
267
|
|
|
226
268
|
const users = await getCacheQuery('SELECT * FROM users WHERE company_id = ?', [5]); // key users:company_id:<hash>
|
|
227
|
-
await QuaryCache('UPDATE users SET active = 0 WHERE id = ?', [9]); // clears users
|
|
269
|
+
await QuaryCache('UPDATE users SET active = 0 WHERE id = ?', [9]); // clears users:*, users_* and profiles:*
|
|
228
270
|
```
|
|
229
271
|
|
|
272
|
+
`configure({ autoKey: false })`, `enableAutoKey(false)` and `enableAutoInvalidation(false)` turn a
|
|
273
|
+
feature off (since 2.9.1; earlier versions turned it on).
|
|
274
|
+
|
|
230
275
|
- **How keys are built:** from the table name, the `WHERE` column names and a hash of the parameters,
|
|
231
|
-
for example `users:company_id:a7b3c2d1`, or `users:all` without a `WHERE`.
|
|
276
|
+
for example `users:company_id:a7b3c2d1`, or `users:all` without a `WHERE`. With four or more
|
|
277
|
+
parameters the column names are left out (`users:<hash>`). Queries without a detectable table
|
|
278
|
+
(`CALL`, `SELECT fn(?)`) get `query:<sqlhash>:<paramhash>` (since 2.9.1). Calls that pass a
|
|
232
279
|
`database` get an `@<database>` suffix.
|
|
233
|
-
- **
|
|
234
|
-
|
|
280
|
+
- **Caveats:** without a `WHERE`, the key is `<table>:all` whatever the SQL. The select list,
|
|
281
|
+
`ORDER BY`, `LIMIT`, literal values and joined tables are not part of the key, and the parameter
|
|
282
|
+
hash is 32-bit, so different queries can share a key. Pass explicit cache names for aggregates,
|
|
283
|
+
joins, `LIMIT`/`ORDER BY` variants, time-relative SQL (`NOW()`, `CURDATE()`) and anything
|
|
284
|
+
user-specific or sensitive.
|
|
285
|
+
- **Custom rules** (`tables`) replace the default `<table>_*` and `<table>:*` patterns, so include
|
|
286
|
+
`<table>:*` when you use automatic keys. A `resetCacheName` passed to a call also replaces the
|
|
287
|
+
automatic patterns for that call.
|
|
288
|
+
- **What auto-invalidation sees:** only statements that start with `INSERT`, `UPDATE`, `DELETE` or
|
|
289
|
+
`REPLACE`, and only the first table they name. Statements behind a leading comment or `WITH`,
|
|
290
|
+
`CALL`, multi-table `UPDATE`/`DELETE`, `db.table` names, tables joined into cached reads and writes
|
|
291
|
+
from other applications need an explicit `resetCacheName`.
|
|
235
292
|
|
|
236
293
|
## Configuration
|
|
237
294
|
|
|
@@ -242,26 +299,27 @@ await QuaryCache('UPDATE users SET active = 0 WHERE id = ?', [9]);
|
|
|
242
299
|
| `DB_PASSWORD` | `''` | MySQL password |
|
|
243
300
|
| `DB_NAME` | *required* | Default database |
|
|
244
301
|
| `DB_PORT` | `3306` | MySQL port |
|
|
245
|
-
| `DB_CONNECTION_LIMIT` | `151` | Max connections per process
|
|
302
|
+
| `DB_CONNECTION_LIMIT` | `151` | Max connections per process. Keep instances × workers × this limit, plus other clients, below MySQL's `max_connections` (default 151) |
|
|
246
303
|
| `DB_QUEUE_LIMIT` | `0` | Max requests queued for a connection (`0` = unlimited) |
|
|
247
304
|
| `DB_MAX_IDLE` | `10` | Idle connections kept open |
|
|
248
305
|
| `DB_IDLE_TIMEOUT` | `60000` | ms before idle connections above `DB_MAX_IDLE` close |
|
|
249
306
|
| `DB_CONNECT_TIMEOUT` | `10000` | ms to establish a connection |
|
|
250
|
-
| `DB_MULTIPLE_STATEMENTS` | `true` | Allow several statements per query |
|
|
307
|
+
| `DB_MULTIPLE_STATEMENTS` | `true` | Allow several statements per query (`false` is recommended for new projects) |
|
|
251
308
|
| `TIMEZONE` | `+00:00` | Connection time zone |
|
|
252
309
|
| `DB_COALESCE_QUERIES` | `true` | Share one query between concurrent identical cache misses |
|
|
310
|
+
| `DB_COALESCE_MAX_WAIT` | `10000` | ms a caller waits for a shared query before running its own (2.9.1) |
|
|
253
311
|
| `DB_PAGINATION_REUSE_TOTAL` | `false` | Reuse a paginated query's total across its pages |
|
|
254
|
-
| `REDIS_ENABLED` | `true` | `false` runs every query on MySQL without caching; Redis is only connected if you call a Redis helper |
|
|
312
|
+
| `REDIS_ENABLED` | `true` | `false` runs every query on MySQL without caching; Redis is only connected if you call a Redis helper (`isRedisConnected()` does not connect since 2.9.1) |
|
|
255
313
|
| `REDIS_SERVER` | *required if enabled* | Redis host |
|
|
256
314
|
| `REDIS_PORT` | `6379` | Redis port |
|
|
257
315
|
| `REDIS_PASSWORD` | none | Redis password |
|
|
258
|
-
| `REDIS_VHOST` | none | Prefix for every key (`<vhost>:<key>`) |
|
|
316
|
+
| `REDIS_VHOST` | none | Prefix for every key (`<vhost>:<key>`); set once per process |
|
|
259
317
|
| `REDIS_WAIT_TIMEOUT` | `10000` | ms an operation waits for Redis to (re)connect |
|
|
260
318
|
| `REDIS_FAIL_FAST` | `true` | After `REDIS_WAIT_TIMEOUT` of downtime, stop waiting and fall back immediately |
|
|
261
319
|
| `REDIS_COMMAND_TIMEOUT` | `max(REDIS_WAIT_TIMEOUT, 10000)` | ms a cache read/write may stay unanswered (`0` = off) |
|
|
262
320
|
| `REDIS_RECONNECT_MAX_DELAY` | `5000` | Max ms between reconnect attempts |
|
|
263
321
|
| `REDIS_SCAN_COUNT` | `1000` | Keys examined per `SCAN` round trip during invalidation |
|
|
264
|
-
| `CORE_AUTO_FEATURES` | `false` | Automatic cache keys |
|
|
322
|
+
| `CORE_AUTO_FEATURES` | `false` | Automatic cache keys only (not invalidation) |
|
|
265
323
|
| `CORE_AUTO_INVALIDATION` | `false` | Automatic invalidation after writes |
|
|
266
324
|
|
|
267
325
|
[docs/CONFIGURATION.md](docs/CONFIGURATION.md) covers pool tuning, bursty traffic and troubleshooting.
|
|
@@ -273,9 +331,12 @@ await QuaryCache('UPDATE users SET active = 0 WHERE id = ?', [9]);
|
|
|
273
331
|
| Redis down | Reads go to MySQL and results are not cached. Calls wait up to `REDIS_WAIT_TIMEOUT` for a reconnect, then fall back immediately for the rest of the outage. Invalidations that could not run are replayed when Redis is back. |
|
|
274
332
|
| Redis connection hangs | Cache reads and writes time out after `REDIS_COMMAND_TIMEOUT`, and a connection that stops answering is reopened. |
|
|
275
333
|
| Redis restarts | Reconnects with exponential backoff and jitter, capped at `REDIS_RECONNECT_MAX_DELAY`, and never gives up. |
|
|
276
|
-
| MySQL connection errors | On `ECONNREFUSED`, `ETIMEDOUT`, `ENOTFOUND`, `PROTOCOL_CONNECTION_LOST` or `ER_CON_COUNT_ERROR` a call is attempted up to 3 times, waiting 1 s and then 2 s (plus jitter). The whole statement is retried,
|
|
334
|
+
| MySQL connection errors | On `ECONNREFUSED`, `ETIMEDOUT`, `ENOTFOUND`, `PROTOCOL_CONNECTION_LOST` or `ER_CON_COUNT_ERROR` a call is attempted up to 3 times, waiting 1 s and then 2 s (plus jitter). The whole statement is retried, including `QuaryCache` and `bulkInsert` writes, which may then run twice: make writes idempotent. An error from `COMMIT` in `bulkInsert({ transaction: true })` is not retried and carries `err.commitOutcomeUnknown = true`. |
|
|
277
335
|
| Error inside `withTransaction` | The transaction is rolled back and buffered invalidations are dropped. |
|
|
278
336
|
|
|
337
|
+
mysql2 errors carry `err.sql` with the parameter values filled in. Do not send whole error objects to
|
|
338
|
+
logs or clients you do not trust.
|
|
339
|
+
|
|
279
340
|
The resilience behavior is covered by unit tests, integration tests against real MySQL and Redis,
|
|
280
341
|
and 18 Docker chaos scenarios. These include killing Redis and MySQL mid-operation, 30-second
|
|
281
342
|
outages, network partitions, pool exhaustion, thundering herds and both services down at once.
|
|
@@ -303,14 +364,20 @@ Tips:
|
|
|
303
364
|
Type definitions ship with the package:
|
|
304
365
|
|
|
305
366
|
```typescript
|
|
306
|
-
import { getCacheQuery, getCacheQueryPagination } from 'sqlcache';
|
|
367
|
+
import { getCacheQuery, getCacheQueryPagination, QuaryCache } from 'sqlcache';
|
|
307
368
|
|
|
308
369
|
interface User { id: number; name: string; }
|
|
309
370
|
|
|
310
371
|
const users = await getCacheQuery<User>('SELECT id, name FROM users WHERE active = ?', [1], 'users:active');
|
|
311
372
|
const page = await getCacheQueryPagination<User>('SELECT id, name FROM users', [], 'users:list', 0, 50);
|
|
373
|
+
const rows = await QuaryCache<User[]>('SELECT id, name FROM users WHERE id = ?', [1]); // result type
|
|
312
374
|
```
|
|
313
375
|
|
|
376
|
+
Since 2.9.1, named imports also work in native ESM (`import { getCacheQuery } from 'sqlcache'`), and
|
|
377
|
+
`import sqlcache from 'sqlcache'` works without `esModuleInterop`. On 2.9.0 and earlier, use the
|
|
378
|
+
default import in native ESM, and named imports or `import sqlcache = require('sqlcache')` in
|
|
379
|
+
CommonJS builds without `esModuleInterop`.
|
|
380
|
+
|
|
314
381
|
## Upgrading
|
|
315
382
|
|
|
316
383
|
### From `node-caching-mysql-connector-with-redis`
|
|
@@ -325,6 +392,12 @@ Then replace `require('node-caching-mysql-connector-with-redis')` and TypeScript
|
|
|
325
392
|
release, which forwards to `sqlcache`, so existing projects get the same code. The old name is
|
|
326
393
|
deprecated and receives no new major versions.
|
|
327
394
|
|
|
395
|
+
### From v2.9.0
|
|
396
|
+
|
|
397
|
+
v2.9.1 is a patch release: it requires Node.js 14+ and mysql2 3.23.1+, fixes ESM and TypeScript
|
|
398
|
+
imports, and changes some caching edge cases. [docs/MIGRATION.md](docs/MIGRATION.md#v290--v291)
|
|
399
|
+
lists every difference.
|
|
400
|
+
|
|
328
401
|
### From v2.8.x
|
|
329
402
|
|
|
330
403
|
v2.9.0 does not change the API. [docs/MIGRATION.md](docs/MIGRATION.md#v28x--v290-performance--resilience)
|
|
@@ -333,13 +406,18 @@ lists every behavior difference from v2.8.x and the settings that restore the ol
|
|
|
333
406
|
## Development
|
|
334
407
|
|
|
335
408
|
```bash
|
|
409
|
+
npm ci # install dependencies (node_modules is not committed)
|
|
336
410
|
npm test # unit tests
|
|
411
|
+
npx -p typescript@5 tsc -p test/types # type definitions check
|
|
337
412
|
INTEGRATION=1 npm test # also integration tests: Redis on 127.0.0.1:6399, MySQL on 127.0.0.1:3399
|
|
338
413
|
INTEGRATION=1 node test/chaos.test.js # Redis chaos suite (Docker containers test-redis / test-mysql)
|
|
339
414
|
INTEGRATION=1 node test/chaos-mysql.test.js # MySQL chaos suite
|
|
340
415
|
```
|
|
341
416
|
|
|
342
|
-
The
|
|
417
|
+
The integration tests run Docker commands through `DOCKER_BIN` (default `docker`) on the container
|
|
418
|
+
named by `REDIS_CONTAINER` (default `test-redis`); the chaos suites also use `DOCKER_BIN`. CI runs the unit tests on Node 18, 20, 22, 24 and
|
|
419
|
+
26 and against the minimum dependency versions, plus the type check and the integration tests
|
|
420
|
+
against MySQL 8.0 and Redis 7.
|
|
343
421
|
See [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md) for contribution guidelines.
|
|
344
422
|
|
|
345
423
|
## Documentation
|
package/core/autoInvalidate.js
CHANGED
|
@@ -10,12 +10,12 @@ let invalidationRules = {};
|
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* Enables auto invalidation feature
|
|
13
|
-
* @param {Object} config - Configuration options
|
|
13
|
+
* @param {Object|boolean} config - Configuration options, or true/false to enable/disable
|
|
14
14
|
* @param {boolean} config.enabled - Enable/disable auto invalidation
|
|
15
15
|
* @param {Object} config.tables - Table-specific invalidation patterns
|
|
16
16
|
*/
|
|
17
17
|
function enableAutoInvalidation(config = {}) {
|
|
18
|
-
autoInvalidationEnabled = config.enabled !== false;
|
|
18
|
+
autoInvalidationEnabled = typeof config === 'boolean' ? config : config.enabled !== false;
|
|
19
19
|
invalidationRules = config.tables || {};
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -123,6 +123,16 @@ function determineInvalidationPatterns(sql, manualPattern = null) {
|
|
|
123
123
|
|
|
124
124
|
const GLOB_SPECIAL_CHARS = /[*?[\]\\]/;
|
|
125
125
|
|
|
126
|
+
/**
|
|
127
|
+
* The pattern without its trailing '*' characters (a loop: /\*+$/ is quadratic on long runs of
|
|
128
|
+
* '*' inside the pattern)
|
|
129
|
+
*/
|
|
130
|
+
function stripTrailingStars(pattern) {
|
|
131
|
+
let end = pattern.length;
|
|
132
|
+
while (end > 0 && pattern.charCodeAt(end - 1) === 42 /* '*' */) end--;
|
|
133
|
+
return end === pattern.length ? pattern : pattern.slice(0, end);
|
|
134
|
+
}
|
|
135
|
+
|
|
126
136
|
/**
|
|
127
137
|
* Reduces a list of prefix patterns to fewer Redis SCAN passes, without changing which keys
|
|
128
138
|
* they match. Patterns are prefixes (delPrefixKeyItem appends '*'), so trailing '*' do not
|
|
@@ -139,7 +149,7 @@ function mergeInvalidationPatterns(patterns) {
|
|
|
139
149
|
const literals = [];
|
|
140
150
|
for (const pattern of patterns) {
|
|
141
151
|
if (typeof pattern !== 'string') continue;
|
|
142
|
-
const literal = pattern
|
|
152
|
+
const literal = stripTrailingStars(pattern);
|
|
143
153
|
if (literal && !GLOB_SPECIAL_CHARS.test(literal)) literals.push(literal);
|
|
144
154
|
}
|
|
145
155
|
|
|
@@ -148,7 +158,7 @@ function mergeInvalidationPatterns(patterns) {
|
|
|
148
158
|
const seen = new Set();
|
|
149
159
|
|
|
150
160
|
for (const pattern of patterns) {
|
|
151
|
-
const literal = typeof pattern === 'string' ? pattern
|
|
161
|
+
const literal = typeof pattern === 'string' ? stripTrailingStars(pattern) : null;
|
|
152
162
|
if (!literal || GLOB_SPECIAL_CHARS.test(literal)) {
|
|
153
163
|
result.push(pattern);
|
|
154
164
|
continue;
|
package/core/autoKey.js
CHANGED
|
@@ -11,10 +11,10 @@ let autoKeyEnabled = false;
|
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Enables auto key generation feature
|
|
14
|
-
* @param {Object} config - Configuration options
|
|
14
|
+
* @param {Object|boolean} config - Configuration options, or true/false to enable/disable
|
|
15
15
|
*/
|
|
16
16
|
function enableAutoKey(config = {}) {
|
|
17
|
-
autoKeyEnabled = config.enabled !== false;
|
|
17
|
+
autoKeyEnabled = typeof config === 'boolean' ? config : config.enabled !== false;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
/**
|
|
@@ -24,6 +24,26 @@ function isAutoKeyEnabled() {
|
|
|
24
24
|
return autoKeyEnabled || process.env.CORE_AUTO_FEATURES === 'true';
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
/**
|
|
28
|
+
* JSON.stringify replacer for parameters: JSON.stringify throws on a BigInt, so it is encoded
|
|
29
|
+
* as {"$bigint":"<digits>"}. Every other value is returned as is (unchanged hashes).
|
|
30
|
+
*/
|
|
31
|
+
function bigIntReplacer(name, value) {
|
|
32
|
+
return typeof value === 'bigint' ? { $bigint: value.toString() } : value;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* JSON of the parameters for hashing. The replacer is only used when plain JSON.stringify
|
|
37
|
+
* throws (a replacer function takes V8 off its fast path for every other call).
|
|
38
|
+
*/
|
|
39
|
+
function serializeParameters(parameters) {
|
|
40
|
+
try {
|
|
41
|
+
return JSON.stringify(parameters);
|
|
42
|
+
} catch (_) {
|
|
43
|
+
return JSON.stringify(parameters, bigIntReplacer);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
27
47
|
/**
|
|
28
48
|
* Creates a short hash from parameters
|
|
29
49
|
* @param {Array} parameters - Query parameters
|
|
@@ -36,7 +56,7 @@ function createParameterHash(parameters) {
|
|
|
36
56
|
|
|
37
57
|
return crypto
|
|
38
58
|
.createHash('md5')
|
|
39
|
-
.update(
|
|
59
|
+
.update(serializeParameters(parameters))
|
|
40
60
|
.digest('hex')
|
|
41
61
|
.substring(0, 8);
|
|
42
62
|
}
|
|
@@ -80,8 +100,9 @@ function extractWhereConditions(sql) {
|
|
|
80
100
|
|
|
81
101
|
const whereClause = whereMatch[1].trim();
|
|
82
102
|
|
|
83
|
-
// Split by AND/OR but preserve the column names
|
|
84
|
-
|
|
103
|
+
// Split by AND/OR but preserve the column names. A match only starts where a whitespace
|
|
104
|
+
// run starts (same splits as /\s+(AND|OR)\s+/i, which is quadratic on long whitespace runs)
|
|
105
|
+
const conditions = whereClause.split(/(?<!\s)\s+(AND|OR)\s+/i)
|
|
85
106
|
.filter(part => !/^(AND|OR)$/i.test(part.trim()));
|
|
86
107
|
|
|
87
108
|
return conditions.map(c => c.trim());
|
|
@@ -100,7 +121,8 @@ function extractColumnNames(conditions) {
|
|
|
100
121
|
// Match: column_name > ?
|
|
101
122
|
// Match: column_name IN (?)
|
|
102
123
|
// Match: column_name LIKE ?
|
|
103
|
-
|
|
124
|
+
// (\b: the match always starts at a word start anyway; without it, long words are quadratic)
|
|
125
|
+
const columnMatch = condition.match(/\b(\w+)\s*(?:=|>|<|>=|<=|!=|<>|IN|LIKE|NOT\s+IN|NOT\s+LIKE)/i);
|
|
104
126
|
|
|
105
127
|
if (columnMatch) {
|
|
106
128
|
columns.push(columnMatch[1].toLowerCase());
|
|
@@ -155,6 +177,20 @@ function analyzeSql(sql) {
|
|
|
155
177
|
return analysis;
|
|
156
178
|
}
|
|
157
179
|
|
|
180
|
+
/**
|
|
181
|
+
* Key of SQL without a detectable table: the SQL hash, plus the parameter hash when there are
|
|
182
|
+
* parameters, so that e.g. CALL get_user(?) for two different users never shares a key
|
|
183
|
+
* @param {string} queryKey - 'query:<sqlhash>'
|
|
184
|
+
* @param {Array} parameters - Query parameters
|
|
185
|
+
* @returns {string} - Cache key
|
|
186
|
+
*/
|
|
187
|
+
function queryKeyWithParameters(queryKey, parameters) {
|
|
188
|
+
if (!parameters || parameters.length === 0) {
|
|
189
|
+
return queryKey;
|
|
190
|
+
}
|
|
191
|
+
return `${queryKey}:${createParameterHash(parameters)}`;
|
|
192
|
+
}
|
|
193
|
+
|
|
158
194
|
/**
|
|
159
195
|
* Generates cache key with detailed column information
|
|
160
196
|
* @param {string} sql - SQL query
|
|
@@ -165,7 +201,7 @@ function generateDetailedKey(sql, parameters) {
|
|
|
165
201
|
const { tableName, hasConditions, columnPart, queryKey } = analyzeSql(sql);
|
|
166
202
|
|
|
167
203
|
if (!tableName) {
|
|
168
|
-
return queryKey;
|
|
204
|
+
return queryKeyWithParameters(queryKey, parameters);
|
|
169
205
|
}
|
|
170
206
|
|
|
171
207
|
if (!hasConditions) {
|
|
@@ -196,7 +232,7 @@ function generateSimpleKey(sql, parameters) {
|
|
|
196
232
|
const { tableName, queryKey } = analyzeSql(sql);
|
|
197
233
|
|
|
198
234
|
if (!tableName) {
|
|
199
|
-
return queryKey;
|
|
235
|
+
return queryKeyWithParameters(queryKey, parameters);
|
|
200
236
|
}
|
|
201
237
|
|
|
202
238
|
const paramHash = createParameterHash(parameters);
|