zerobucket 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -8
- package/dist/{client-C7xs1dlr.d.cts → client-XQ22b1l3.d.cts} +36 -4
- package/dist/{client-C7xs1dlr.d.ts → client-XQ22b1l3.d.ts} +36 -4
- package/dist/http.d.cts +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.cjs +728 -4
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +110 -3
- package/dist/index.d.ts +110 -3
- package/dist/index.js +725 -2
- package/dist/index.js.map +1 -1
- package/package.json +9 -4
package/dist/index.cjs
CHANGED
|
@@ -39,6 +39,7 @@ __export(src_exports, {
|
|
|
39
39
|
ImageNotFoundError: () => ImageNotFoundError,
|
|
40
40
|
ImageTooLargeError: () => ImageTooLargeError,
|
|
41
41
|
ImageValidationError: () => ImageValidationError,
|
|
42
|
+
MySQLBackend: () => MySQLBackend,
|
|
42
43
|
ObjectStorage: () => ObjectStorage,
|
|
43
44
|
PDFValidator: () => PDFValidator,
|
|
44
45
|
PostgresBackend: () => PostgresBackend,
|
|
@@ -59,7 +60,7 @@ var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${_
|
|
|
59
60
|
var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
|
|
60
61
|
|
|
61
62
|
// src/client.ts
|
|
62
|
-
var
|
|
63
|
+
var import_node_crypto3 = require("crypto");
|
|
63
64
|
var import_promises = require("fs/promises");
|
|
64
65
|
var import_node_path2 = require("path");
|
|
65
66
|
|
|
@@ -1984,6 +1985,711 @@ var SQLiteBackend = class {
|
|
|
1984
1985
|
}
|
|
1985
1986
|
};
|
|
1986
1987
|
|
|
1988
|
+
// src/adapters/mysql.ts
|
|
1989
|
+
var import_node_crypto2 = require("crypto");
|
|
1990
|
+
var SCHEMA3 = `
|
|
1991
|
+
CREATE TABLE IF NOT EXISTS zerobucket_images (
|
|
1992
|
+
id CHAR(36) PRIMARY KEY,
|
|
1993
|
+
data LONGBLOB NOT NULL,
|
|
1994
|
+
mime_type VARCHAR(255) NOT NULL,
|
|
1995
|
+
original_filename TEXT,
|
|
1996
|
+
size_bytes INT NOT NULL,
|
|
1997
|
+
width INT,
|
|
1998
|
+
height INT,
|
|
1999
|
+
checksum_sha256 CHAR(64) NOT NULL,
|
|
2000
|
+
created_at DATETIME(6) NOT NULL,
|
|
2001
|
+
updated_at DATETIME(6) NOT NULL,
|
|
2002
|
+
INDEX idx_zerobucket_checksum (checksum_sha256),
|
|
2003
|
+
INDEX idx_zerobucket_created_at (created_at)
|
|
2004
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
|
|
2005
|
+
var TIERING_CHECK = "chk_zerobucket_storage";
|
|
2006
|
+
var MIGRATE_LOCK = "zerobucket_migrate";
|
|
2007
|
+
var COLS2 = `id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, storage_backend, object_storage_bucket, object_storage_key`;
|
|
2008
|
+
var INSERT3 = `INSERT INTO zerobucket_images (id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
|
|
2009
|
+
var SELECT_FULL3 = `SELECT ${COLS2} FROM zerobucket_images WHERE id = ?`;
|
|
2010
|
+
var SELECT_MANY2 = `SELECT ${COLS2} FROM zerobucket_images WHERE id IN (?)`;
|
|
2011
|
+
var SELECT_METADATA3 = `SELECT id, mime_type, original_filename, size_bytes, width, height, checksum_sha256 FROM zerobucket_images WHERE id = ?`;
|
|
2012
|
+
var STREAM_INFO2 = `SELECT size_bytes, storage_backend, object_storage_key FROM zerobucket_images WHERE id = ?`;
|
|
2013
|
+
var CHUNK2 = `SELECT SUBSTRING(data FROM ? FOR ?) AS chunk FROM zerobucket_images WHERE id = ?`;
|
|
2014
|
+
var EXISTS3 = `SELECT 1 FROM zerobucket_images WHERE id = ?`;
|
|
2015
|
+
var LOCK_ONE = `SELECT storage_backend, object_storage_key FROM zerobucket_images WHERE id = ? FOR UPDATE`;
|
|
2016
|
+
var DELETE_ONE2 = `DELETE FROM zerobucket_images WHERE id = ?`;
|
|
2017
|
+
var LOCK_MANY = `SELECT id, storage_backend, object_storage_key FROM zerobucket_images WHERE id IN (?) ORDER BY id FOR UPDATE`;
|
|
2018
|
+
var DELETE_MANY2 = `DELETE FROM zerobucket_images WHERE id IN (?)`;
|
|
2019
|
+
var SELECT_FOR_TIERING3 = `SELECT data, mime_type, size_bytes, storage_backend FROM zerobucket_images WHERE id = ? FOR UPDATE`;
|
|
2020
|
+
var UPDATE_AFTER_TIERING3 = `UPDATE zerobucket_images SET data = NULL, storage_backend = 'object_storage', object_storage_bucket = ?, object_storage_key = ?, updated_at = ? WHERE id = ?`;
|
|
2021
|
+
var LOCAL = "mysql";
|
|
2022
|
+
var RETRYABLE_ERRNOS = /* @__PURE__ */ new Set([
|
|
2023
|
+
1205,
|
|
2024
|
+
// ER_LOCK_WAIT_TIMEOUT
|
|
2025
|
+
1213,
|
|
2026
|
+
// ER_LOCK_DEADLOCK
|
|
2027
|
+
1040,
|
|
2028
|
+
// ER_CON_COUNT_ERROR (too many connections)
|
|
2029
|
+
1053,
|
|
2030
|
+
// ER_SERVER_SHUTDOWN
|
|
2031
|
+
1927,
|
|
2032
|
+
// ER_CONNECTION_KILLED (MariaDB)
|
|
2033
|
+
2002,
|
|
2034
|
+
// CR_CONNECTION_ERROR
|
|
2035
|
+
2003,
|
|
2036
|
+
// CR_CONN_HOST_ERROR
|
|
2037
|
+
2006,
|
|
2038
|
+
// CR_SERVER_GONE_ERROR
|
|
2039
|
+
2013
|
|
2040
|
+
// CR_SERVER_LOST
|
|
2041
|
+
]);
|
|
2042
|
+
var CONNECTION_CODES = /* @__PURE__ */ new Set([
|
|
2043
|
+
"PROTOCOL_CONNECTION_LOST",
|
|
2044
|
+
"PROTOCOL_ENQUEUE_AFTER_FATAL_ERROR",
|
|
2045
|
+
"PROTOCOL_ENQUEUE_AFTER_QUIT",
|
|
2046
|
+
"ECONNRESET",
|
|
2047
|
+
"ECONNREFUSED",
|
|
2048
|
+
"ETIMEDOUT",
|
|
2049
|
+
"EPIPE",
|
|
2050
|
+
"ECONNABORTED"
|
|
2051
|
+
]);
|
|
2052
|
+
function errnoOf(exc) {
|
|
2053
|
+
const n = exc?.errno;
|
|
2054
|
+
return typeof n === "number" ? n : void 0;
|
|
2055
|
+
}
|
|
2056
|
+
var PoolTimeoutError = class extends StorageError {
|
|
2057
|
+
};
|
|
2058
|
+
function isRetryable2(exc) {
|
|
2059
|
+
if (!(exc instanceof Error) || exc instanceof PoolTimeoutError) return false;
|
|
2060
|
+
const e = exc;
|
|
2061
|
+
const n = errnoOf(exc);
|
|
2062
|
+
if (n !== void 0 && RETRYABLE_ERRNOS.has(n)) return true;
|
|
2063
|
+
if (typeof e.code === "string") {
|
|
2064
|
+
if (CONNECTION_CODES.has(e.code)) return true;
|
|
2065
|
+
if (e.code === "ER_LOCK_WAIT_TIMEOUT" || e.code === "ER_LOCK_DEADLOCK" || e.code === "ER_CON_COUNT_ERROR" || e.code === "ER_SERVER_SHUTDOWN" || e.code === "ER_CONNECTION_KILLED")
|
|
2066
|
+
return true;
|
|
2067
|
+
}
|
|
2068
|
+
return false;
|
|
2069
|
+
}
|
|
2070
|
+
function isConnectionLevel2(exc) {
|
|
2071
|
+
if (!(exc instanceof Error)) return false;
|
|
2072
|
+
const e = exc;
|
|
2073
|
+
if (e.fatal === true) return true;
|
|
2074
|
+
if (typeof e.code === "string" && CONNECTION_CODES.has(e.code)) return true;
|
|
2075
|
+
const n = errnoOf(exc);
|
|
2076
|
+
return n === 2006 || n === 2013 || n === 1927 || n === 1053;
|
|
2077
|
+
}
|
|
2078
|
+
var MAX_BACKOFF_MS2 = 2e3;
|
|
2079
|
+
function backoffDelayMs2(attempt, baseDelayMs) {
|
|
2080
|
+
const exponential = baseDelayMs * 2 ** (attempt - 1);
|
|
2081
|
+
const jitter = Math.random() * baseDelayMs;
|
|
2082
|
+
return Math.min(exponential + jitter, MAX_BACKOFF_MS2);
|
|
2083
|
+
}
|
|
2084
|
+
function friendly(prefix, exc) {
|
|
2085
|
+
if (errnoOf(exc) === 1153) {
|
|
2086
|
+
return new StorageError(
|
|
2087
|
+
`${prefix}: the image is larger than the server's max_allowed_packet. Raise it (for example \`SET GLOBAL max_allowed_packet = 67108864\`, and set it in my.cnf to make it permanent), or lower \`maxBytes\`.`,
|
|
2088
|
+
{ cause: exc }
|
|
2089
|
+
);
|
|
2090
|
+
}
|
|
2091
|
+
return wrap(prefix, exc);
|
|
2092
|
+
}
|
|
2093
|
+
function mysqlNow() {
|
|
2094
|
+
return `${(/* @__PURE__ */ new Date()).toISOString().replace("T", " ").replace("Z", "")}000`;
|
|
2095
|
+
}
|
|
2096
|
+
function normalizeUrl(url) {
|
|
2097
|
+
return url.replace(/^mariadb:\/\//i, "mysql://");
|
|
2098
|
+
}
|
|
2099
|
+
function isMysqlConnection(c) {
|
|
2100
|
+
const m = c;
|
|
2101
|
+
return typeof m.query === "function" && typeof m.execute === "function";
|
|
2102
|
+
}
|
|
2103
|
+
var MySQLBackend = class {
|
|
2104
|
+
constructor(opts) {
|
|
2105
|
+
this.opts = opts;
|
|
2106
|
+
if (!opts.connectionString && !opts.pool) {
|
|
2107
|
+
throw new TypeError(
|
|
2108
|
+
"MySQLBackend requires either `connectionString` or `pool`"
|
|
2109
|
+
);
|
|
2110
|
+
}
|
|
2111
|
+
if (opts.dedup) {
|
|
2112
|
+
throw new TypeError(
|
|
2113
|
+
"dedup mode is not available for MySQL in the npm package yet (the Python package supports it). Use classic mode, or Postgres/SQLite for dedup."
|
|
2114
|
+
);
|
|
2115
|
+
}
|
|
2116
|
+
this.ownsPool = !opts.pool;
|
|
2117
|
+
this.pool = opts.pool;
|
|
2118
|
+
this.maxRetries = opts.maxRetries ?? 3;
|
|
2119
|
+
this.retryBaseDelayMs = opts.retryBaseDelayMs ?? 100;
|
|
2120
|
+
this.poolTimeoutMs = opts.poolTimeoutMs ?? 1e4;
|
|
2121
|
+
this.autoMigrate = opts.autoMigrate ?? true;
|
|
2122
|
+
}
|
|
2123
|
+
opts;
|
|
2124
|
+
pool;
|
|
2125
|
+
ownsPool;
|
|
2126
|
+
readyPromise;
|
|
2127
|
+
closed = false;
|
|
2128
|
+
maxRetries;
|
|
2129
|
+
retryBaseDelayMs;
|
|
2130
|
+
poolTimeoutMs;
|
|
2131
|
+
autoMigrate;
|
|
2132
|
+
// ---- lifecycle -------------------------------------------------------
|
|
2133
|
+
ready() {
|
|
2134
|
+
if (this.closed)
|
|
2135
|
+
return Promise.reject(
|
|
2136
|
+
new StorageError("This ZeroBucket instance has been closed")
|
|
2137
|
+
);
|
|
2138
|
+
this.readyPromise ??= this.init().catch((exc) => {
|
|
2139
|
+
this.readyPromise = void 0;
|
|
2140
|
+
throw exc;
|
|
2141
|
+
});
|
|
2142
|
+
return this.readyPromise;
|
|
2143
|
+
}
|
|
2144
|
+
async init() {
|
|
2145
|
+
if (!this.pool) {
|
|
2146
|
+
let mod;
|
|
2147
|
+
try {
|
|
2148
|
+
mod = await import("mysql2/promise");
|
|
2149
|
+
} catch (cause) {
|
|
2150
|
+
throw new StorageError(
|
|
2151
|
+
"MySQLBackend requires the 'mysql2' package. Install it with: npm install mysql2",
|
|
2152
|
+
{ cause }
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
const createPool = (mod.default ?? mod).createPool;
|
|
2156
|
+
this.pool = createPool({
|
|
2157
|
+
uri: normalizeUrl(this.opts.connectionString),
|
|
2158
|
+
connectionLimit: this.opts.poolMaxSize ?? 5,
|
|
2159
|
+
waitForConnections: true,
|
|
2160
|
+
queueLimit: 0,
|
|
2161
|
+
connectTimeout: this.poolTimeoutMs,
|
|
2162
|
+
charset: "utf8mb4",
|
|
2163
|
+
timezone: "Z",
|
|
2164
|
+
enableKeepAlive: true
|
|
2165
|
+
});
|
|
2166
|
+
}
|
|
2167
|
+
try {
|
|
2168
|
+
if (this.autoMigrate) {
|
|
2169
|
+
await this.runPooled("migrate", (q) => this.migrate(q), false);
|
|
2170
|
+
} else {
|
|
2171
|
+
await this.runPooled("ping", (q) => q.query("SELECT 1"), false);
|
|
2172
|
+
}
|
|
2173
|
+
} catch (exc) {
|
|
2174
|
+
if (this.ownsPool) {
|
|
2175
|
+
await this.pool.end().catch(() => {
|
|
2176
|
+
});
|
|
2177
|
+
this.pool = void 0;
|
|
2178
|
+
}
|
|
2179
|
+
throw exc instanceof StorageError ? exc : new StorageError(`Could not connect to MySQL/MariaDB: ${msg(exc)}`, {
|
|
2180
|
+
cause: exc
|
|
2181
|
+
});
|
|
2182
|
+
}
|
|
2183
|
+
}
|
|
2184
|
+
/**
|
|
2185
|
+
* Create the schema if missing, under a server-side named lock so concurrent first callers (in this
|
|
2186
|
+
* or any other process) run the DDL one at a time. Every step checks before it changes anything, so
|
|
2187
|
+
* re-running is a no-op.
|
|
2188
|
+
*/
|
|
2189
|
+
async migrate(q) {
|
|
2190
|
+
const locked = (await q.query("SELECT GET_LOCK(?, 30) AS got", [MIGRATE_LOCK]))[0][0]?.got;
|
|
2191
|
+
if (locked !== 1) {
|
|
2192
|
+
throw new StorageError(
|
|
2193
|
+
"Timed out waiting for another process to finish migrating the ZeroBucket schema."
|
|
2194
|
+
);
|
|
2195
|
+
}
|
|
2196
|
+
try {
|
|
2197
|
+
const tableExists = (await q.query(
|
|
2198
|
+
"SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images'"
|
|
2199
|
+
))[0].length > 0;
|
|
2200
|
+
if (!tableExists) await q.query(SCHEMA3);
|
|
2201
|
+
const cols = new Map(
|
|
2202
|
+
(await q.query(
|
|
2203
|
+
"SELECT COLUMN_NAME AS c, IS_NULLABLE AS nullable FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images'"
|
|
2204
|
+
))[0].map((r) => [r.c, r.nullable])
|
|
2205
|
+
);
|
|
2206
|
+
if (!cols.has("storage_backend"))
|
|
2207
|
+
await q.query(
|
|
2208
|
+
`ALTER TABLE zerobucket_images ADD COLUMN storage_backend VARCHAR(20) NOT NULL DEFAULT '${LOCAL}'`
|
|
2209
|
+
);
|
|
2210
|
+
if (!cols.has("object_storage_bucket"))
|
|
2211
|
+
await q.query(
|
|
2212
|
+
"ALTER TABLE zerobucket_images ADD COLUMN object_storage_bucket VARCHAR(255)"
|
|
2213
|
+
);
|
|
2214
|
+
if (!cols.has("object_storage_key"))
|
|
2215
|
+
await q.query(
|
|
2216
|
+
"ALTER TABLE zerobucket_images ADD COLUMN object_storage_key VARCHAR(512)"
|
|
2217
|
+
);
|
|
2218
|
+
if (cols.get("data") === "NO")
|
|
2219
|
+
await q.query(
|
|
2220
|
+
"ALTER TABLE zerobucket_images MODIFY data LONGBLOB NULL"
|
|
2221
|
+
);
|
|
2222
|
+
const hasCheck = (await q.query(
|
|
2223
|
+
"SELECT 1 FROM information_schema.TABLE_CONSTRAINTS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images' AND CONSTRAINT_NAME = ?",
|
|
2224
|
+
[TIERING_CHECK]
|
|
2225
|
+
))[0].length > 0;
|
|
2226
|
+
if (!hasCheck) {
|
|
2227
|
+
await q.query(
|
|
2228
|
+
`ALTER TABLE zerobucket_images ADD CONSTRAINT ${TIERING_CHECK} CHECK ((storage_backend = '${LOCAL}' AND data IS NOT NULL AND object_storage_bucket IS NULL AND object_storage_key IS NULL) OR (storage_backend = 'object_storage' AND data IS NULL AND object_storage_bucket IS NOT NULL AND object_storage_key IS NOT NULL))`
|
|
2229
|
+
);
|
|
2230
|
+
}
|
|
2231
|
+
} catch (exc) {
|
|
2232
|
+
throw new StorageError(`Migration failed: ${msg(exc)}`, { cause: exc });
|
|
2233
|
+
} finally {
|
|
2234
|
+
await q.query("SELECT RELEASE_LOCK(?)", [MIGRATE_LOCK]).catch(() => {
|
|
2235
|
+
});
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
async close() {
|
|
2239
|
+
this.closed = true;
|
|
2240
|
+
const pool = this.pool;
|
|
2241
|
+
this.pool = void 0;
|
|
2242
|
+
this.readyPromise = void 0;
|
|
2243
|
+
if (pool && this.ownsPool) await pool.end();
|
|
2244
|
+
}
|
|
2245
|
+
// ---- execution core --------------------------------------------------
|
|
2246
|
+
emit(operation, start, success, error, retryCount) {
|
|
2247
|
+
const cb = this.opts.onOperation;
|
|
2248
|
+
if (!cb) return;
|
|
2249
|
+
try {
|
|
2250
|
+
cb({
|
|
2251
|
+
operation,
|
|
2252
|
+
durationMs: performance.now() - start,
|
|
2253
|
+
success,
|
|
2254
|
+
error,
|
|
2255
|
+
retryCount
|
|
2256
|
+
});
|
|
2257
|
+
} catch {
|
|
2258
|
+
}
|
|
2259
|
+
}
|
|
2260
|
+
/** Wait for a pool slot, but never longer than poolTimeoutMs. */
|
|
2261
|
+
acquire() {
|
|
2262
|
+
const pool = this.pool;
|
|
2263
|
+
if (this.closed || !pool)
|
|
2264
|
+
return Promise.reject(
|
|
2265
|
+
new StorageError("This ZeroBucket instance has been closed")
|
|
2266
|
+
);
|
|
2267
|
+
return new Promise((resolve, reject) => {
|
|
2268
|
+
let settled = false;
|
|
2269
|
+
const timer = setTimeout(() => {
|
|
2270
|
+
settled = true;
|
|
2271
|
+
reject(
|
|
2272
|
+
new PoolTimeoutError(
|
|
2273
|
+
`Timed out after ${this.poolTimeoutMs}ms waiting for a MySQL/MariaDB connection (poolMaxSize=${this.opts.poolMaxSize ?? 5}; all connections are in use).`
|
|
2274
|
+
)
|
|
2275
|
+
);
|
|
2276
|
+
}, this.poolTimeoutMs);
|
|
2277
|
+
pool.getConnection().then(
|
|
2278
|
+
(conn) => {
|
|
2279
|
+
if (settled) {
|
|
2280
|
+
conn.release();
|
|
2281
|
+
return;
|
|
2282
|
+
}
|
|
2283
|
+
settled = true;
|
|
2284
|
+
clearTimeout(timer);
|
|
2285
|
+
resolve(conn);
|
|
2286
|
+
},
|
|
2287
|
+
(exc) => {
|
|
2288
|
+
if (settled) return;
|
|
2289
|
+
settled = true;
|
|
2290
|
+
clearTimeout(timer);
|
|
2291
|
+
reject(exc);
|
|
2292
|
+
}
|
|
2293
|
+
);
|
|
2294
|
+
});
|
|
2295
|
+
}
|
|
2296
|
+
/**
|
|
2297
|
+
* Run `work` on a pooled connection. `tx` wraps it in a transaction. Transient failures retry the
|
|
2298
|
+
* WHOLE closure on a fresh connection after a full rollback.
|
|
2299
|
+
*/
|
|
2300
|
+
async runPooled(operation, work, tx) {
|
|
2301
|
+
const start = performance.now();
|
|
2302
|
+
let attempt = 0;
|
|
2303
|
+
for (; ; ) {
|
|
2304
|
+
let conn;
|
|
2305
|
+
let destroy = false;
|
|
2306
|
+
try {
|
|
2307
|
+
conn = await this.acquire();
|
|
2308
|
+
if (tx) await conn.beginTransaction();
|
|
2309
|
+
const result = await work(conn);
|
|
2310
|
+
if (tx) await conn.commit();
|
|
2311
|
+
this.emit(operation, start, true, null, attempt);
|
|
2312
|
+
return result;
|
|
2313
|
+
} catch (exc) {
|
|
2314
|
+
if (conn && tx) {
|
|
2315
|
+
try {
|
|
2316
|
+
await conn.rollback();
|
|
2317
|
+
} catch {
|
|
2318
|
+
destroy = true;
|
|
2319
|
+
}
|
|
2320
|
+
}
|
|
2321
|
+
if (isConnectionLevel2(exc)) destroy = true;
|
|
2322
|
+
attempt += 1;
|
|
2323
|
+
if (attempt > this.maxRetries || !isRetryable2(exc)) {
|
|
2324
|
+
this.emit(operation, start, false, msg(exc), attempt - 1);
|
|
2325
|
+
throw exc;
|
|
2326
|
+
}
|
|
2327
|
+
} finally {
|
|
2328
|
+
if (conn) {
|
|
2329
|
+
if (destroy) conn.destroy();
|
|
2330
|
+
else conn.release();
|
|
2331
|
+
}
|
|
2332
|
+
}
|
|
2333
|
+
await sleep(backoffDelayMs2(attempt, this.retryBaseDelayMs));
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
/**
|
|
2337
|
+
* Run `work` and return its result, emitting exactly one OperationEvent.
|
|
2338
|
+
*
|
|
2339
|
+
* connection given -> run once on it; no retry, and no commit/rollback of the caller's transaction.
|
|
2340
|
+
* connection absent -> pooled, retried (see runPooled).
|
|
2341
|
+
*/
|
|
2342
|
+
async run(operation, connection, work, { tx = false } = {}) {
|
|
2343
|
+
if (connection) {
|
|
2344
|
+
if (!isMysqlConnection(connection)) {
|
|
2345
|
+
throw new TypeError(
|
|
2346
|
+
"MySQLBackend expects a mysql2/promise connection as `connection` (an object with query() and execute())."
|
|
2347
|
+
);
|
|
2348
|
+
}
|
|
2349
|
+
const start = performance.now();
|
|
2350
|
+
try {
|
|
2351
|
+
const result = tx ? await this.atomic(connection, work) : await work(connection);
|
|
2352
|
+
this.emit(operation, start, true, null, 0);
|
|
2353
|
+
return result;
|
|
2354
|
+
} catch (exc) {
|
|
2355
|
+
this.emit(operation, start, false, msg(exc), 0);
|
|
2356
|
+
throw exc;
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
await this.ready();
|
|
2360
|
+
return this.runPooled(operation, work, tx);
|
|
2361
|
+
}
|
|
2362
|
+
/**
|
|
2363
|
+
* Make multi-statement work all-or-nothing on a CALLER-supplied connection.
|
|
2364
|
+
*
|
|
2365
|
+
* Is a transaction open? SAVEPOINT followed by ROLLBACK TO SAVEPOINT succeeds only inside one (in
|
|
2366
|
+
* autocommit mode the savepoint is gone before the next statement and the rollback fails with 1305).
|
|
2367
|
+
* - Open -> reuse that SAVEPOINT: a failure undoes only our work; success joins the caller's transaction.
|
|
2368
|
+
* - None -> START TRANSACTION and COMMIT our own, so a failure can never leave half-applied state.
|
|
2369
|
+
*/
|
|
2370
|
+
async atomic(conn, work) {
|
|
2371
|
+
let inTransaction;
|
|
2372
|
+
await conn.query("SAVEPOINT zerobucket_atomic");
|
|
2373
|
+
try {
|
|
2374
|
+
await conn.query("ROLLBACK TO SAVEPOINT zerobucket_atomic");
|
|
2375
|
+
inTransaction = true;
|
|
2376
|
+
} catch (exc) {
|
|
2377
|
+
if (errnoOf(exc) !== 1305) throw exc;
|
|
2378
|
+
inTransaction = false;
|
|
2379
|
+
await conn.query("START TRANSACTION");
|
|
2380
|
+
}
|
|
2381
|
+
try {
|
|
2382
|
+
const result = await work(conn);
|
|
2383
|
+
await conn.query(
|
|
2384
|
+
inTransaction ? "RELEASE SAVEPOINT zerobucket_atomic" : "COMMIT"
|
|
2385
|
+
);
|
|
2386
|
+
return result;
|
|
2387
|
+
} catch (exc) {
|
|
2388
|
+
try {
|
|
2389
|
+
await conn.query(
|
|
2390
|
+
inTransaction ? "ROLLBACK TO SAVEPOINT zerobucket_atomic" : "ROLLBACK"
|
|
2391
|
+
);
|
|
2392
|
+
if (inTransaction)
|
|
2393
|
+
await conn.query("RELEASE SAVEPOINT zerobucket_atomic");
|
|
2394
|
+
} catch {
|
|
2395
|
+
}
|
|
2396
|
+
throw exc;
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
// ---- StorageBackend --------------------------------------------------
|
|
2400
|
+
async put(row, { connection } = {}) {
|
|
2401
|
+
try {
|
|
2402
|
+
const id = (0, import_node_crypto2.randomUUID)();
|
|
2403
|
+
await this.run("put", connection, async (q) => {
|
|
2404
|
+
await q.execute(INSERT3, rowParams2(id, row, mysqlNow()));
|
|
2405
|
+
});
|
|
2406
|
+
return id;
|
|
2407
|
+
} catch (exc) {
|
|
2408
|
+
throw friendly("Failed to store image", exc);
|
|
2409
|
+
}
|
|
2410
|
+
}
|
|
2411
|
+
async putMany(rows, { connection } = {}) {
|
|
2412
|
+
if (rows.length === 0) return [];
|
|
2413
|
+
try {
|
|
2414
|
+
const ids = rows.map(() => (0, import_node_crypto2.randomUUID)());
|
|
2415
|
+
await this.run(
|
|
2416
|
+
"put_many",
|
|
2417
|
+
connection,
|
|
2418
|
+
async (q) => {
|
|
2419
|
+
const now = mysqlNow();
|
|
2420
|
+
for (let i = 0; i < rows.length; i++)
|
|
2421
|
+
await q.execute(INSERT3, rowParams2(ids[i], rows[i], now));
|
|
2422
|
+
},
|
|
2423
|
+
{ tx: true }
|
|
2424
|
+
);
|
|
2425
|
+
return ids;
|
|
2426
|
+
} catch (exc) {
|
|
2427
|
+
throw friendly("Failed to store image batch", exc);
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
async get(id, { connection } = {}) {
|
|
2431
|
+
if (!isValidId(id)) return null;
|
|
2432
|
+
let row;
|
|
2433
|
+
try {
|
|
2434
|
+
row = await this.run(
|
|
2435
|
+
"get",
|
|
2436
|
+
connection,
|
|
2437
|
+
async (q) => (await q.query(SELECT_FULL3, [id]))[0][0]
|
|
2438
|
+
);
|
|
2439
|
+
} catch (exc) {
|
|
2440
|
+
throw wrap("Failed to retrieve image", exc);
|
|
2441
|
+
}
|
|
2442
|
+
return row ? this.hydrate(row, id) : null;
|
|
2443
|
+
}
|
|
2444
|
+
async getMany(ids, { connection } = {}) {
|
|
2445
|
+
const valid = ids.filter(isValidId);
|
|
2446
|
+
if (valid.length === 0) return [];
|
|
2447
|
+
let rows;
|
|
2448
|
+
try {
|
|
2449
|
+
rows = await this.run(
|
|
2450
|
+
"get_many",
|
|
2451
|
+
connection,
|
|
2452
|
+
async (q) => (await q.query(SELECT_MANY2, [valid]))[0]
|
|
2453
|
+
);
|
|
2454
|
+
} catch (exc) {
|
|
2455
|
+
throw wrap("Failed to retrieve image batch", exc);
|
|
2456
|
+
}
|
|
2457
|
+
return mapLimit(rows, 4, (r) => this.hydrate(r, String(r.id)));
|
|
2458
|
+
}
|
|
2459
|
+
async getMetadata(id, { connection } = {}) {
|
|
2460
|
+
if (!isValidId(id)) return null;
|
|
2461
|
+
let row;
|
|
2462
|
+
try {
|
|
2463
|
+
row = await this.run(
|
|
2464
|
+
"get_metadata",
|
|
2465
|
+
connection,
|
|
2466
|
+
async (q) => (await q.query(SELECT_METADATA3, [id]))[0][0]
|
|
2467
|
+
);
|
|
2468
|
+
} catch (exc) {
|
|
2469
|
+
throw wrap("Failed to retrieve image metadata", exc);
|
|
2470
|
+
}
|
|
2471
|
+
if (!row) return null;
|
|
2472
|
+
return {
|
|
2473
|
+
id: String(row.id),
|
|
2474
|
+
mimeType: row.mime_type,
|
|
2475
|
+
originalFilename: row.original_filename,
|
|
2476
|
+
sizeBytes: row.size_bytes,
|
|
2477
|
+
width: row.width,
|
|
2478
|
+
height: row.height,
|
|
2479
|
+
checksumSha256: row.checksum_sha256
|
|
2480
|
+
};
|
|
2481
|
+
}
|
|
2482
|
+
async getStream(id, opts) {
|
|
2483
|
+
if (!isValidId(id)) return null;
|
|
2484
|
+
const { connection, chunkSize } = opts;
|
|
2485
|
+
if (!Number.isInteger(chunkSize) || chunkSize <= 0)
|
|
2486
|
+
throw new RangeError("chunkSize must be a positive integer");
|
|
2487
|
+
let info;
|
|
2488
|
+
try {
|
|
2489
|
+
info = await this.run(
|
|
2490
|
+
"get_metadata",
|
|
2491
|
+
connection,
|
|
2492
|
+
async (q) => (await q.query(STREAM_INFO2, [id]))[0][0]
|
|
2493
|
+
);
|
|
2494
|
+
} catch (exc) {
|
|
2495
|
+
throw wrap("Failed to retrieve image metadata", exc);
|
|
2496
|
+
}
|
|
2497
|
+
if (!info) return null;
|
|
2498
|
+
const totalSize = info.size_bytes;
|
|
2499
|
+
const start = opts.range?.start ?? 0;
|
|
2500
|
+
const end = Math.min(opts.range?.end ?? totalSize - 1, totalSize - 1);
|
|
2501
|
+
if (info.storage_backend === "object_storage") {
|
|
2502
|
+
const os = this.opts.objectStorage;
|
|
2503
|
+
if (!os) throw tieredWithoutStorage(id, info.object_storage_key);
|
|
2504
|
+
return streamFromObjectStorage(
|
|
2505
|
+
os,
|
|
2506
|
+
id,
|
|
2507
|
+
info.object_storage_key,
|
|
2508
|
+
start,
|
|
2509
|
+
end,
|
|
2510
|
+
chunkSize
|
|
2511
|
+
);
|
|
2512
|
+
}
|
|
2513
|
+
const self = this;
|
|
2514
|
+
async function* generate() {
|
|
2515
|
+
let offset = start;
|
|
2516
|
+
let delivered = 0;
|
|
2517
|
+
const wanted = Math.max(0, end - start + 1);
|
|
2518
|
+
while (delivered < wanted) {
|
|
2519
|
+
const length = Math.min(chunkSize, wanted - delivered);
|
|
2520
|
+
let row;
|
|
2521
|
+
try {
|
|
2522
|
+
row = await self.run(
|
|
2523
|
+
"get_stream",
|
|
2524
|
+
connection,
|
|
2525
|
+
async (q) => (await q.query(CHUNK2, [offset + 1, length, id]))[0][0]
|
|
2526
|
+
);
|
|
2527
|
+
} catch (exc) {
|
|
2528
|
+
throw wrap("Failed to stream image", exc);
|
|
2529
|
+
}
|
|
2530
|
+
if (!row) {
|
|
2531
|
+
throw new StorageError(
|
|
2532
|
+
`Image ${JSON.stringify(id)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
|
|
2533
|
+
);
|
|
2534
|
+
}
|
|
2535
|
+
const chunk = row.chunk;
|
|
2536
|
+
if (!chunk || chunk.length === 0) {
|
|
2537
|
+
throw new StorageError(
|
|
2538
|
+
`Image ${JSON.stringify(id)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
|
|
2539
|
+
);
|
|
2540
|
+
}
|
|
2541
|
+
yield chunk;
|
|
2542
|
+
offset += chunk.length;
|
|
2543
|
+
delivered += chunk.length;
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
return generate();
|
|
2547
|
+
}
|
|
2548
|
+
async delete(id, { connection } = {}) {
|
|
2549
|
+
if (!isValidId(id)) return false;
|
|
2550
|
+
try {
|
|
2551
|
+
const row = await this.run(
|
|
2552
|
+
"delete",
|
|
2553
|
+
connection,
|
|
2554
|
+
async (q) => {
|
|
2555
|
+
const found = (await q.query(LOCK_ONE, [id]))[0][0];
|
|
2556
|
+
if (!found) return void 0;
|
|
2557
|
+
await q.query(DELETE_ONE2, [id]);
|
|
2558
|
+
return found;
|
|
2559
|
+
},
|
|
2560
|
+
{ tx: true }
|
|
2561
|
+
);
|
|
2562
|
+
if (row?.storage_backend === "object_storage")
|
|
2563
|
+
await this.cleanupObject([row.object_storage_key], connection);
|
|
2564
|
+
return row !== void 0;
|
|
2565
|
+
} catch (exc) {
|
|
2566
|
+
throw wrap("Failed to delete image", exc);
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2569
|
+
async deleteMany(ids, { connection } = {}) {
|
|
2570
|
+
const valid = [...new Set(ids.filter(isValidId))];
|
|
2571
|
+
if (valid.length === 0) return [];
|
|
2572
|
+
try {
|
|
2573
|
+
const rows = await this.run(
|
|
2574
|
+
"delete_many",
|
|
2575
|
+
connection,
|
|
2576
|
+
async (q) => {
|
|
2577
|
+
const found = (await q.query(LOCK_MANY, [valid]))[0];
|
|
2578
|
+
if (found.length === 0) return [];
|
|
2579
|
+
await q.query(DELETE_MANY2, [found.map((r) => r.id)]);
|
|
2580
|
+
return found;
|
|
2581
|
+
},
|
|
2582
|
+
{ tx: true }
|
|
2583
|
+
);
|
|
2584
|
+
await this.cleanupObject(
|
|
2585
|
+
rows.filter((r) => r.storage_backend === "object_storage").map((r) => r.object_storage_key),
|
|
2586
|
+
connection
|
|
2587
|
+
);
|
|
2588
|
+
return rows.map((r) => String(r.id));
|
|
2589
|
+
} catch (exc) {
|
|
2590
|
+
throw wrap("Failed to delete image batch", exc);
|
|
2591
|
+
}
|
|
2592
|
+
}
|
|
2593
|
+
async exists(id, { connection } = {}) {
|
|
2594
|
+
if (!isValidId(id)) return false;
|
|
2595
|
+
try {
|
|
2596
|
+
return await this.run(
|
|
2597
|
+
"exists",
|
|
2598
|
+
connection,
|
|
2599
|
+
async (q) => (await q.query(EXISTS3, [id]))[0].length > 0
|
|
2600
|
+
);
|
|
2601
|
+
} catch (exc) {
|
|
2602
|
+
throw wrap("Failed to check image existence", exc);
|
|
2603
|
+
}
|
|
2604
|
+
}
|
|
2605
|
+
// ---- helpers ---------------------------------------------------------
|
|
2606
|
+
hydrate(row, id) {
|
|
2607
|
+
return hydrateRow(row, id, this.opts.objectStorage);
|
|
2608
|
+
}
|
|
2609
|
+
/**
|
|
2610
|
+
* Remove tiered objects AFTER their rows are gone: once the row is deleted the image is correctly
|
|
2611
|
+
* "not found" to everyone, so a failed object delete only leaves a harmless orphan, whereas deleting
|
|
2612
|
+
* the object first and then failing the row delete would leave a row pointing at nothing.
|
|
2613
|
+
*
|
|
2614
|
+
* Never runs inside a caller-owned transaction (`connection`): we cannot know whether it will
|
|
2615
|
+
* commit. In that case the object stays until you remove it yourself after committing. Failures are
|
|
2616
|
+
* reported through onOperation and never fail the delete itself.
|
|
2617
|
+
*/
|
|
2618
|
+
async cleanupObject(keys, connection) {
|
|
2619
|
+
const os = this.opts.objectStorage;
|
|
2620
|
+
if (!os || connection || keys.length === 0) return;
|
|
2621
|
+
await Promise.all(
|
|
2622
|
+
keys.map(async (key) => {
|
|
2623
|
+
const start = performance.now();
|
|
2624
|
+
try {
|
|
2625
|
+
await os.delete(key);
|
|
2626
|
+
this.emit("object_storage_delete", start, true, null, 0);
|
|
2627
|
+
} catch (exc) {
|
|
2628
|
+
this.emit("object_storage_delete", start, false, msg(exc), 0);
|
|
2629
|
+
}
|
|
2630
|
+
})
|
|
2631
|
+
);
|
|
2632
|
+
}
|
|
2633
|
+
/**
|
|
2634
|
+
* Move an image's bytes out of the database into object storage, replacing the row's `data` with a
|
|
2635
|
+
* pointer. Returns null (no such image), false (already tiered: a safe no-op, so backfills can be
|
|
2636
|
+
* re-run) or true (tiered just now).
|
|
2637
|
+
*
|
|
2638
|
+
* SAFETY: the upload happens INSIDE the same transaction as the row lock (SELECT ... FOR UPDATE)
|
|
2639
|
+
* and the UPDATE that flips the row. If the upload fails the transaction rolls back and the row is
|
|
2640
|
+
* untouched: there is no moment when the bytes exist nowhere, or when a row claims to be tiered
|
|
2641
|
+
* without a completed upload.
|
|
2642
|
+
*
|
|
2643
|
+
* TRADEOFF: InnoDB row-locks only THIS row for the duration of the upload. Concurrent writes to this
|
|
2644
|
+
* image wait (up to innodb_lock_wait_timeout, 50s by default); every other row is unaffected and
|
|
2645
|
+
* plain reads are never blocked. Meant for explicit maintenance, not hot paths.
|
|
2646
|
+
*
|
|
2647
|
+
* Retry is safe: a transient error replays the whole closure, and the deterministic key (the image
|
|
2648
|
+
* id) makes the re-upload an idempotent overwrite.
|
|
2649
|
+
*/
|
|
2650
|
+
async tierToObjectStorage(id, { connection } = {}) {
|
|
2651
|
+
const os = this.opts.objectStorage;
|
|
2652
|
+
if (!os) {
|
|
2653
|
+
throw new StorageError(
|
|
2654
|
+
"tierToObjectStorage() requires objectStorage to be configured (new ZeroBucket({ objectStorage: new ObjectStorage({ bucket }) }))."
|
|
2655
|
+
);
|
|
2656
|
+
}
|
|
2657
|
+
if (!isValidId(id)) return null;
|
|
2658
|
+
const key = id.toLowerCase();
|
|
2659
|
+
try {
|
|
2660
|
+
return await this.run(
|
|
2661
|
+
"tier_to_object_storage",
|
|
2662
|
+
connection,
|
|
2663
|
+
async (q) => {
|
|
2664
|
+
const row = (await q.query(SELECT_FOR_TIERING3, [id]))[0][0];
|
|
2665
|
+
if (!row) return null;
|
|
2666
|
+
if (row.storage_backend !== LOCAL) return false;
|
|
2667
|
+
await os.upload(key, row.data, { mimeType: row.mime_type });
|
|
2668
|
+
await q.query(UPDATE_AFTER_TIERING3, [os.bucket, key, mysqlNow(), id]);
|
|
2669
|
+
return true;
|
|
2670
|
+
},
|
|
2671
|
+
{ tx: true }
|
|
2672
|
+
);
|
|
2673
|
+
} catch (exc) {
|
|
2674
|
+
throw wrap("Failed to tier image to object storage", exc);
|
|
2675
|
+
}
|
|
2676
|
+
}
|
|
2677
|
+
};
|
|
2678
|
+
function rowParams2(id, r, now) {
|
|
2679
|
+
return [
|
|
2680
|
+
id,
|
|
2681
|
+
r.data,
|
|
2682
|
+
r.mimeType,
|
|
2683
|
+
r.originalFilename,
|
|
2684
|
+
r.sizeBytes,
|
|
2685
|
+
r.width,
|
|
2686
|
+
r.height,
|
|
2687
|
+
r.checksumSha256,
|
|
2688
|
+
now,
|
|
2689
|
+
now
|
|
2690
|
+
];
|
|
2691
|
+
}
|
|
2692
|
+
|
|
1987
2693
|
// src/heic.ts
|
|
1988
2694
|
var import_node_module = require("module");
|
|
1989
2695
|
var import_node_path = require("path");
|
|
@@ -2372,9 +3078,14 @@ var ZeroBucket = class _ZeroBucket {
|
|
|
2372
3078
|
beforeGet;
|
|
2373
3079
|
beforePut;
|
|
2374
3080
|
constructor(options = {}) {
|
|
2375
|
-
|
|
3081
|
+
const chosen = [
|
|
3082
|
+
options.sqlite !== void 0 && "`sqlite`",
|
|
3083
|
+
(options.mysql !== void 0 || options.mysqlPool) && "`mysql`/`mysqlPool`",
|
|
3084
|
+
(options.connectionString || options.pool) && "`connectionString`/`pool`"
|
|
3085
|
+
].filter(Boolean);
|
|
3086
|
+
if (chosen.length > 1) {
|
|
2376
3087
|
throw new TypeError(
|
|
2377
|
-
|
|
3088
|
+
`Choose one database: ${chosen.join(" OR ")}, not several.`
|
|
2378
3089
|
);
|
|
2379
3090
|
}
|
|
2380
3091
|
this.backend = options.backend ?? (options.sqlite !== void 0 ? new SQLiteBackend({
|
|
@@ -2383,6 +3094,18 @@ var ZeroBucket = class _ZeroBucket {
|
|
|
2383
3094
|
dedup: options.dedup,
|
|
2384
3095
|
objectStorage: options.objectStorage,
|
|
2385
3096
|
onOperation: options.onOperation
|
|
3097
|
+
}) : options.mysql !== void 0 || options.mysqlPool ? new MySQLBackend({
|
|
3098
|
+
connectionString: options.mysql,
|
|
3099
|
+
pool: options.mysqlPool,
|
|
3100
|
+
autoMigrate: options.autoMigrate,
|
|
3101
|
+
maxRetries: options.maxRetries,
|
|
3102
|
+
retryBaseDelayMs: options.retryBaseDelayMs,
|
|
3103
|
+
poolMinSize: options.poolMinSize,
|
|
3104
|
+
poolMaxSize: options.poolMaxSize,
|
|
3105
|
+
poolTimeoutMs: options.poolTimeoutMs,
|
|
3106
|
+
onOperation: options.onOperation,
|
|
3107
|
+
objectStorage: options.objectStorage,
|
|
3108
|
+
dedup: options.dedup
|
|
2386
3109
|
}) : new PostgresBackend({
|
|
2387
3110
|
connectionString: options.connectionString,
|
|
2388
3111
|
pool: options.pool,
|
|
@@ -2479,7 +3202,7 @@ var ZeroBucket = class _ZeroBucket {
|
|
|
2479
3202
|
sizeBytes,
|
|
2480
3203
|
width,
|
|
2481
3204
|
height,
|
|
2482
|
-
checksumSha256: (0,
|
|
3205
|
+
checksumSha256: (0, import_node_crypto3.createHash)("sha256").update(finalData).digest("hex")
|
|
2483
3206
|
};
|
|
2484
3207
|
}
|
|
2485
3208
|
/** Validate, optionally optimize, checksum and store one image. Returns its id (a UUID). */
|
|
@@ -2984,6 +3707,7 @@ var PDFValidator = class {
|
|
|
2984
3707
|
ImageNotFoundError,
|
|
2985
3708
|
ImageTooLargeError,
|
|
2986
3709
|
ImageValidationError,
|
|
3710
|
+
MySQLBackend,
|
|
2987
3711
|
ObjectStorage,
|
|
2988
3712
|
PDFValidator,
|
|
2989
3713
|
PostgresBackend,
|