zerobucket 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -35
- package/dist/bin.js +1010 -17
- package/dist/{client-XQ22b1l3.d.cts → client-l4jPIFJI.d.cts} +21 -8
- package/dist/{client-XQ22b1l3.d.ts → client-l4jPIFJI.d.ts} +21 -8
- package/dist/http.d.cts +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.cjs +294 -24
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +28 -14
- package/dist/index.d.ts +28 -14
- package/dist/index.js +294 -24
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -875,13 +875,13 @@ var PostgresBackend = class {
|
|
|
875
875
|
* dedup instance's tables, preserving every id (external references keep working) and
|
|
876
876
|
* deduplicating identical content on the way. The classic table is never modified.
|
|
877
877
|
*
|
|
878
|
-
*
|
|
879
|
-
*
|
|
880
|
-
*
|
|
881
|
-
*
|
|
878
|
+
* Rows are copied in small batches (bounded memory instead of loading the whole table);
|
|
879
|
+
* re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
|
|
880
|
+
* already tiered to object storage are refused up front with a clear message (their bytes are
|
|
881
|
+
* not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
|
|
882
882
|
* Still all-or-nothing: one transaction.
|
|
883
883
|
*/
|
|
884
|
-
async migrateClassicToDedup() {
|
|
884
|
+
async migrateClassicToDedup(opts = {}) {
|
|
885
885
|
if (!this.dedup) {
|
|
886
886
|
throw new StorageError(
|
|
887
887
|
"migrateClassicToDedup() must be called on an instance created with dedup: true."
|
|
@@ -900,6 +900,7 @@ var PostgresBackend = class {
|
|
|
900
900
|
"No classic zerobucket_images table found in this database -- nothing to migrate."
|
|
901
901
|
);
|
|
902
902
|
}
|
|
903
|
+
let tieredSkipped = 0;
|
|
903
904
|
const hasTierCol = (await client.query(
|
|
904
905
|
"SELECT 1 FROM information_schema.columns WHERE table_name = 'zerobucket_images' AND column_name = 'storage_backend'"
|
|
905
906
|
)).rowCount;
|
|
@@ -907,12 +908,14 @@ var PostgresBackend = class {
|
|
|
907
908
|
const tiered = (await client.query(
|
|
908
909
|
"SELECT count(*)::int AS n FROM zerobucket_images WHERE storage_backend <> 'postgres'"
|
|
909
910
|
)).rows[0].n;
|
|
910
|
-
if (tiered > 0) {
|
|
911
|
+
if (tiered > 0 && !opts.skipTiered) {
|
|
911
912
|
throw new StorageError(
|
|
912
|
-
`${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table,
|
|
913
|
+
`${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table, bring those images back first, or pass skipTiered: true (CLI: --skip-tiered) to skip them and migrate the rest.`
|
|
913
914
|
);
|
|
914
915
|
}
|
|
916
|
+
tieredSkipped = tiered;
|
|
915
917
|
}
|
|
918
|
+
const onlyInDb = hasTierCol && tieredSkipped > 0;
|
|
916
919
|
const seen = /* @__PURE__ */ new Set();
|
|
917
920
|
let migrated = 0;
|
|
918
921
|
let already = 0;
|
|
@@ -921,11 +924,11 @@ var PostgresBackend = class {
|
|
|
921
924
|
for (; ; ) {
|
|
922
925
|
const page = cursor ? await client.query(
|
|
923
926
|
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
924
|
-
FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid) ORDER BY created_at, id LIMIT ${BATCH}`,
|
|
927
|
+
FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid)${onlyInDb ? " AND storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`,
|
|
925
928
|
[cursor.createdAt, cursor.id]
|
|
926
929
|
) : await client.query(
|
|
927
930
|
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
928
|
-
FROM zerobucket_images ORDER BY created_at, id LIMIT ${BATCH}`
|
|
931
|
+
FROM zerobucket_images${onlyInDb ? " WHERE storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
|
|
929
932
|
);
|
|
930
933
|
if (page.rows.length === 0) break;
|
|
931
934
|
const last = page.rows[page.rows.length - 1];
|
|
@@ -975,7 +978,8 @@ var PostgresBackend = class {
|
|
|
975
978
|
imagesMigrated: migrated,
|
|
976
979
|
distinctBlobsCreated: seen.size,
|
|
977
980
|
duplicateReferencesFound: migrated - seen.size,
|
|
978
|
-
alreadyMigrated: already
|
|
981
|
+
alreadyMigrated: already,
|
|
982
|
+
tieredSkipped
|
|
979
983
|
};
|
|
980
984
|
} catch (exc) {
|
|
981
985
|
try {
|
|
@@ -1909,11 +1913,60 @@ var CHUNK2 = `SELECT SUBSTRING(data FROM ? FOR ?) AS chunk FROM zerobucket_image
|
|
|
1909
1913
|
var EXISTS3 = `SELECT 1 FROM zerobucket_images WHERE id = ?`;
|
|
1910
1914
|
var LOCK_ONE = `SELECT storage_backend, object_storage_key FROM zerobucket_images WHERE id = ? FOR UPDATE`;
|
|
1911
1915
|
var DELETE_ONE2 = `DELETE FROM zerobucket_images WHERE id = ?`;
|
|
1912
|
-
var LOCK_MANY = `SELECT id, storage_backend, object_storage_key FROM zerobucket_images WHERE id IN (?) ORDER BY id FOR UPDATE`;
|
|
1913
|
-
var DELETE_MANY2 = `DELETE FROM zerobucket_images WHERE id IN (?)`;
|
|
1916
|
+
var LOCK_MANY = `SELECT id, storage_backend, object_storage_key FROM zerobucket_images FORCE INDEX (PRIMARY) WHERE id IN (?) ORDER BY id FOR UPDATE`;
|
|
1914
1917
|
var SELECT_FOR_TIERING3 = `SELECT data, mime_type, size_bytes, storage_backend FROM zerobucket_images WHERE id = ? FOR UPDATE`;
|
|
1915
1918
|
var UPDATE_AFTER_TIERING3 = `UPDATE zerobucket_images SET data = NULL, storage_backend = 'object_storage', object_storage_bucket = ?, object_storage_key = ?, updated_at = ? WHERE id = ?`;
|
|
1916
1919
|
var LOCAL = "mysql";
|
|
1920
|
+
var DEDUP_BLOBS_SCHEMA = `
|
|
1921
|
+
CREATE TABLE IF NOT EXISTS zerobucket_blobs (
|
|
1922
|
+
checksum_sha256 CHAR(64) PRIMARY KEY,
|
|
1923
|
+
data LONGBLOB NOT NULL,
|
|
1924
|
+
mime_type VARCHAR(255) NOT NULL,
|
|
1925
|
+
size_bytes INT NOT NULL,
|
|
1926
|
+
width INT,
|
|
1927
|
+
height INT,
|
|
1928
|
+
ref_count INT NOT NULL DEFAULT 0,
|
|
1929
|
+
created_at DATETIME(6) NOT NULL
|
|
1930
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
|
|
1931
|
+
var DEDUP_REFS_SCHEMA = `
|
|
1932
|
+
CREATE TABLE IF NOT EXISTS zerobucket_image_refs (
|
|
1933
|
+
id CHAR(36) PRIMARY KEY,
|
|
1934
|
+
checksum_sha256 CHAR(64) NOT NULL,
|
|
1935
|
+
original_filename TEXT,
|
|
1936
|
+
created_at DATETIME(6) NOT NULL,
|
|
1937
|
+
updated_at DATETIME(6) NOT NULL,
|
|
1938
|
+
INDEX idx_zerobucket_image_refs_checksum (checksum_sha256),
|
|
1939
|
+
INDEX idx_zerobucket_image_refs_created_at (created_at),
|
|
1940
|
+
FOREIGN KEY (checksum_sha256) REFERENCES zerobucket_blobs(checksum_sha256)
|
|
1941
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
|
|
1942
|
+
function upsertBlobSql(n) {
|
|
1943
|
+
if (!Number.isInteger(n) || n < 1) throw new RangeError("invalid ref count");
|
|
1944
|
+
return `INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, width, height, ref_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ${n}, ?) ON DUPLICATE KEY UPDATE ref_count = ref_count + ${n}`;
|
|
1945
|
+
}
|
|
1946
|
+
var D_INSERT_REF3 = `INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at, updated_at) VALUES (?, ?, ?, ?, ?)`;
|
|
1947
|
+
var D_COLS3 = `r.id, b.data, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256, '${LOCAL}' AS storage_backend, NULL AS object_storage_bucket, NULL AS object_storage_key`;
|
|
1948
|
+
var D_FROM3 = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
|
|
1949
|
+
var D_LOCK_REF = `SELECT checksum_sha256 FROM zerobucket_image_refs WHERE id = ? FOR UPDATE`;
|
|
1950
|
+
var D_DELETE_REF3 = `DELETE FROM zerobucket_image_refs WHERE id = ?`;
|
|
1951
|
+
var D_LOCK_REFS = `SELECT id, checksum_sha256 FROM zerobucket_image_refs FORCE INDEX (PRIMARY) WHERE id IN (?) ORDER BY id FOR UPDATE`;
|
|
1952
|
+
var D_DECREMENT3 = `UPDATE zerobucket_blobs SET ref_count = ref_count - ? WHERE checksum_sha256 = ?`;
|
|
1953
|
+
var D_DELETE_EMPTY3 = `DELETE FROM zerobucket_blobs WHERE checksum_sha256 = ? AND ref_count <= 0`;
|
|
1954
|
+
var CLASSIC_SQL2 = {
|
|
1955
|
+
selectFull: SELECT_FULL3,
|
|
1956
|
+
selectMany: SELECT_MANY2,
|
|
1957
|
+
selectMetadata: SELECT_METADATA3,
|
|
1958
|
+
streamInfo: STREAM_INFO2,
|
|
1959
|
+
chunk: CHUNK2,
|
|
1960
|
+
exists: EXISTS3
|
|
1961
|
+
};
|
|
1962
|
+
var DEDUP_SQL2 = {
|
|
1963
|
+
selectFull: `SELECT ${D_COLS3} ${D_FROM3} WHERE r.id = ?`,
|
|
1964
|
+
selectMany: `SELECT ${D_COLS3} ${D_FROM3} WHERE r.id IN (?)`,
|
|
1965
|
+
selectMetadata: `SELECT r.id, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256 ${D_FROM3} WHERE r.id = ?`,
|
|
1966
|
+
streamInfo: `SELECT b.size_bytes, '${LOCAL}' AS storage_backend, NULL AS object_storage_key ${D_FROM3} WHERE r.id = ?`,
|
|
1967
|
+
chunk: `SELECT SUBSTRING(b.data FROM ? FOR ?) AS chunk ${D_FROM3} WHERE r.id = ?`,
|
|
1968
|
+
exists: `SELECT 1 FROM zerobucket_image_refs WHERE id = ?`
|
|
1969
|
+
};
|
|
1917
1970
|
var RETRYABLE_ERRNOS = /* @__PURE__ */ new Set([
|
|
1918
1971
|
1205,
|
|
1919
1972
|
// ER_LOCK_WAIT_TIMEOUT
|
|
@@ -2003,11 +2056,13 @@ var MySQLBackend = class {
|
|
|
2003
2056
|
"MySQLBackend requires either `connectionString` or `pool`"
|
|
2004
2057
|
);
|
|
2005
2058
|
}
|
|
2006
|
-
|
|
2059
|
+
this.dedup = opts.dedup ?? false;
|
|
2060
|
+
if (this.dedup && opts.objectStorage) {
|
|
2007
2061
|
throw new TypeError(
|
|
2008
|
-
"
|
|
2062
|
+
"objectStorage is not supported together with dedup: true (same restriction as the Python package). Tiering moves one image's bytes, but a deduplicated blob can be shared by many images."
|
|
2009
2063
|
);
|
|
2010
2064
|
}
|
|
2065
|
+
this.sql = this.dedup ? DEDUP_SQL2 : CLASSIC_SQL2;
|
|
2011
2066
|
this.ownsPool = !opts.pool;
|
|
2012
2067
|
this.pool = opts.pool;
|
|
2013
2068
|
this.maxRetries = opts.maxRetries ?? 3;
|
|
@@ -2024,6 +2079,8 @@ var MySQLBackend = class {
|
|
|
2024
2079
|
retryBaseDelayMs;
|
|
2025
2080
|
poolTimeoutMs;
|
|
2026
2081
|
autoMigrate;
|
|
2082
|
+
dedup;
|
|
2083
|
+
sql;
|
|
2027
2084
|
// ---- lifecycle -------------------------------------------------------
|
|
2028
2085
|
ready() {
|
|
2029
2086
|
if (this.closed)
|
|
@@ -2089,6 +2146,10 @@ var MySQLBackend = class {
|
|
|
2089
2146
|
);
|
|
2090
2147
|
}
|
|
2091
2148
|
try {
|
|
2149
|
+
if (this.dedup) {
|
|
2150
|
+
await this.migrateDedup(q);
|
|
2151
|
+
return;
|
|
2152
|
+
}
|
|
2092
2153
|
const tableExists = (await q.query(
|
|
2093
2154
|
"SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images'"
|
|
2094
2155
|
))[0].length > 0;
|
|
@@ -2130,6 +2191,16 @@ var MySQLBackend = class {
|
|
|
2130
2191
|
});
|
|
2131
2192
|
}
|
|
2132
2193
|
}
|
|
2194
|
+
/** Dedup tables, created only if missing (same no-needless-DDL rule as the classic table). */
|
|
2195
|
+
async migrateDedup(q) {
|
|
2196
|
+
const have = new Set(
|
|
2197
|
+
(await q.query(
|
|
2198
|
+
"SELECT TABLE_NAME AS t FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN ('zerobucket_blobs', 'zerobucket_image_refs')"
|
|
2199
|
+
))[0].map((r) => r.t)
|
|
2200
|
+
);
|
|
2201
|
+
if (!have.has("zerobucket_blobs")) await q.query(DEDUP_BLOBS_SCHEMA);
|
|
2202
|
+
if (!have.has("zerobucket_image_refs")) await q.query(DEDUP_REFS_SCHEMA);
|
|
2203
|
+
}
|
|
2133
2204
|
async close() {
|
|
2134
2205
|
this.closed = true;
|
|
2135
2206
|
const pool = this.pool;
|
|
@@ -2295,6 +2366,19 @@ var MySQLBackend = class {
|
|
|
2295
2366
|
async put(row, { connection } = {}) {
|
|
2296
2367
|
try {
|
|
2297
2368
|
const id = randomUUID2();
|
|
2369
|
+
if (this.dedup) {
|
|
2370
|
+
await this.run(
|
|
2371
|
+
"put",
|
|
2372
|
+
connection,
|
|
2373
|
+
async (q) => {
|
|
2374
|
+
const now = mysqlNow();
|
|
2375
|
+
await q.execute(upsertBlobSql(1), blobParams2(row, now));
|
|
2376
|
+
await q.execute(D_INSERT_REF3, refParams(id, row, now));
|
|
2377
|
+
},
|
|
2378
|
+
{ tx: true }
|
|
2379
|
+
);
|
|
2380
|
+
return id;
|
|
2381
|
+
}
|
|
2298
2382
|
await this.run("put", connection, async (q) => {
|
|
2299
2383
|
await q.execute(INSERT3, rowParams2(id, row, mysqlNow()));
|
|
2300
2384
|
});
|
|
@@ -2312,6 +2396,21 @@ var MySQLBackend = class {
|
|
|
2312
2396
|
connection,
|
|
2313
2397
|
async (q) => {
|
|
2314
2398
|
const now = mysqlNow();
|
|
2399
|
+
if (this.dedup) {
|
|
2400
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2401
|
+
for (const r of rows) {
|
|
2402
|
+
const g = groups.get(r.checksumSha256);
|
|
2403
|
+
if (g) g.n += 1;
|
|
2404
|
+
else groups.set(r.checksumSha256, { row: r, n: 1 });
|
|
2405
|
+
}
|
|
2406
|
+
for (const checksum of [...groups.keys()].sort()) {
|
|
2407
|
+
const g = groups.get(checksum);
|
|
2408
|
+
await q.execute(upsertBlobSql(g.n), blobParams2(g.row, now));
|
|
2409
|
+
}
|
|
2410
|
+
for (let i = 0; i < rows.length; i++)
|
|
2411
|
+
await q.execute(D_INSERT_REF3, refParams(ids[i], rows[i], now));
|
|
2412
|
+
return;
|
|
2413
|
+
}
|
|
2315
2414
|
for (let i = 0; i < rows.length; i++)
|
|
2316
2415
|
await q.execute(INSERT3, rowParams2(ids[i], rows[i], now));
|
|
2317
2416
|
},
|
|
@@ -2329,7 +2428,7 @@ var MySQLBackend = class {
|
|
|
2329
2428
|
row = await this.run(
|
|
2330
2429
|
"get",
|
|
2331
2430
|
connection,
|
|
2332
|
-
async (q) => (await q.query(
|
|
2431
|
+
async (q) => (await q.query(this.sql.selectFull, [id]))[0][0]
|
|
2333
2432
|
);
|
|
2334
2433
|
} catch (exc) {
|
|
2335
2434
|
throw wrap("Failed to retrieve image", exc);
|
|
@@ -2344,7 +2443,7 @@ var MySQLBackend = class {
|
|
|
2344
2443
|
rows = await this.run(
|
|
2345
2444
|
"get_many",
|
|
2346
2445
|
connection,
|
|
2347
|
-
async (q) => (await q.query(
|
|
2446
|
+
async (q) => (await q.query(this.sql.selectMany, [valid]))[0]
|
|
2348
2447
|
);
|
|
2349
2448
|
} catch (exc) {
|
|
2350
2449
|
throw wrap("Failed to retrieve image batch", exc);
|
|
@@ -2358,7 +2457,7 @@ var MySQLBackend = class {
|
|
|
2358
2457
|
row = await this.run(
|
|
2359
2458
|
"get_metadata",
|
|
2360
2459
|
connection,
|
|
2361
|
-
async (q) => (await q.query(
|
|
2460
|
+
async (q) => (await q.query(this.sql.selectMetadata, [id]))[0][0]
|
|
2362
2461
|
);
|
|
2363
2462
|
} catch (exc) {
|
|
2364
2463
|
throw wrap("Failed to retrieve image metadata", exc);
|
|
@@ -2384,7 +2483,7 @@ var MySQLBackend = class {
|
|
|
2384
2483
|
info = await this.run(
|
|
2385
2484
|
"get_metadata",
|
|
2386
2485
|
connection,
|
|
2387
|
-
async (q) => (await q.query(
|
|
2486
|
+
async (q) => (await q.query(this.sql.streamInfo, [id]))[0][0]
|
|
2388
2487
|
);
|
|
2389
2488
|
} catch (exc) {
|
|
2390
2489
|
throw wrap("Failed to retrieve image metadata", exc);
|
|
@@ -2417,7 +2516,7 @@ var MySQLBackend = class {
|
|
|
2417
2516
|
row = await self.run(
|
|
2418
2517
|
"get_stream",
|
|
2419
2518
|
connection,
|
|
2420
|
-
async (q) => (await q.query(
|
|
2519
|
+
async (q) => (await q.query(self.sql.chunk, [offset + 1, length, id]))[0][0]
|
|
2421
2520
|
);
|
|
2422
2521
|
} catch (exc) {
|
|
2423
2522
|
throw wrap("Failed to stream image", exc);
|
|
@@ -2442,6 +2541,7 @@ var MySQLBackend = class {
|
|
|
2442
2541
|
}
|
|
2443
2542
|
async delete(id, { connection } = {}) {
|
|
2444
2543
|
if (!isValidId(id)) return false;
|
|
2544
|
+
if (this.dedup) return this.deleteDedup(id, connection);
|
|
2445
2545
|
try {
|
|
2446
2546
|
const row = await this.run(
|
|
2447
2547
|
"delete",
|
|
@@ -2464,14 +2564,14 @@ var MySQLBackend = class {
|
|
|
2464
2564
|
async deleteMany(ids, { connection } = {}) {
|
|
2465
2565
|
const valid = [...new Set(ids.filter(isValidId))];
|
|
2466
2566
|
if (valid.length === 0) return [];
|
|
2567
|
+
if (this.dedup) return this.deleteManyDedup(valid, connection);
|
|
2467
2568
|
try {
|
|
2468
2569
|
const rows = await this.run(
|
|
2469
2570
|
"delete_many",
|
|
2470
2571
|
connection,
|
|
2471
2572
|
async (q) => {
|
|
2472
2573
|
const found = (await q.query(LOCK_MANY, [valid]))[0];
|
|
2473
|
-
|
|
2474
|
-
await q.query(DELETE_MANY2, [found.map((r) => r.id)]);
|
|
2574
|
+
for (const r of found) await q.query(DELETE_ONE2, [r.id]);
|
|
2475
2575
|
return found;
|
|
2476
2576
|
},
|
|
2477
2577
|
{ tx: true }
|
|
@@ -2491,12 +2591,163 @@ var MySQLBackend = class {
|
|
|
2491
2591
|
return await this.run(
|
|
2492
2592
|
"exists",
|
|
2493
2593
|
connection,
|
|
2494
|
-
async (q) => (await q.query(
|
|
2594
|
+
async (q) => (await q.query(this.sql.exists, [id]))[0].length > 0
|
|
2495
2595
|
);
|
|
2496
2596
|
} catch (exc) {
|
|
2497
2597
|
throw wrap("Failed to check image existence", exc);
|
|
2498
2598
|
}
|
|
2499
2599
|
}
|
|
2600
|
+
// ---- maintenance (CLI) -----------------------------------------------
|
|
2601
|
+
/** Storage statistics, or null if the tables do not exist yet (nothing is created). */
|
|
2602
|
+
async getInfo() {
|
|
2603
|
+
try {
|
|
2604
|
+
return await this.run("info", void 0, async (q) => {
|
|
2605
|
+
const need = this.dedup ? ["zerobucket_image_refs", "zerobucket_blobs"] : ["zerobucket_images"];
|
|
2606
|
+
const present = new Set(
|
|
2607
|
+
(await q.query(
|
|
2608
|
+
"SELECT TABLE_NAME AS t FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?)",
|
|
2609
|
+
[need]
|
|
2610
|
+
))[0].map((r) => r.t)
|
|
2611
|
+
);
|
|
2612
|
+
if (!need.every((t) => present.has(t))) return null;
|
|
2613
|
+
const j = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
|
|
2614
|
+
const size = this.dedup ? "b.size_bytes" : "size_bytes";
|
|
2615
|
+
const created = this.dedup ? "r.created_at" : "created_at";
|
|
2616
|
+
const mime = this.dedup ? "b.mime_type" : "mime_type";
|
|
2617
|
+
const tot = (await q.query(
|
|
2618
|
+
`SELECT COUNT(*) AS n, COALESCE(SUM(${size}), 0) AS bytes, CAST(MIN(${created}) AS CHAR) AS oldest, CAST(MAX(${created}) AS CHAR) AS newest ${j}`
|
|
2619
|
+
))[0][0];
|
|
2620
|
+
const disk = (await q.query(
|
|
2621
|
+
"SELECT COALESCE(SUM(DATA_LENGTH + INDEX_LENGTH), 0) AS s FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?)",
|
|
2622
|
+
[need]
|
|
2623
|
+
))[0][0];
|
|
2624
|
+
const fmt = (await q.query(
|
|
2625
|
+
`SELECT ${mime} AS m, COUNT(*) AS n, COALESCE(SUM(${size}), 0) AS bytes ${j} GROUP BY ${mime} ORDER BY COUNT(*) DESC, ${mime}`
|
|
2626
|
+
))[0];
|
|
2627
|
+
const utc = (v) => v ? `${v}+00:00` : null;
|
|
2628
|
+
const info = {
|
|
2629
|
+
mode: this.dedup ? "dedup" : "classic",
|
|
2630
|
+
count: Number(tot.n),
|
|
2631
|
+
totalBytes: Number(tot.bytes),
|
|
2632
|
+
oldest: utc(tot.oldest),
|
|
2633
|
+
newest: utc(tot.newest),
|
|
2634
|
+
onDisk: prettySize(Number(disk.s)),
|
|
2635
|
+
byFormat: fmt.map((r) => ({
|
|
2636
|
+
mimeType: r.m,
|
|
2637
|
+
count: Number(r.n),
|
|
2638
|
+
bytes: Number(r.bytes)
|
|
2639
|
+
}))
|
|
2640
|
+
};
|
|
2641
|
+
if (this.dedup) {
|
|
2642
|
+
const b = (await q.query(
|
|
2643
|
+
"SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes FROM zerobucket_blobs"
|
|
2644
|
+
))[0][0];
|
|
2645
|
+
info.dedup = { blobs: Number(b.n), storedBytes: Number(b.bytes) };
|
|
2646
|
+
} else {
|
|
2647
|
+
const t = (await q.query(
|
|
2648
|
+
"SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
|
|
2649
|
+
))[0][0];
|
|
2650
|
+
info.tiered = { count: Number(t.n), bytes: Number(t.bytes) };
|
|
2651
|
+
}
|
|
2652
|
+
return info;
|
|
2653
|
+
});
|
|
2654
|
+
} catch (exc) {
|
|
2655
|
+
throw wrap("Failed to read storage info", exc);
|
|
2656
|
+
}
|
|
2657
|
+
}
|
|
2658
|
+
/** Ids matching the filters: oldest first, or a random sample. Used by the CLI's verify and tier. */
|
|
2659
|
+
async listImages(o = {}) {
|
|
2660
|
+
const where = [];
|
|
2661
|
+
const params = [];
|
|
2662
|
+
const created = this.dedup ? "r.created_at" : "created_at";
|
|
2663
|
+
if (o.minSize !== void 0) {
|
|
2664
|
+
where.push(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= ?`);
|
|
2665
|
+
params.push(o.minSize);
|
|
2666
|
+
}
|
|
2667
|
+
if (o.olderThanDays !== void 0) {
|
|
2668
|
+
where.push(`${created} <= UTC_TIMESTAMP(6) - INTERVAL ? DAY`);
|
|
2669
|
+
params.push(o.olderThanDays);
|
|
2670
|
+
}
|
|
2671
|
+
if (o.onlyUntiered && !this.dedup)
|
|
2672
|
+
where.push(`storage_backend = '${LOCAL}'`);
|
|
2673
|
+
const from = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
|
|
2674
|
+
const idCol = this.dedup ? "r.id" : "id";
|
|
2675
|
+
const tierCol = this.dedup ? "0 AS tiered" : "(storage_backend = 'object_storage') AS tiered";
|
|
2676
|
+
const order = o.sample !== void 0 ? "ORDER BY RAND()" : `ORDER BY ${created}, ${idCol}`;
|
|
2677
|
+
const lim = o.sample ?? o.limit;
|
|
2678
|
+
let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
|
|
2679
|
+
if (lim !== void 0) {
|
|
2680
|
+
if (!Number.isInteger(lim) || lim < 0)
|
|
2681
|
+
throw new RangeError("limit/sample must be a non-negative integer");
|
|
2682
|
+
sql += ` LIMIT ${lim}`;
|
|
2683
|
+
}
|
|
2684
|
+
try {
|
|
2685
|
+
const rows = await this.run(
|
|
2686
|
+
"list_images",
|
|
2687
|
+
void 0,
|
|
2688
|
+
async (q) => (await q.query(sql, params))[0]
|
|
2689
|
+
);
|
|
2690
|
+
return rows.map((r) => ({
|
|
2691
|
+
id: String(r.id),
|
|
2692
|
+
tiered: Boolean(Number(r.tiered))
|
|
2693
|
+
}));
|
|
2694
|
+
} catch (exc) {
|
|
2695
|
+
throw wrap("Failed to list images", exc);
|
|
2696
|
+
}
|
|
2697
|
+
}
|
|
2698
|
+
// ---- dedup deletes ---------------------------------------------------
|
|
2699
|
+
/**
|
|
2700
|
+
* Lock the ref row, delete it, then decrement its blob (the UPDATE takes the blob row lock, so two
|
|
2701
|
+
* deletes of different refs to one blob serialize instead of racing) and remove the blob once nothing
|
|
2702
|
+
* references it. All in one transaction: a failure can never leave a ref without its count or the reverse.
|
|
2703
|
+
*/
|
|
2704
|
+
async deleteDedup(id, connection) {
|
|
2705
|
+
try {
|
|
2706
|
+
return await this.run(
|
|
2707
|
+
"delete",
|
|
2708
|
+
connection,
|
|
2709
|
+
async (q) => {
|
|
2710
|
+
const ref = (await q.query(D_LOCK_REF, [id]))[0][0];
|
|
2711
|
+
if (!ref) return false;
|
|
2712
|
+
const checksum = ref.checksum_sha256;
|
|
2713
|
+
await q.query(D_DELETE_REF3, [id]);
|
|
2714
|
+
await q.query(D_DECREMENT3, [1, checksum]);
|
|
2715
|
+
await q.query(D_DELETE_EMPTY3, [checksum]);
|
|
2716
|
+
return true;
|
|
2717
|
+
},
|
|
2718
|
+
{ tx: true }
|
|
2719
|
+
);
|
|
2720
|
+
} catch (exc) {
|
|
2721
|
+
throw wrap("Failed to delete image", exc);
|
|
2722
|
+
}
|
|
2723
|
+
}
|
|
2724
|
+
async deleteManyDedup(ids, connection) {
|
|
2725
|
+
try {
|
|
2726
|
+
return await this.run(
|
|
2727
|
+
"delete_many",
|
|
2728
|
+
connection,
|
|
2729
|
+
async (q) => {
|
|
2730
|
+
const found = (await q.query(D_LOCK_REFS, [ids]))[0];
|
|
2731
|
+
if (found.length === 0) return [];
|
|
2732
|
+
for (const r of found) await q.query(D_DELETE_REF3, [r.id]);
|
|
2733
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2734
|
+
for (const r of found)
|
|
2735
|
+
counts.set(
|
|
2736
|
+
r.checksum_sha256,
|
|
2737
|
+
(counts.get(r.checksum_sha256) ?? 0) + 1
|
|
2738
|
+
);
|
|
2739
|
+
for (const checksum of [...counts.keys()].sort()) {
|
|
2740
|
+
await q.query(D_DECREMENT3, [counts.get(checksum), checksum]);
|
|
2741
|
+
await q.query(D_DELETE_EMPTY3, [checksum]);
|
|
2742
|
+
}
|
|
2743
|
+
return found.map((r) => String(r.id));
|
|
2744
|
+
},
|
|
2745
|
+
{ tx: true }
|
|
2746
|
+
);
|
|
2747
|
+
} catch (exc) {
|
|
2748
|
+
throw wrap("Failed to delete image batch", exc);
|
|
2749
|
+
}
|
|
2750
|
+
}
|
|
2500
2751
|
// ---- helpers ---------------------------------------------------------
|
|
2501
2752
|
hydrate(row, id) {
|
|
2502
2753
|
return hydrateRow(row, id, this.opts.objectStorage);
|
|
@@ -2543,6 +2794,11 @@ var MySQLBackend = class {
|
|
|
2543
2794
|
* id) makes the re-upload an idempotent overwrite.
|
|
2544
2795
|
*/
|
|
2545
2796
|
async tierToObjectStorage(id, { connection } = {}) {
|
|
2797
|
+
if (this.dedup) {
|
|
2798
|
+
throw new StorageError(
|
|
2799
|
+
"tierToObjectStorage() is not supported in dedup mode (a blob may be shared by many images)."
|
|
2800
|
+
);
|
|
2801
|
+
}
|
|
2546
2802
|
const os = this.opts.objectStorage;
|
|
2547
2803
|
if (!os) {
|
|
2548
2804
|
throw new StorageError(
|
|
@@ -2570,6 +2826,20 @@ var MySQLBackend = class {
|
|
|
2570
2826
|
}
|
|
2571
2827
|
}
|
|
2572
2828
|
};
|
|
2829
|
+
function blobParams2(r, now) {
|
|
2830
|
+
return [
|
|
2831
|
+
r.checksumSha256,
|
|
2832
|
+
r.data,
|
|
2833
|
+
r.mimeType,
|
|
2834
|
+
r.sizeBytes,
|
|
2835
|
+
r.width,
|
|
2836
|
+
r.height,
|
|
2837
|
+
now
|
|
2838
|
+
];
|
|
2839
|
+
}
|
|
2840
|
+
function refParams(id, r, now) {
|
|
2841
|
+
return [id, r.checksumSha256, r.originalFilename, now, now];
|
|
2842
|
+
}
|
|
2573
2843
|
function rowParams2(id, r, now) {
|
|
2574
2844
|
return [
|
|
2575
2845
|
id,
|
|
@@ -3368,13 +3638,13 @@ var ZeroBucket = class _ZeroBucket {
|
|
|
3368
3638
|
* One-time, non-destructive copy of classic-mode images into a dedup instance's tables, keeping every
|
|
3369
3639
|
* id. Call it on an instance created with `dedup: true`. Safe to re-run. See PostgresBackend for details.
|
|
3370
3640
|
*/
|
|
3371
|
-
async migrateClassicToDedup() {
|
|
3641
|
+
async migrateClassicToDedup(opts) {
|
|
3372
3642
|
if (!this.backend.migrateClassicToDedup) {
|
|
3373
3643
|
throw new StorageError(
|
|
3374
3644
|
"This backend does not support migrateClassicToDedup()."
|
|
3375
3645
|
);
|
|
3376
3646
|
}
|
|
3377
|
-
return this.backend.migrateClassicToDedup();
|
|
3647
|
+
return this.backend.migrateClassicToDedup(opts);
|
|
3378
3648
|
}
|
|
3379
3649
|
/** Release the pool (if ZeroBucket created it). */
|
|
3380
3650
|
close() {
|