zerobucket 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -35
- package/dist/bin.js +1010 -17
- package/dist/{client-XQ22b1l3.d.cts → client-l4jPIFJI.d.cts} +21 -8
- package/dist/{client-XQ22b1l3.d.ts → client-l4jPIFJI.d.ts} +21 -8
- package/dist/http.d.cts +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.cjs +294 -24
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +28 -14
- package/dist/index.d.ts +28 -14
- package/dist/index.js +294 -24
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/bin.js
CHANGED
|
@@ -875,13 +875,13 @@ var PostgresBackend = class {
|
|
|
875
875
|
* dedup instance's tables, preserving every id (external references keep working) and
|
|
876
876
|
* deduplicating identical content on the way. The classic table is never modified.
|
|
877
877
|
*
|
|
878
|
-
*
|
|
879
|
-
*
|
|
880
|
-
*
|
|
881
|
-
*
|
|
878
|
+
* Rows are copied in small batches (bounded memory instead of loading the whole table);
|
|
879
|
+
* re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
|
|
880
|
+
* already tiered to object storage are refused up front with a clear message (their bytes are
|
|
881
|
+
* not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
|
|
882
882
|
* Still all-or-nothing: one transaction.
|
|
883
883
|
*/
|
|
884
|
-
async migrateClassicToDedup() {
|
|
884
|
+
async migrateClassicToDedup(opts = {}) {
|
|
885
885
|
if (!this.dedup) {
|
|
886
886
|
throw new StorageError(
|
|
887
887
|
"migrateClassicToDedup() must be called on an instance created with dedup: true."
|
|
@@ -900,6 +900,7 @@ var PostgresBackend = class {
|
|
|
900
900
|
"No classic zerobucket_images table found in this database -- nothing to migrate."
|
|
901
901
|
);
|
|
902
902
|
}
|
|
903
|
+
let tieredSkipped = 0;
|
|
903
904
|
const hasTierCol = (await client.query(
|
|
904
905
|
"SELECT 1 FROM information_schema.columns WHERE table_name = 'zerobucket_images' AND column_name = 'storage_backend'"
|
|
905
906
|
)).rowCount;
|
|
@@ -907,12 +908,14 @@ var PostgresBackend = class {
|
|
|
907
908
|
const tiered = (await client.query(
|
|
908
909
|
"SELECT count(*)::int AS n FROM zerobucket_images WHERE storage_backend <> 'postgres'"
|
|
909
910
|
)).rows[0].n;
|
|
910
|
-
if (tiered > 0) {
|
|
911
|
+
if (tiered > 0 && !opts.skipTiered) {
|
|
911
912
|
throw new StorageError(
|
|
912
|
-
`${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table,
|
|
913
|
+
`${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table, bring those images back first, or pass skipTiered: true (CLI: --skip-tiered) to skip them and migrate the rest.`
|
|
913
914
|
);
|
|
914
915
|
}
|
|
916
|
+
tieredSkipped = tiered;
|
|
915
917
|
}
|
|
918
|
+
const onlyInDb = hasTierCol && tieredSkipped > 0;
|
|
916
919
|
const seen = /* @__PURE__ */ new Set();
|
|
917
920
|
let migrated = 0;
|
|
918
921
|
let already = 0;
|
|
@@ -921,11 +924,11 @@ var PostgresBackend = class {
|
|
|
921
924
|
for (; ; ) {
|
|
922
925
|
const page = cursor ? await client.query(
|
|
923
926
|
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
924
|
-
FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid) ORDER BY created_at, id LIMIT ${BATCH}`,
|
|
927
|
+
FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid)${onlyInDb ? " AND storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`,
|
|
925
928
|
[cursor.createdAt, cursor.id]
|
|
926
929
|
) : await client.query(
|
|
927
930
|
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
928
|
-
FROM zerobucket_images ORDER BY created_at, id LIMIT ${BATCH}`
|
|
931
|
+
FROM zerobucket_images${onlyInDb ? " WHERE storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
|
|
929
932
|
);
|
|
930
933
|
if (page.rows.length === 0) break;
|
|
931
934
|
const last = page.rows[page.rows.length - 1];
|
|
@@ -975,7 +978,8 @@ var PostgresBackend = class {
|
|
|
975
978
|
imagesMigrated: migrated,
|
|
976
979
|
distinctBlobsCreated: seen.size,
|
|
977
980
|
duplicateReferencesFound: migrated - seen.size,
|
|
978
|
-
alreadyMigrated: already
|
|
981
|
+
alreadyMigrated: already,
|
|
982
|
+
tieredSkipped
|
|
979
983
|
};
|
|
980
984
|
} catch (exc) {
|
|
981
985
|
try {
|
|
@@ -1880,6 +1884,977 @@ var SQLiteBackend = class {
|
|
|
1880
1884
|
}
|
|
1881
1885
|
};
|
|
1882
1886
|
|
|
1887
|
+
// src/adapters/mysql.ts
|
|
1888
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
1889
|
+
var SCHEMA3 = `
|
|
1890
|
+
CREATE TABLE IF NOT EXISTS zerobucket_images (
|
|
1891
|
+
id CHAR(36) PRIMARY KEY,
|
|
1892
|
+
data LONGBLOB NOT NULL,
|
|
1893
|
+
mime_type VARCHAR(255) NOT NULL,
|
|
1894
|
+
original_filename TEXT,
|
|
1895
|
+
size_bytes INT NOT NULL,
|
|
1896
|
+
width INT,
|
|
1897
|
+
height INT,
|
|
1898
|
+
checksum_sha256 CHAR(64) NOT NULL,
|
|
1899
|
+
created_at DATETIME(6) NOT NULL,
|
|
1900
|
+
updated_at DATETIME(6) NOT NULL,
|
|
1901
|
+
INDEX idx_zerobucket_checksum (checksum_sha256),
|
|
1902
|
+
INDEX idx_zerobucket_created_at (created_at)
|
|
1903
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
|
|
1904
|
+
var TIERING_CHECK = "chk_zerobucket_storage";
|
|
1905
|
+
var MIGRATE_LOCK = "zerobucket_migrate";
|
|
1906
|
+
var COLS2 = `id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, storage_backend, object_storage_bucket, object_storage_key`;
|
|
1907
|
+
var INSERT3 = `INSERT INTO zerobucket_images (id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
|
|
1908
|
+
var SELECT_FULL3 = `SELECT ${COLS2} FROM zerobucket_images WHERE id = ?`;
|
|
1909
|
+
var SELECT_MANY2 = `SELECT ${COLS2} FROM zerobucket_images WHERE id IN (?)`;
|
|
1910
|
+
var SELECT_METADATA3 = `SELECT id, mime_type, original_filename, size_bytes, width, height, checksum_sha256 FROM zerobucket_images WHERE id = ?`;
|
|
1911
|
+
var STREAM_INFO2 = `SELECT size_bytes, storage_backend, object_storage_key FROM zerobucket_images WHERE id = ?`;
|
|
1912
|
+
var CHUNK2 = `SELECT SUBSTRING(data FROM ? FOR ?) AS chunk FROM zerobucket_images WHERE id = ?`;
|
|
1913
|
+
var EXISTS3 = `SELECT 1 FROM zerobucket_images WHERE id = ?`;
|
|
1914
|
+
var LOCK_ONE = `SELECT storage_backend, object_storage_key FROM zerobucket_images WHERE id = ? FOR UPDATE`;
|
|
1915
|
+
var DELETE_ONE2 = `DELETE FROM zerobucket_images WHERE id = ?`;
|
|
1916
|
+
var LOCK_MANY = `SELECT id, storage_backend, object_storage_key FROM zerobucket_images FORCE INDEX (PRIMARY) WHERE id IN (?) ORDER BY id FOR UPDATE`;
|
|
1917
|
+
var SELECT_FOR_TIERING3 = `SELECT data, mime_type, size_bytes, storage_backend FROM zerobucket_images WHERE id = ? FOR UPDATE`;
|
|
1918
|
+
var UPDATE_AFTER_TIERING3 = `UPDATE zerobucket_images SET data = NULL, storage_backend = 'object_storage', object_storage_bucket = ?, object_storage_key = ?, updated_at = ? WHERE id = ?`;
|
|
1919
|
+
var LOCAL = "mysql";
|
|
1920
|
+
var DEDUP_BLOBS_SCHEMA = `
|
|
1921
|
+
CREATE TABLE IF NOT EXISTS zerobucket_blobs (
|
|
1922
|
+
checksum_sha256 CHAR(64) PRIMARY KEY,
|
|
1923
|
+
data LONGBLOB NOT NULL,
|
|
1924
|
+
mime_type VARCHAR(255) NOT NULL,
|
|
1925
|
+
size_bytes INT NOT NULL,
|
|
1926
|
+
width INT,
|
|
1927
|
+
height INT,
|
|
1928
|
+
ref_count INT NOT NULL DEFAULT 0,
|
|
1929
|
+
created_at DATETIME(6) NOT NULL
|
|
1930
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
|
|
1931
|
+
var DEDUP_REFS_SCHEMA = `
|
|
1932
|
+
CREATE TABLE IF NOT EXISTS zerobucket_image_refs (
|
|
1933
|
+
id CHAR(36) PRIMARY KEY,
|
|
1934
|
+
checksum_sha256 CHAR(64) NOT NULL,
|
|
1935
|
+
original_filename TEXT,
|
|
1936
|
+
created_at DATETIME(6) NOT NULL,
|
|
1937
|
+
updated_at DATETIME(6) NOT NULL,
|
|
1938
|
+
INDEX idx_zerobucket_image_refs_checksum (checksum_sha256),
|
|
1939
|
+
INDEX idx_zerobucket_image_refs_created_at (created_at),
|
|
1940
|
+
FOREIGN KEY (checksum_sha256) REFERENCES zerobucket_blobs(checksum_sha256)
|
|
1941
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
|
|
1942
|
+
function upsertBlobSql(n) {
|
|
1943
|
+
if (!Number.isInteger(n) || n < 1) throw new RangeError("invalid ref count");
|
|
1944
|
+
return `INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, width, height, ref_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ${n}, ?) ON DUPLICATE KEY UPDATE ref_count = ref_count + ${n}`;
|
|
1945
|
+
}
|
|
1946
|
+
var D_INSERT_REF3 = `INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at, updated_at) VALUES (?, ?, ?, ?, ?)`;
|
|
1947
|
+
var D_COLS3 = `r.id, b.data, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256, '${LOCAL}' AS storage_backend, NULL AS object_storage_bucket, NULL AS object_storage_key`;
|
|
1948
|
+
var D_FROM3 = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
|
|
1949
|
+
var D_LOCK_REF = `SELECT checksum_sha256 FROM zerobucket_image_refs WHERE id = ? FOR UPDATE`;
|
|
1950
|
+
var D_DELETE_REF3 = `DELETE FROM zerobucket_image_refs WHERE id = ?`;
|
|
1951
|
+
var D_LOCK_REFS = `SELECT id, checksum_sha256 FROM zerobucket_image_refs FORCE INDEX (PRIMARY) WHERE id IN (?) ORDER BY id FOR UPDATE`;
|
|
1952
|
+
var D_DECREMENT3 = `UPDATE zerobucket_blobs SET ref_count = ref_count - ? WHERE checksum_sha256 = ?`;
|
|
1953
|
+
var D_DELETE_EMPTY3 = `DELETE FROM zerobucket_blobs WHERE checksum_sha256 = ? AND ref_count <= 0`;
|
|
1954
|
+
var CLASSIC_SQL2 = {
|
|
1955
|
+
selectFull: SELECT_FULL3,
|
|
1956
|
+
selectMany: SELECT_MANY2,
|
|
1957
|
+
selectMetadata: SELECT_METADATA3,
|
|
1958
|
+
streamInfo: STREAM_INFO2,
|
|
1959
|
+
chunk: CHUNK2,
|
|
1960
|
+
exists: EXISTS3
|
|
1961
|
+
};
|
|
1962
|
+
var DEDUP_SQL2 = {
|
|
1963
|
+
selectFull: `SELECT ${D_COLS3} ${D_FROM3} WHERE r.id = ?`,
|
|
1964
|
+
selectMany: `SELECT ${D_COLS3} ${D_FROM3} WHERE r.id IN (?)`,
|
|
1965
|
+
selectMetadata: `SELECT r.id, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256 ${D_FROM3} WHERE r.id = ?`,
|
|
1966
|
+
streamInfo: `SELECT b.size_bytes, '${LOCAL}' AS storage_backend, NULL AS object_storage_key ${D_FROM3} WHERE r.id = ?`,
|
|
1967
|
+
chunk: `SELECT SUBSTRING(b.data FROM ? FOR ?) AS chunk ${D_FROM3} WHERE r.id = ?`,
|
|
1968
|
+
exists: `SELECT 1 FROM zerobucket_image_refs WHERE id = ?`
|
|
1969
|
+
};
|
|
1970
|
+
var RETRYABLE_ERRNOS = /* @__PURE__ */ new Set([
|
|
1971
|
+
1205,
|
|
1972
|
+
// ER_LOCK_WAIT_TIMEOUT
|
|
1973
|
+
1213,
|
|
1974
|
+
// ER_LOCK_DEADLOCK
|
|
1975
|
+
1040,
|
|
1976
|
+
// ER_CON_COUNT_ERROR (too many connections)
|
|
1977
|
+
1053,
|
|
1978
|
+
// ER_SERVER_SHUTDOWN
|
|
1979
|
+
1927,
|
|
1980
|
+
// ER_CONNECTION_KILLED (MariaDB)
|
|
1981
|
+
2002,
|
|
1982
|
+
// CR_CONNECTION_ERROR
|
|
1983
|
+
2003,
|
|
1984
|
+
// CR_CONN_HOST_ERROR
|
|
1985
|
+
2006,
|
|
1986
|
+
// CR_SERVER_GONE_ERROR
|
|
1987
|
+
2013
|
|
1988
|
+
// CR_SERVER_LOST
|
|
1989
|
+
]);
|
|
1990
|
+
var CONNECTION_CODES = /* @__PURE__ */ new Set([
|
|
1991
|
+
"PROTOCOL_CONNECTION_LOST",
|
|
1992
|
+
"PROTOCOL_ENQUEUE_AFTER_FATAL_ERROR",
|
|
1993
|
+
"PROTOCOL_ENQUEUE_AFTER_QUIT",
|
|
1994
|
+
"ECONNRESET",
|
|
1995
|
+
"ECONNREFUSED",
|
|
1996
|
+
"ETIMEDOUT",
|
|
1997
|
+
"EPIPE",
|
|
1998
|
+
"ECONNABORTED"
|
|
1999
|
+
]);
|
|
2000
|
+
function errnoOf(exc) {
|
|
2001
|
+
const n = exc?.errno;
|
|
2002
|
+
return typeof n === "number" ? n : void 0;
|
|
2003
|
+
}
|
|
2004
|
+
var PoolTimeoutError = class extends StorageError {
|
|
2005
|
+
};
|
|
2006
|
+
function isRetryable2(exc) {
|
|
2007
|
+
if (!(exc instanceof Error) || exc instanceof PoolTimeoutError) return false;
|
|
2008
|
+
const e = exc;
|
|
2009
|
+
const n = errnoOf(exc);
|
|
2010
|
+
if (n !== void 0 && RETRYABLE_ERRNOS.has(n)) return true;
|
|
2011
|
+
if (typeof e.code === "string") {
|
|
2012
|
+
if (CONNECTION_CODES.has(e.code)) return true;
|
|
2013
|
+
if (e.code === "ER_LOCK_WAIT_TIMEOUT" || e.code === "ER_LOCK_DEADLOCK" || e.code === "ER_CON_COUNT_ERROR" || e.code === "ER_SERVER_SHUTDOWN" || e.code === "ER_CONNECTION_KILLED")
|
|
2014
|
+
return true;
|
|
2015
|
+
}
|
|
2016
|
+
return false;
|
|
2017
|
+
}
|
|
2018
|
+
function isConnectionLevel2(exc) {
|
|
2019
|
+
if (!(exc instanceof Error)) return false;
|
|
2020
|
+
const e = exc;
|
|
2021
|
+
if (e.fatal === true) return true;
|
|
2022
|
+
if (typeof e.code === "string" && CONNECTION_CODES.has(e.code)) return true;
|
|
2023
|
+
const n = errnoOf(exc);
|
|
2024
|
+
return n === 2006 || n === 2013 || n === 1927 || n === 1053;
|
|
2025
|
+
}
|
|
2026
|
+
var MAX_BACKOFF_MS2 = 2e3;
|
|
2027
|
+
function backoffDelayMs2(attempt, baseDelayMs) {
|
|
2028
|
+
const exponential = baseDelayMs * 2 ** (attempt - 1);
|
|
2029
|
+
const jitter = Math.random() * baseDelayMs;
|
|
2030
|
+
return Math.min(exponential + jitter, MAX_BACKOFF_MS2);
|
|
2031
|
+
}
|
|
2032
|
+
function friendly(prefix, exc) {
|
|
2033
|
+
if (errnoOf(exc) === 1153) {
|
|
2034
|
+
return new StorageError(
|
|
2035
|
+
`${prefix}: the image is larger than the server's max_allowed_packet. Raise it (for example \`SET GLOBAL max_allowed_packet = 67108864\`, and set it in my.cnf to make it permanent), or lower \`maxBytes\`.`,
|
|
2036
|
+
{ cause: exc }
|
|
2037
|
+
);
|
|
2038
|
+
}
|
|
2039
|
+
return wrap(prefix, exc);
|
|
2040
|
+
}
|
|
2041
|
+
function mysqlNow() {
|
|
2042
|
+
return `${(/* @__PURE__ */ new Date()).toISOString().replace("T", " ").replace("Z", "")}000`;
|
|
2043
|
+
}
|
|
2044
|
+
function normalizeUrl(url) {
|
|
2045
|
+
return url.replace(/^mariadb:\/\//i, "mysql://");
|
|
2046
|
+
}
|
|
2047
|
+
function isMysqlConnection(c) {
|
|
2048
|
+
const m = c;
|
|
2049
|
+
return typeof m.query === "function" && typeof m.execute === "function";
|
|
2050
|
+
}
|
|
2051
|
+
var MySQLBackend = class {
|
|
2052
|
+
constructor(opts) {
|
|
2053
|
+
this.opts = opts;
|
|
2054
|
+
if (!opts.connectionString && !opts.pool) {
|
|
2055
|
+
throw new TypeError(
|
|
2056
|
+
"MySQLBackend requires either `connectionString` or `pool`"
|
|
2057
|
+
);
|
|
2058
|
+
}
|
|
2059
|
+
this.dedup = opts.dedup ?? false;
|
|
2060
|
+
if (this.dedup && opts.objectStorage) {
|
|
2061
|
+
throw new TypeError(
|
|
2062
|
+
"objectStorage is not supported together with dedup: true (same restriction as the Python package). Tiering moves one image's bytes, but a deduplicated blob can be shared by many images."
|
|
2063
|
+
);
|
|
2064
|
+
}
|
|
2065
|
+
this.sql = this.dedup ? DEDUP_SQL2 : CLASSIC_SQL2;
|
|
2066
|
+
this.ownsPool = !opts.pool;
|
|
2067
|
+
this.pool = opts.pool;
|
|
2068
|
+
this.maxRetries = opts.maxRetries ?? 3;
|
|
2069
|
+
this.retryBaseDelayMs = opts.retryBaseDelayMs ?? 100;
|
|
2070
|
+
this.poolTimeoutMs = opts.poolTimeoutMs ?? 1e4;
|
|
2071
|
+
this.autoMigrate = opts.autoMigrate ?? true;
|
|
2072
|
+
}
|
|
2073
|
+
opts;
|
|
2074
|
+
pool;
|
|
2075
|
+
ownsPool;
|
|
2076
|
+
readyPromise;
|
|
2077
|
+
closed = false;
|
|
2078
|
+
maxRetries;
|
|
2079
|
+
retryBaseDelayMs;
|
|
2080
|
+
poolTimeoutMs;
|
|
2081
|
+
autoMigrate;
|
|
2082
|
+
dedup;
|
|
2083
|
+
sql;
|
|
2084
|
+
// ---- lifecycle -------------------------------------------------------
|
|
2085
|
+
ready() {
|
|
2086
|
+
if (this.closed)
|
|
2087
|
+
return Promise.reject(
|
|
2088
|
+
new StorageError("This ZeroBucket instance has been closed")
|
|
2089
|
+
);
|
|
2090
|
+
this.readyPromise ??= this.init().catch((exc) => {
|
|
2091
|
+
this.readyPromise = void 0;
|
|
2092
|
+
throw exc;
|
|
2093
|
+
});
|
|
2094
|
+
return this.readyPromise;
|
|
2095
|
+
}
|
|
2096
|
+
async init() {
|
|
2097
|
+
if (!this.pool) {
|
|
2098
|
+
let mod;
|
|
2099
|
+
try {
|
|
2100
|
+
mod = await import("mysql2/promise");
|
|
2101
|
+
} catch (cause) {
|
|
2102
|
+
throw new StorageError(
|
|
2103
|
+
"MySQLBackend requires the 'mysql2' package. Install it with: npm install mysql2",
|
|
2104
|
+
{ cause }
|
|
2105
|
+
);
|
|
2106
|
+
}
|
|
2107
|
+
const createPool = (mod.default ?? mod).createPool;
|
|
2108
|
+
this.pool = createPool({
|
|
2109
|
+
uri: normalizeUrl(this.opts.connectionString),
|
|
2110
|
+
connectionLimit: this.opts.poolMaxSize ?? 5,
|
|
2111
|
+
waitForConnections: true,
|
|
2112
|
+
queueLimit: 0,
|
|
2113
|
+
connectTimeout: this.poolTimeoutMs,
|
|
2114
|
+
charset: "utf8mb4",
|
|
2115
|
+
timezone: "Z",
|
|
2116
|
+
enableKeepAlive: true
|
|
2117
|
+
});
|
|
2118
|
+
}
|
|
2119
|
+
try {
|
|
2120
|
+
if (this.autoMigrate) {
|
|
2121
|
+
await this.runPooled("migrate", (q) => this.migrate(q), false);
|
|
2122
|
+
} else {
|
|
2123
|
+
await this.runPooled("ping", (q) => q.query("SELECT 1"), false);
|
|
2124
|
+
}
|
|
2125
|
+
} catch (exc) {
|
|
2126
|
+
if (this.ownsPool) {
|
|
2127
|
+
await this.pool.end().catch(() => {
|
|
2128
|
+
});
|
|
2129
|
+
this.pool = void 0;
|
|
2130
|
+
}
|
|
2131
|
+
throw exc instanceof StorageError ? exc : new StorageError(`Could not connect to MySQL/MariaDB: ${msg(exc)}`, {
|
|
2132
|
+
cause: exc
|
|
2133
|
+
});
|
|
2134
|
+
}
|
|
2135
|
+
}
|
|
2136
|
+
/**
|
|
2137
|
+
* Create the schema if missing, under a server-side named lock so concurrent first callers (in this
|
|
2138
|
+
* or any other process) run the DDL one at a time. Every step checks before it changes anything, so
|
|
2139
|
+
* re-running is a no-op.
|
|
2140
|
+
*/
|
|
2141
|
+
async migrate(q) {
|
|
2142
|
+
const locked = (await q.query("SELECT GET_LOCK(?, 30) AS got", [MIGRATE_LOCK]))[0][0]?.got;
|
|
2143
|
+
if (locked !== 1) {
|
|
2144
|
+
throw new StorageError(
|
|
2145
|
+
"Timed out waiting for another process to finish migrating the ZeroBucket schema."
|
|
2146
|
+
);
|
|
2147
|
+
}
|
|
2148
|
+
try {
|
|
2149
|
+
if (this.dedup) {
|
|
2150
|
+
await this.migrateDedup(q);
|
|
2151
|
+
return;
|
|
2152
|
+
}
|
|
2153
|
+
const tableExists = (await q.query(
|
|
2154
|
+
"SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images'"
|
|
2155
|
+
))[0].length > 0;
|
|
2156
|
+
if (!tableExists) await q.query(SCHEMA3);
|
|
2157
|
+
const cols = new Map(
|
|
2158
|
+
(await q.query(
|
|
2159
|
+
"SELECT COLUMN_NAME AS c, IS_NULLABLE AS nullable FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images'"
|
|
2160
|
+
))[0].map((r) => [r.c, r.nullable])
|
|
2161
|
+
);
|
|
2162
|
+
if (!cols.has("storage_backend"))
|
|
2163
|
+
await q.query(
|
|
2164
|
+
`ALTER TABLE zerobucket_images ADD COLUMN storage_backend VARCHAR(20) NOT NULL DEFAULT '${LOCAL}'`
|
|
2165
|
+
);
|
|
2166
|
+
if (!cols.has("object_storage_bucket"))
|
|
2167
|
+
await q.query(
|
|
2168
|
+
"ALTER TABLE zerobucket_images ADD COLUMN object_storage_bucket VARCHAR(255)"
|
|
2169
|
+
);
|
|
2170
|
+
if (!cols.has("object_storage_key"))
|
|
2171
|
+
await q.query(
|
|
2172
|
+
"ALTER TABLE zerobucket_images ADD COLUMN object_storage_key VARCHAR(512)"
|
|
2173
|
+
);
|
|
2174
|
+
if (cols.get("data") === "NO")
|
|
2175
|
+
await q.query(
|
|
2176
|
+
"ALTER TABLE zerobucket_images MODIFY data LONGBLOB NULL"
|
|
2177
|
+
);
|
|
2178
|
+
const hasCheck = (await q.query(
|
|
2179
|
+
"SELECT 1 FROM information_schema.TABLE_CONSTRAINTS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images' AND CONSTRAINT_NAME = ?",
|
|
2180
|
+
[TIERING_CHECK]
|
|
2181
|
+
))[0].length > 0;
|
|
2182
|
+
if (!hasCheck) {
|
|
2183
|
+
await q.query(
|
|
2184
|
+
`ALTER TABLE zerobucket_images ADD CONSTRAINT ${TIERING_CHECK} CHECK ((storage_backend = '${LOCAL}' AND data IS NOT NULL AND object_storage_bucket IS NULL AND object_storage_key IS NULL) OR (storage_backend = 'object_storage' AND data IS NULL AND object_storage_bucket IS NOT NULL AND object_storage_key IS NOT NULL))`
|
|
2185
|
+
);
|
|
2186
|
+
}
|
|
2187
|
+
} catch (exc) {
|
|
2188
|
+
throw new StorageError(`Migration failed: ${msg(exc)}`, { cause: exc });
|
|
2189
|
+
} finally {
|
|
2190
|
+
await q.query("SELECT RELEASE_LOCK(?)", [MIGRATE_LOCK]).catch(() => {
|
|
2191
|
+
});
|
|
2192
|
+
}
|
|
2193
|
+
}
|
|
2194
|
+
/** Dedup tables, created only if missing (same no-needless-DDL rule as the classic table). */
|
|
2195
|
+
async migrateDedup(q) {
|
|
2196
|
+
const have = new Set(
|
|
2197
|
+
(await q.query(
|
|
2198
|
+
"SELECT TABLE_NAME AS t FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN ('zerobucket_blobs', 'zerobucket_image_refs')"
|
|
2199
|
+
))[0].map((r) => r.t)
|
|
2200
|
+
);
|
|
2201
|
+
if (!have.has("zerobucket_blobs")) await q.query(DEDUP_BLOBS_SCHEMA);
|
|
2202
|
+
if (!have.has("zerobucket_image_refs")) await q.query(DEDUP_REFS_SCHEMA);
|
|
2203
|
+
}
|
|
2204
|
+
async close() {
|
|
2205
|
+
this.closed = true;
|
|
2206
|
+
const pool = this.pool;
|
|
2207
|
+
this.pool = void 0;
|
|
2208
|
+
this.readyPromise = void 0;
|
|
2209
|
+
if (pool && this.ownsPool) await pool.end();
|
|
2210
|
+
}
|
|
2211
|
+
// ---- execution core --------------------------------------------------
|
|
2212
|
+
emit(operation, start, success, error, retryCount) {
|
|
2213
|
+
const cb = this.opts.onOperation;
|
|
2214
|
+
if (!cb) return;
|
|
2215
|
+
try {
|
|
2216
|
+
cb({
|
|
2217
|
+
operation,
|
|
2218
|
+
durationMs: performance.now() - start,
|
|
2219
|
+
success,
|
|
2220
|
+
error,
|
|
2221
|
+
retryCount
|
|
2222
|
+
});
|
|
2223
|
+
} catch {
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
/** Wait for a pool slot, but never longer than poolTimeoutMs. */
|
|
2227
|
+
acquire() {
|
|
2228
|
+
const pool = this.pool;
|
|
2229
|
+
if (this.closed || !pool)
|
|
2230
|
+
return Promise.reject(
|
|
2231
|
+
new StorageError("This ZeroBucket instance has been closed")
|
|
2232
|
+
);
|
|
2233
|
+
return new Promise((resolve, reject) => {
|
|
2234
|
+
let settled = false;
|
|
2235
|
+
const timer = setTimeout(() => {
|
|
2236
|
+
settled = true;
|
|
2237
|
+
reject(
|
|
2238
|
+
new PoolTimeoutError(
|
|
2239
|
+
`Timed out after ${this.poolTimeoutMs}ms waiting for a MySQL/MariaDB connection (poolMaxSize=${this.opts.poolMaxSize ?? 5}; all connections are in use).`
|
|
2240
|
+
)
|
|
2241
|
+
);
|
|
2242
|
+
}, this.poolTimeoutMs);
|
|
2243
|
+
pool.getConnection().then(
|
|
2244
|
+
(conn) => {
|
|
2245
|
+
if (settled) {
|
|
2246
|
+
conn.release();
|
|
2247
|
+
return;
|
|
2248
|
+
}
|
|
2249
|
+
settled = true;
|
|
2250
|
+
clearTimeout(timer);
|
|
2251
|
+
resolve(conn);
|
|
2252
|
+
},
|
|
2253
|
+
(exc) => {
|
|
2254
|
+
if (settled) return;
|
|
2255
|
+
settled = true;
|
|
2256
|
+
clearTimeout(timer);
|
|
2257
|
+
reject(exc);
|
|
2258
|
+
}
|
|
2259
|
+
);
|
|
2260
|
+
});
|
|
2261
|
+
}
|
|
2262
|
+
/**
|
|
2263
|
+
* Run `work` on a pooled connection. `tx` wraps it in a transaction. Transient failures retry the
|
|
2264
|
+
* WHOLE closure on a fresh connection after a full rollback.
|
|
2265
|
+
*/
|
|
2266
|
+
async runPooled(operation, work, tx) {
|
|
2267
|
+
const start = performance.now();
|
|
2268
|
+
let attempt = 0;
|
|
2269
|
+
for (; ; ) {
|
|
2270
|
+
let conn;
|
|
2271
|
+
let destroy = false;
|
|
2272
|
+
try {
|
|
2273
|
+
conn = await this.acquire();
|
|
2274
|
+
if (tx) await conn.beginTransaction();
|
|
2275
|
+
const result = await work(conn);
|
|
2276
|
+
if (tx) await conn.commit();
|
|
2277
|
+
this.emit(operation, start, true, null, attempt);
|
|
2278
|
+
return result;
|
|
2279
|
+
} catch (exc) {
|
|
2280
|
+
if (conn && tx) {
|
|
2281
|
+
try {
|
|
2282
|
+
await conn.rollback();
|
|
2283
|
+
} catch {
|
|
2284
|
+
destroy = true;
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
2287
|
+
if (isConnectionLevel2(exc)) destroy = true;
|
|
2288
|
+
attempt += 1;
|
|
2289
|
+
if (attempt > this.maxRetries || !isRetryable2(exc)) {
|
|
2290
|
+
this.emit(operation, start, false, msg(exc), attempt - 1);
|
|
2291
|
+
throw exc;
|
|
2292
|
+
}
|
|
2293
|
+
} finally {
|
|
2294
|
+
if (conn) {
|
|
2295
|
+
if (destroy) conn.destroy();
|
|
2296
|
+
else conn.release();
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
await sleep(backoffDelayMs2(attempt, this.retryBaseDelayMs));
|
|
2300
|
+
}
|
|
2301
|
+
}
|
|
2302
|
+
/**
|
|
2303
|
+
* Run `work` and return its result, emitting exactly one OperationEvent.
|
|
2304
|
+
*
|
|
2305
|
+
* connection given -> run once on it; no retry, and no commit/rollback of the caller's transaction.
|
|
2306
|
+
* connection absent -> pooled, retried (see runPooled).
|
|
2307
|
+
*/
|
|
2308
|
+
async run(operation, connection, work, { tx = false } = {}) {
|
|
2309
|
+
if (connection) {
|
|
2310
|
+
if (!isMysqlConnection(connection)) {
|
|
2311
|
+
throw new TypeError(
|
|
2312
|
+
"MySQLBackend expects a mysql2/promise connection as `connection` (an object with query() and execute())."
|
|
2313
|
+
);
|
|
2314
|
+
}
|
|
2315
|
+
const start = performance.now();
|
|
2316
|
+
try {
|
|
2317
|
+
const result = tx ? await this.atomic(connection, work) : await work(connection);
|
|
2318
|
+
this.emit(operation, start, true, null, 0);
|
|
2319
|
+
return result;
|
|
2320
|
+
} catch (exc) {
|
|
2321
|
+
this.emit(operation, start, false, msg(exc), 0);
|
|
2322
|
+
throw exc;
|
|
2323
|
+
}
|
|
2324
|
+
}
|
|
2325
|
+
await this.ready();
|
|
2326
|
+
return this.runPooled(operation, work, tx);
|
|
2327
|
+
}
|
|
2328
|
+
/**
|
|
2329
|
+
* Make multi-statement work all-or-nothing on a CALLER-supplied connection.
|
|
2330
|
+
*
|
|
2331
|
+
* Is a transaction open? SAVEPOINT followed by ROLLBACK TO SAVEPOINT succeeds only inside one (in
|
|
2332
|
+
* autocommit mode the savepoint is gone before the next statement and the rollback fails with 1305).
|
|
2333
|
+
* - Open -> reuse that SAVEPOINT: a failure undoes only our work; success joins the caller's transaction.
|
|
2334
|
+
* - None -> START TRANSACTION and COMMIT our own, so a failure can never leave half-applied state.
|
|
2335
|
+
*/
|
|
2336
|
+
async atomic(conn, work) {
|
|
2337
|
+
let inTransaction;
|
|
2338
|
+
await conn.query("SAVEPOINT zerobucket_atomic");
|
|
2339
|
+
try {
|
|
2340
|
+
await conn.query("ROLLBACK TO SAVEPOINT zerobucket_atomic");
|
|
2341
|
+
inTransaction = true;
|
|
2342
|
+
} catch (exc) {
|
|
2343
|
+
if (errnoOf(exc) !== 1305) throw exc;
|
|
2344
|
+
inTransaction = false;
|
|
2345
|
+
await conn.query("START TRANSACTION");
|
|
2346
|
+
}
|
|
2347
|
+
try {
|
|
2348
|
+
const result = await work(conn);
|
|
2349
|
+
await conn.query(
|
|
2350
|
+
inTransaction ? "RELEASE SAVEPOINT zerobucket_atomic" : "COMMIT"
|
|
2351
|
+
);
|
|
2352
|
+
return result;
|
|
2353
|
+
} catch (exc) {
|
|
2354
|
+
try {
|
|
2355
|
+
await conn.query(
|
|
2356
|
+
inTransaction ? "ROLLBACK TO SAVEPOINT zerobucket_atomic" : "ROLLBACK"
|
|
2357
|
+
);
|
|
2358
|
+
if (inTransaction)
|
|
2359
|
+
await conn.query("RELEASE SAVEPOINT zerobucket_atomic");
|
|
2360
|
+
} catch {
|
|
2361
|
+
}
|
|
2362
|
+
throw exc;
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
// ---- StorageBackend --------------------------------------------------
|
|
2366
|
+
async put(row, { connection } = {}) {
|
|
2367
|
+
try {
|
|
2368
|
+
const id = randomUUID2();
|
|
2369
|
+
if (this.dedup) {
|
|
2370
|
+
await this.run(
|
|
2371
|
+
"put",
|
|
2372
|
+
connection,
|
|
2373
|
+
async (q) => {
|
|
2374
|
+
const now = mysqlNow();
|
|
2375
|
+
await q.execute(upsertBlobSql(1), blobParams2(row, now));
|
|
2376
|
+
await q.execute(D_INSERT_REF3, refParams(id, row, now));
|
|
2377
|
+
},
|
|
2378
|
+
{ tx: true }
|
|
2379
|
+
);
|
|
2380
|
+
return id;
|
|
2381
|
+
}
|
|
2382
|
+
await this.run("put", connection, async (q) => {
|
|
2383
|
+
await q.execute(INSERT3, rowParams2(id, row, mysqlNow()));
|
|
2384
|
+
});
|
|
2385
|
+
return id;
|
|
2386
|
+
} catch (exc) {
|
|
2387
|
+
throw friendly("Failed to store image", exc);
|
|
2388
|
+
}
|
|
2389
|
+
}
|
|
2390
|
+
async putMany(rows, { connection } = {}) {
|
|
2391
|
+
if (rows.length === 0) return [];
|
|
2392
|
+
try {
|
|
2393
|
+
const ids = rows.map(() => randomUUID2());
|
|
2394
|
+
await this.run(
|
|
2395
|
+
"put_many",
|
|
2396
|
+
connection,
|
|
2397
|
+
async (q) => {
|
|
2398
|
+
const now = mysqlNow();
|
|
2399
|
+
if (this.dedup) {
|
|
2400
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2401
|
+
for (const r of rows) {
|
|
2402
|
+
const g = groups.get(r.checksumSha256);
|
|
2403
|
+
if (g) g.n += 1;
|
|
2404
|
+
else groups.set(r.checksumSha256, { row: r, n: 1 });
|
|
2405
|
+
}
|
|
2406
|
+
for (const checksum of [...groups.keys()].sort()) {
|
|
2407
|
+
const g = groups.get(checksum);
|
|
2408
|
+
await q.execute(upsertBlobSql(g.n), blobParams2(g.row, now));
|
|
2409
|
+
}
|
|
2410
|
+
for (let i = 0; i < rows.length; i++)
|
|
2411
|
+
await q.execute(D_INSERT_REF3, refParams(ids[i], rows[i], now));
|
|
2412
|
+
return;
|
|
2413
|
+
}
|
|
2414
|
+
for (let i = 0; i < rows.length; i++)
|
|
2415
|
+
await q.execute(INSERT3, rowParams2(ids[i], rows[i], now));
|
|
2416
|
+
},
|
|
2417
|
+
{ tx: true }
|
|
2418
|
+
);
|
|
2419
|
+
return ids;
|
|
2420
|
+
} catch (exc) {
|
|
2421
|
+
throw friendly("Failed to store image batch", exc);
|
|
2422
|
+
}
|
|
2423
|
+
}
|
|
2424
|
+
async get(id, { connection } = {}) {
|
|
2425
|
+
if (!isValidId(id)) return null;
|
|
2426
|
+
let row;
|
|
2427
|
+
try {
|
|
2428
|
+
row = await this.run(
|
|
2429
|
+
"get",
|
|
2430
|
+
connection,
|
|
2431
|
+
async (q) => (await q.query(this.sql.selectFull, [id]))[0][0]
|
|
2432
|
+
);
|
|
2433
|
+
} catch (exc) {
|
|
2434
|
+
throw wrap("Failed to retrieve image", exc);
|
|
2435
|
+
}
|
|
2436
|
+
return row ? this.hydrate(row, id) : null;
|
|
2437
|
+
}
|
|
2438
|
+
async getMany(ids, { connection } = {}) {
|
|
2439
|
+
const valid = ids.filter(isValidId);
|
|
2440
|
+
if (valid.length === 0) return [];
|
|
2441
|
+
let rows;
|
|
2442
|
+
try {
|
|
2443
|
+
rows = await this.run(
|
|
2444
|
+
"get_many",
|
|
2445
|
+
connection,
|
|
2446
|
+
async (q) => (await q.query(this.sql.selectMany, [valid]))[0]
|
|
2447
|
+
);
|
|
2448
|
+
} catch (exc) {
|
|
2449
|
+
throw wrap("Failed to retrieve image batch", exc);
|
|
2450
|
+
}
|
|
2451
|
+
return mapLimit(rows, 4, (r) => this.hydrate(r, String(r.id)));
|
|
2452
|
+
}
|
|
2453
|
+
async getMetadata(id, { connection } = {}) {
|
|
2454
|
+
if (!isValidId(id)) return null;
|
|
2455
|
+
let row;
|
|
2456
|
+
try {
|
|
2457
|
+
row = await this.run(
|
|
2458
|
+
"get_metadata",
|
|
2459
|
+
connection,
|
|
2460
|
+
async (q) => (await q.query(this.sql.selectMetadata, [id]))[0][0]
|
|
2461
|
+
);
|
|
2462
|
+
} catch (exc) {
|
|
2463
|
+
throw wrap("Failed to retrieve image metadata", exc);
|
|
2464
|
+
}
|
|
2465
|
+
if (!row) return null;
|
|
2466
|
+
return {
|
|
2467
|
+
id: String(row.id),
|
|
2468
|
+
mimeType: row.mime_type,
|
|
2469
|
+
originalFilename: row.original_filename,
|
|
2470
|
+
sizeBytes: row.size_bytes,
|
|
2471
|
+
width: row.width,
|
|
2472
|
+
height: row.height,
|
|
2473
|
+
checksumSha256: row.checksum_sha256
|
|
2474
|
+
};
|
|
2475
|
+
}
|
|
2476
|
+
async getStream(id, opts) {
|
|
2477
|
+
if (!isValidId(id)) return null;
|
|
2478
|
+
const { connection, chunkSize } = opts;
|
|
2479
|
+
if (!Number.isInteger(chunkSize) || chunkSize <= 0)
|
|
2480
|
+
throw new RangeError("chunkSize must be a positive integer");
|
|
2481
|
+
let info;
|
|
2482
|
+
try {
|
|
2483
|
+
info = await this.run(
|
|
2484
|
+
"get_metadata",
|
|
2485
|
+
connection,
|
|
2486
|
+
async (q) => (await q.query(this.sql.streamInfo, [id]))[0][0]
|
|
2487
|
+
);
|
|
2488
|
+
} catch (exc) {
|
|
2489
|
+
throw wrap("Failed to retrieve image metadata", exc);
|
|
2490
|
+
}
|
|
2491
|
+
if (!info) return null;
|
|
2492
|
+
const totalSize = info.size_bytes;
|
|
2493
|
+
const start = opts.range?.start ?? 0;
|
|
2494
|
+
const end = Math.min(opts.range?.end ?? totalSize - 1, totalSize - 1);
|
|
2495
|
+
if (info.storage_backend === "object_storage") {
|
|
2496
|
+
const os = this.opts.objectStorage;
|
|
2497
|
+
if (!os) throw tieredWithoutStorage(id, info.object_storage_key);
|
|
2498
|
+
return streamFromObjectStorage(
|
|
2499
|
+
os,
|
|
2500
|
+
id,
|
|
2501
|
+
info.object_storage_key,
|
|
2502
|
+
start,
|
|
2503
|
+
end,
|
|
2504
|
+
chunkSize
|
|
2505
|
+
);
|
|
2506
|
+
}
|
|
2507
|
+
const self = this;
|
|
2508
|
+
async function* generate() {
|
|
2509
|
+
let offset = start;
|
|
2510
|
+
let delivered = 0;
|
|
2511
|
+
const wanted = Math.max(0, end - start + 1);
|
|
2512
|
+
while (delivered < wanted) {
|
|
2513
|
+
const length = Math.min(chunkSize, wanted - delivered);
|
|
2514
|
+
let row;
|
|
2515
|
+
try {
|
|
2516
|
+
row = await self.run(
|
|
2517
|
+
"get_stream",
|
|
2518
|
+
connection,
|
|
2519
|
+
async (q) => (await q.query(self.sql.chunk, [offset + 1, length, id]))[0][0]
|
|
2520
|
+
);
|
|
2521
|
+
} catch (exc) {
|
|
2522
|
+
throw wrap("Failed to stream image", exc);
|
|
2523
|
+
}
|
|
2524
|
+
if (!row) {
|
|
2525
|
+
throw new StorageError(
|
|
2526
|
+
`Image ${JSON.stringify(id)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
|
|
2527
|
+
);
|
|
2528
|
+
}
|
|
2529
|
+
const chunk = row.chunk;
|
|
2530
|
+
if (!chunk || chunk.length === 0) {
|
|
2531
|
+
throw new StorageError(
|
|
2532
|
+
`Image ${JSON.stringify(id)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
|
|
2533
|
+
);
|
|
2534
|
+
}
|
|
2535
|
+
yield chunk;
|
|
2536
|
+
offset += chunk.length;
|
|
2537
|
+
delivered += chunk.length;
|
|
2538
|
+
}
|
|
2539
|
+
}
|
|
2540
|
+
return generate();
|
|
2541
|
+
}
|
|
2542
|
+
async delete(id, { connection } = {}) {
|
|
2543
|
+
if (!isValidId(id)) return false;
|
|
2544
|
+
if (this.dedup) return this.deleteDedup(id, connection);
|
|
2545
|
+
try {
|
|
2546
|
+
const row = await this.run(
|
|
2547
|
+
"delete",
|
|
2548
|
+
connection,
|
|
2549
|
+
async (q) => {
|
|
2550
|
+
const found = (await q.query(LOCK_ONE, [id]))[0][0];
|
|
2551
|
+
if (!found) return void 0;
|
|
2552
|
+
await q.query(DELETE_ONE2, [id]);
|
|
2553
|
+
return found;
|
|
2554
|
+
},
|
|
2555
|
+
{ tx: true }
|
|
2556
|
+
);
|
|
2557
|
+
if (row?.storage_backend === "object_storage")
|
|
2558
|
+
await this.cleanupObject([row.object_storage_key], connection);
|
|
2559
|
+
return row !== void 0;
|
|
2560
|
+
} catch (exc) {
|
|
2561
|
+
throw wrap("Failed to delete image", exc);
|
|
2562
|
+
}
|
|
2563
|
+
}
|
|
2564
|
+
async deleteMany(ids, { connection } = {}) {
|
|
2565
|
+
const valid = [...new Set(ids.filter(isValidId))];
|
|
2566
|
+
if (valid.length === 0) return [];
|
|
2567
|
+
if (this.dedup) return this.deleteManyDedup(valid, connection);
|
|
2568
|
+
try {
|
|
2569
|
+
const rows = await this.run(
|
|
2570
|
+
"delete_many",
|
|
2571
|
+
connection,
|
|
2572
|
+
async (q) => {
|
|
2573
|
+
const found = (await q.query(LOCK_MANY, [valid]))[0];
|
|
2574
|
+
for (const r of found) await q.query(DELETE_ONE2, [r.id]);
|
|
2575
|
+
return found;
|
|
2576
|
+
},
|
|
2577
|
+
{ tx: true }
|
|
2578
|
+
);
|
|
2579
|
+
await this.cleanupObject(
|
|
2580
|
+
rows.filter((r) => r.storage_backend === "object_storage").map((r) => r.object_storage_key),
|
|
2581
|
+
connection
|
|
2582
|
+
);
|
|
2583
|
+
return rows.map((r) => String(r.id));
|
|
2584
|
+
} catch (exc) {
|
|
2585
|
+
throw wrap("Failed to delete image batch", exc);
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2588
|
+
async exists(id, { connection } = {}) {
|
|
2589
|
+
if (!isValidId(id)) return false;
|
|
2590
|
+
try {
|
|
2591
|
+
return await this.run(
|
|
2592
|
+
"exists",
|
|
2593
|
+
connection,
|
|
2594
|
+
async (q) => (await q.query(this.sql.exists, [id]))[0].length > 0
|
|
2595
|
+
);
|
|
2596
|
+
} catch (exc) {
|
|
2597
|
+
throw wrap("Failed to check image existence", exc);
|
|
2598
|
+
}
|
|
2599
|
+
}
|
|
2600
|
+
// ---- maintenance (CLI) -----------------------------------------------
|
|
2601
|
+
/** Storage statistics, or null if the tables do not exist yet (nothing is created). */
|
|
2602
|
+
async getInfo() {
|
|
2603
|
+
try {
|
|
2604
|
+
return await this.run("info", void 0, async (q) => {
|
|
2605
|
+
const need = this.dedup ? ["zerobucket_image_refs", "zerobucket_blobs"] : ["zerobucket_images"];
|
|
2606
|
+
const present = new Set(
|
|
2607
|
+
(await q.query(
|
|
2608
|
+
"SELECT TABLE_NAME AS t FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?)",
|
|
2609
|
+
[need]
|
|
2610
|
+
))[0].map((r) => r.t)
|
|
2611
|
+
);
|
|
2612
|
+
if (!need.every((t) => present.has(t))) return null;
|
|
2613
|
+
const j = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
|
|
2614
|
+
const size = this.dedup ? "b.size_bytes" : "size_bytes";
|
|
2615
|
+
const created = this.dedup ? "r.created_at" : "created_at";
|
|
2616
|
+
const mime = this.dedup ? "b.mime_type" : "mime_type";
|
|
2617
|
+
const tot = (await q.query(
|
|
2618
|
+
`SELECT COUNT(*) AS n, COALESCE(SUM(${size}), 0) AS bytes, CAST(MIN(${created}) AS CHAR) AS oldest, CAST(MAX(${created}) AS CHAR) AS newest ${j}`
|
|
2619
|
+
))[0][0];
|
|
2620
|
+
const disk = (await q.query(
|
|
2621
|
+
"SELECT COALESCE(SUM(DATA_LENGTH + INDEX_LENGTH), 0) AS s FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?)",
|
|
2622
|
+
[need]
|
|
2623
|
+
))[0][0];
|
|
2624
|
+
const fmt = (await q.query(
|
|
2625
|
+
`SELECT ${mime} AS m, COUNT(*) AS n, COALESCE(SUM(${size}), 0) AS bytes ${j} GROUP BY ${mime} ORDER BY COUNT(*) DESC, ${mime}`
|
|
2626
|
+
))[0];
|
|
2627
|
+
const utc = (v) => v ? `${v}+00:00` : null;
|
|
2628
|
+
const info = {
|
|
2629
|
+
mode: this.dedup ? "dedup" : "classic",
|
|
2630
|
+
count: Number(tot.n),
|
|
2631
|
+
totalBytes: Number(tot.bytes),
|
|
2632
|
+
oldest: utc(tot.oldest),
|
|
2633
|
+
newest: utc(tot.newest),
|
|
2634
|
+
onDisk: prettySize(Number(disk.s)),
|
|
2635
|
+
byFormat: fmt.map((r) => ({
|
|
2636
|
+
mimeType: r.m,
|
|
2637
|
+
count: Number(r.n),
|
|
2638
|
+
bytes: Number(r.bytes)
|
|
2639
|
+
}))
|
|
2640
|
+
};
|
|
2641
|
+
if (this.dedup) {
|
|
2642
|
+
const b = (await q.query(
|
|
2643
|
+
"SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes FROM zerobucket_blobs"
|
|
2644
|
+
))[0][0];
|
|
2645
|
+
info.dedup = { blobs: Number(b.n), storedBytes: Number(b.bytes) };
|
|
2646
|
+
} else {
|
|
2647
|
+
const t = (await q.query(
|
|
2648
|
+
"SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
|
|
2649
|
+
))[0][0];
|
|
2650
|
+
info.tiered = { count: Number(t.n), bytes: Number(t.bytes) };
|
|
2651
|
+
}
|
|
2652
|
+
return info;
|
|
2653
|
+
});
|
|
2654
|
+
} catch (exc) {
|
|
2655
|
+
throw wrap("Failed to read storage info", exc);
|
|
2656
|
+
}
|
|
2657
|
+
}
|
|
2658
|
+
/** Ids matching the filters: oldest first, or a random sample. Used by the CLI's verify and tier. */
|
|
2659
|
+
async listImages(o = {}) {
|
|
2660
|
+
const where = [];
|
|
2661
|
+
const params = [];
|
|
2662
|
+
const created = this.dedup ? "r.created_at" : "created_at";
|
|
2663
|
+
if (o.minSize !== void 0) {
|
|
2664
|
+
where.push(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= ?`);
|
|
2665
|
+
params.push(o.minSize);
|
|
2666
|
+
}
|
|
2667
|
+
if (o.olderThanDays !== void 0) {
|
|
2668
|
+
where.push(`${created} <= UTC_TIMESTAMP(6) - INTERVAL ? DAY`);
|
|
2669
|
+
params.push(o.olderThanDays);
|
|
2670
|
+
}
|
|
2671
|
+
if (o.onlyUntiered && !this.dedup)
|
|
2672
|
+
where.push(`storage_backend = '${LOCAL}'`);
|
|
2673
|
+
const from = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
|
|
2674
|
+
const idCol = this.dedup ? "r.id" : "id";
|
|
2675
|
+
const tierCol = this.dedup ? "0 AS tiered" : "(storage_backend = 'object_storage') AS tiered";
|
|
2676
|
+
const order = o.sample !== void 0 ? "ORDER BY RAND()" : `ORDER BY ${created}, ${idCol}`;
|
|
2677
|
+
const lim = o.sample ?? o.limit;
|
|
2678
|
+
let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
|
|
2679
|
+
if (lim !== void 0) {
|
|
2680
|
+
if (!Number.isInteger(lim) || lim < 0)
|
|
2681
|
+
throw new RangeError("limit/sample must be a non-negative integer");
|
|
2682
|
+
sql += ` LIMIT ${lim}`;
|
|
2683
|
+
}
|
|
2684
|
+
try {
|
|
2685
|
+
const rows = await this.run(
|
|
2686
|
+
"list_images",
|
|
2687
|
+
void 0,
|
|
2688
|
+
async (q) => (await q.query(sql, params))[0]
|
|
2689
|
+
);
|
|
2690
|
+
return rows.map((r) => ({
|
|
2691
|
+
id: String(r.id),
|
|
2692
|
+
tiered: Boolean(Number(r.tiered))
|
|
2693
|
+
}));
|
|
2694
|
+
} catch (exc) {
|
|
2695
|
+
throw wrap("Failed to list images", exc);
|
|
2696
|
+
}
|
|
2697
|
+
}
|
|
2698
|
+
// ---- dedup deletes ---------------------------------------------------
|
|
2699
|
+
/**
|
|
2700
|
+
* Lock the ref row, delete it, then decrement its blob (the UPDATE takes the blob row lock, so two
|
|
2701
|
+
* deletes of different refs to one blob serialize instead of racing) and remove the blob once nothing
|
|
2702
|
+
* references it. All in one transaction: a failure can never leave a ref without its count or the reverse.
|
|
2703
|
+
*/
|
|
2704
|
+
async deleteDedup(id, connection) {
|
|
2705
|
+
try {
|
|
2706
|
+
return await this.run(
|
|
2707
|
+
"delete",
|
|
2708
|
+
connection,
|
|
2709
|
+
async (q) => {
|
|
2710
|
+
const ref = (await q.query(D_LOCK_REF, [id]))[0][0];
|
|
2711
|
+
if (!ref) return false;
|
|
2712
|
+
const checksum = ref.checksum_sha256;
|
|
2713
|
+
await q.query(D_DELETE_REF3, [id]);
|
|
2714
|
+
await q.query(D_DECREMENT3, [1, checksum]);
|
|
2715
|
+
await q.query(D_DELETE_EMPTY3, [checksum]);
|
|
2716
|
+
return true;
|
|
2717
|
+
},
|
|
2718
|
+
{ tx: true }
|
|
2719
|
+
);
|
|
2720
|
+
} catch (exc) {
|
|
2721
|
+
throw wrap("Failed to delete image", exc);
|
|
2722
|
+
}
|
|
2723
|
+
}
|
|
2724
|
+
async deleteManyDedup(ids, connection) {
|
|
2725
|
+
try {
|
|
2726
|
+
return await this.run(
|
|
2727
|
+
"delete_many",
|
|
2728
|
+
connection,
|
|
2729
|
+
async (q) => {
|
|
2730
|
+
const found = (await q.query(D_LOCK_REFS, [ids]))[0];
|
|
2731
|
+
if (found.length === 0) return [];
|
|
2732
|
+
for (const r of found) await q.query(D_DELETE_REF3, [r.id]);
|
|
2733
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2734
|
+
for (const r of found)
|
|
2735
|
+
counts.set(
|
|
2736
|
+
r.checksum_sha256,
|
|
2737
|
+
(counts.get(r.checksum_sha256) ?? 0) + 1
|
|
2738
|
+
);
|
|
2739
|
+
for (const checksum of [...counts.keys()].sort()) {
|
|
2740
|
+
await q.query(D_DECREMENT3, [counts.get(checksum), checksum]);
|
|
2741
|
+
await q.query(D_DELETE_EMPTY3, [checksum]);
|
|
2742
|
+
}
|
|
2743
|
+
return found.map((r) => String(r.id));
|
|
2744
|
+
},
|
|
2745
|
+
{ tx: true }
|
|
2746
|
+
);
|
|
2747
|
+
} catch (exc) {
|
|
2748
|
+
throw wrap("Failed to delete image batch", exc);
|
|
2749
|
+
}
|
|
2750
|
+
}
|
|
2751
|
+
// ---- helpers ---------------------------------------------------------
|
|
2752
|
+
hydrate(row, id) {
|
|
2753
|
+
return hydrateRow(row, id, this.opts.objectStorage);
|
|
2754
|
+
}
|
|
2755
|
+
/**
|
|
2756
|
+
* Remove tiered objects AFTER their rows are gone: once the row is deleted the image is correctly
|
|
2757
|
+
* "not found" to everyone, so a failed object delete only leaves a harmless orphan, whereas deleting
|
|
2758
|
+
* the object first and then failing the row delete would leave a row pointing at nothing.
|
|
2759
|
+
*
|
|
2760
|
+
* Never runs inside a caller-owned transaction (`connection`): we cannot know whether it will
|
|
2761
|
+
* commit. In that case the object stays until you remove it yourself after committing. Failures are
|
|
2762
|
+
* reported through onOperation and never fail the delete itself.
|
|
2763
|
+
*/
|
|
2764
|
+
async cleanupObject(keys, connection) {
|
|
2765
|
+
const os = this.opts.objectStorage;
|
|
2766
|
+
if (!os || connection || keys.length === 0) return;
|
|
2767
|
+
await Promise.all(
|
|
2768
|
+
keys.map(async (key) => {
|
|
2769
|
+
const start = performance.now();
|
|
2770
|
+
try {
|
|
2771
|
+
await os.delete(key);
|
|
2772
|
+
this.emit("object_storage_delete", start, true, null, 0);
|
|
2773
|
+
} catch (exc) {
|
|
2774
|
+
this.emit("object_storage_delete", start, false, msg(exc), 0);
|
|
2775
|
+
}
|
|
2776
|
+
})
|
|
2777
|
+
);
|
|
2778
|
+
}
|
|
2779
|
+
/**
|
|
2780
|
+
* Move an image's bytes out of the database into object storage, replacing the row's `data` with a
|
|
2781
|
+
* pointer. Returns null (no such image), false (already tiered: a safe no-op, so backfills can be
|
|
2782
|
+
* re-run) or true (tiered just now).
|
|
2783
|
+
*
|
|
2784
|
+
* SAFETY: the upload happens INSIDE the same transaction as the row lock (SELECT ... FOR UPDATE)
|
|
2785
|
+
* and the UPDATE that flips the row. If the upload fails the transaction rolls back and the row is
|
|
2786
|
+
* untouched: there is no moment when the bytes exist nowhere, or when a row claims to be tiered
|
|
2787
|
+
* without a completed upload.
|
|
2788
|
+
*
|
|
2789
|
+
* TRADEOFF: InnoDB row-locks only THIS row for the duration of the upload. Concurrent writes to this
|
|
2790
|
+
* image wait (up to innodb_lock_wait_timeout, 50s by default); every other row is unaffected and
|
|
2791
|
+
* plain reads are never blocked. Meant for explicit maintenance, not hot paths.
|
|
2792
|
+
*
|
|
2793
|
+
* Retry is safe: a transient error replays the whole closure, and the deterministic key (the image
|
|
2794
|
+
* id) makes the re-upload an idempotent overwrite.
|
|
2795
|
+
*/
|
|
2796
|
+
async tierToObjectStorage(id, { connection } = {}) {
|
|
2797
|
+
if (this.dedup) {
|
|
2798
|
+
throw new StorageError(
|
|
2799
|
+
"tierToObjectStorage() is not supported in dedup mode (a blob may be shared by many images)."
|
|
2800
|
+
);
|
|
2801
|
+
}
|
|
2802
|
+
const os = this.opts.objectStorage;
|
|
2803
|
+
if (!os) {
|
|
2804
|
+
throw new StorageError(
|
|
2805
|
+
"tierToObjectStorage() requires objectStorage to be configured (new ZeroBucket({ objectStorage: new ObjectStorage({ bucket }) }))."
|
|
2806
|
+
);
|
|
2807
|
+
}
|
|
2808
|
+
if (!isValidId(id)) return null;
|
|
2809
|
+
const key = id.toLowerCase();
|
|
2810
|
+
try {
|
|
2811
|
+
return await this.run(
|
|
2812
|
+
"tier_to_object_storage",
|
|
2813
|
+
connection,
|
|
2814
|
+
async (q) => {
|
|
2815
|
+
const row = (await q.query(SELECT_FOR_TIERING3, [id]))[0][0];
|
|
2816
|
+
if (!row) return null;
|
|
2817
|
+
if (row.storage_backend !== LOCAL) return false;
|
|
2818
|
+
await os.upload(key, row.data, { mimeType: row.mime_type });
|
|
2819
|
+
await q.query(UPDATE_AFTER_TIERING3, [os.bucket, key, mysqlNow(), id]);
|
|
2820
|
+
return true;
|
|
2821
|
+
},
|
|
2822
|
+
{ tx: true }
|
|
2823
|
+
);
|
|
2824
|
+
} catch (exc) {
|
|
2825
|
+
throw wrap("Failed to tier image to object storage", exc);
|
|
2826
|
+
}
|
|
2827
|
+
}
|
|
2828
|
+
};
|
|
2829
|
+
function blobParams2(r, now) {
|
|
2830
|
+
return [
|
|
2831
|
+
r.checksumSha256,
|
|
2832
|
+
r.data,
|
|
2833
|
+
r.mimeType,
|
|
2834
|
+
r.sizeBytes,
|
|
2835
|
+
r.width,
|
|
2836
|
+
r.height,
|
|
2837
|
+
now
|
|
2838
|
+
];
|
|
2839
|
+
}
|
|
2840
|
+
function refParams(id, r, now) {
|
|
2841
|
+
return [id, r.checksumSha256, r.originalFilename, now, now];
|
|
2842
|
+
}
|
|
2843
|
+
function rowParams2(id, r, now) {
|
|
2844
|
+
return [
|
|
2845
|
+
id,
|
|
2846
|
+
r.data,
|
|
2847
|
+
r.mimeType,
|
|
2848
|
+
r.originalFilename,
|
|
2849
|
+
r.sizeBytes,
|
|
2850
|
+
r.width,
|
|
2851
|
+
r.height,
|
|
2852
|
+
r.checksumSha256,
|
|
2853
|
+
now,
|
|
2854
|
+
now
|
|
2855
|
+
];
|
|
2856
|
+
}
|
|
2857
|
+
|
|
1883
2858
|
// src/object-storage.ts
|
|
1884
2859
|
var ObjectStorage = class {
|
|
1885
2860
|
constructor(opts) {
|
|
@@ -2024,6 +2999,7 @@ Commands:
|
|
|
2024
2999
|
init Create the schema (tables and indexes) if missing.
|
|
2025
3000
|
migrate Apply schema migrations (currently: same as init).
|
|
2026
3001
|
--to-dedup copy every classic image into the dedup tables (PostgreSQL only; non-destructive)
|
|
3002
|
+
--skip-tiered with --to-dedup: skip images tiered to object storage instead of refusing
|
|
2027
3003
|
info Show storage stats: count, size, breakdown by format.
|
|
2028
3004
|
verify Re-checksum stored images to detect corruption.
|
|
2029
3005
|
--sample N check a random sample of N images instead of every image
|
|
@@ -2036,7 +3012,8 @@ Commands:
|
|
|
2036
3012
|
--region NAME --aws-access-key-id ID --aws-secret-access-key KEY
|
|
2037
3013
|
|
|
2038
3014
|
Database (choose one; flags beat environment variables):
|
|
2039
|
-
--database-url URL PostgreSQL
|
|
3015
|
+
--database-url URL PostgreSQL (postgresql://) or MySQL/MariaDB (mysql://, mariadb://)
|
|
3016
|
+
connection string (env: ZEROBUCKET_DATABASE_URL)
|
|
2040
3017
|
--sqlite PATH SQLite database file (env: ZEROBUCKET_SQLITE_PATH)
|
|
2041
3018
|
--dedup operate on the dedup-mode tables instead of the classic table
|
|
2042
3019
|
|
|
@@ -2052,6 +3029,7 @@ var OPTIONS = {
|
|
|
2052
3029
|
help: { type: "boolean", short: "h" },
|
|
2053
3030
|
version: { type: "boolean", short: "v" },
|
|
2054
3031
|
"to-dedup": { type: "boolean" },
|
|
3032
|
+
"skip-tiered": { type: "boolean" },
|
|
2055
3033
|
sample: { type: "string" },
|
|
2056
3034
|
bucket: { type: "string" },
|
|
2057
3035
|
"endpoint-url": { type: "string" },
|
|
@@ -2066,7 +3044,7 @@ var OPTIONS = {
|
|
|
2066
3044
|
};
|
|
2067
3045
|
var COMMAND_OPTIONS = {
|
|
2068
3046
|
init: [],
|
|
2069
|
-
migrate: ["to-dedup"],
|
|
3047
|
+
migrate: ["to-dedup", "skip-tiered"],
|
|
2070
3048
|
info: [],
|
|
2071
3049
|
verify: [
|
|
2072
3050
|
"sample",
|
|
@@ -2090,13 +3068,14 @@ var COMMAND_OPTIONS = {
|
|
|
2090
3068
|
]
|
|
2091
3069
|
};
|
|
2092
3070
|
var COMMON = /* @__PURE__ */ new Set(["database-url", "sqlite", "dedup", "help", "version"]);
|
|
3071
|
+
var urlTarget = (url) => /^(mysql|mariadb):\/\//i.test(url) ? { kind: "mysql", url } : { kind: "postgres", url };
|
|
2093
3072
|
function resolveTarget(v, env) {
|
|
2094
3073
|
if (v.sqlite && v["database-url"])
|
|
2095
3074
|
throw usageError(
|
|
2096
3075
|
"Error: choose one database: --sqlite OR --database-url, not both."
|
|
2097
3076
|
);
|
|
2098
3077
|
if (v.sqlite) return { kind: "sqlite", path: v.sqlite };
|
|
2099
|
-
if (v["database-url"]) return
|
|
3078
|
+
if (v["database-url"]) return urlTarget(v["database-url"]);
|
|
2100
3079
|
const url = env.ZEROBUCKET_DATABASE_URL;
|
|
2101
3080
|
const path = env.ZEROBUCKET_SQLITE_PATH;
|
|
2102
3081
|
if (url && path) {
|
|
@@ -2104,10 +3083,10 @@ function resolveTarget(v, env) {
|
|
|
2104
3083
|
"Error: both ZEROBUCKET_DATABASE_URL and ZEROBUCKET_SQLITE_PATH are set. Pass --database-url or --sqlite to choose."
|
|
2105
3084
|
);
|
|
2106
3085
|
}
|
|
2107
|
-
if (url) return
|
|
3086
|
+
if (url) return urlTarget(url);
|
|
2108
3087
|
if (path) return { kind: "sqlite", path };
|
|
2109
3088
|
throw usageError(
|
|
2110
|
-
"Error: no database provided. Pass --database-url (PostgreSQL) or --sqlite PATH, or set the ZEROBUCKET_DATABASE_URL / ZEROBUCKET_SQLITE_PATH environment variable."
|
|
3089
|
+
"Error: no database provided. Pass --database-url (PostgreSQL or MySQL/MariaDB) or --sqlite PATH, or set the ZEROBUCKET_DATABASE_URL / ZEROBUCKET_SQLITE_PATH environment variable."
|
|
2111
3090
|
);
|
|
2112
3091
|
}
|
|
2113
3092
|
function openBackend(t, o) {
|
|
@@ -2116,6 +3095,12 @@ function openBackend(t, o) {
|
|
|
2116
3095
|
dedup: o.dedup,
|
|
2117
3096
|
autoMigrate: o.autoMigrate,
|
|
2118
3097
|
objectStorage: o.objectStorage
|
|
3098
|
+
}) : t.kind === "mysql" ? new MySQLBackend({
|
|
3099
|
+
connectionString: t.url,
|
|
3100
|
+
dedup: o.dedup,
|
|
3101
|
+
autoMigrate: o.autoMigrate,
|
|
3102
|
+
objectStorage: o.objectStorage,
|
|
3103
|
+
maxRetries: 0
|
|
2119
3104
|
}) : new PostgresBackend({
|
|
2120
3105
|
connectionString: t.url,
|
|
2121
3106
|
dedup: o.dedup,
|
|
@@ -2176,6 +3161,8 @@ async function cmdInit(v, env, io) {
|
|
|
2176
3161
|
return 0;
|
|
2177
3162
|
}
|
|
2178
3163
|
async function cmdMigrate(v, env, io) {
|
|
3164
|
+
if (v["skip-tiered"] && !v["to-dedup"])
|
|
3165
|
+
throw usageError("Error: --skip-tiered only applies with --to-dedup.");
|
|
2179
3166
|
if (v["to-dedup"]) {
|
|
2180
3167
|
const target = resolveTarget(v, env);
|
|
2181
3168
|
if (target.kind !== "postgres")
|
|
@@ -2183,7 +3170,9 @@ async function cmdMigrate(v, env, io) {
|
|
|
2183
3170
|
const backend = openBackend(target, { dedup: true, autoMigrate: true });
|
|
2184
3171
|
await connect(backend, "connect");
|
|
2185
3172
|
try {
|
|
2186
|
-
const r = await backend.migrateClassicToDedup(
|
|
3173
|
+
const r = await backend.migrateClassicToDedup({
|
|
3174
|
+
skipTiered: Boolean(v["skip-tiered"])
|
|
3175
|
+
});
|
|
2187
3176
|
io.out(
|
|
2188
3177
|
`Migrated ${num(r.imagesMigrated)} image(s) into ${num(r.distinctBlobsCreated)} distinct blob(s).`
|
|
2189
3178
|
);
|
|
@@ -2191,6 +3180,10 @@ async function cmdMigrate(v, env, io) {
|
|
|
2191
3180
|
` Duplicate references found: ${num(r.duplicateReferencesFound)}`
|
|
2192
3181
|
);
|
|
2193
3182
|
io.out(` Already migrated (skipped): ${num(r.alreadyMigrated)}`);
|
|
3183
|
+
if (r.tieredSkipped > 0)
|
|
3184
|
+
io.out(
|
|
3185
|
+
` Tiered to object storage (skipped, NOT in the dedup tables): ${num(r.tieredSkipped)}`
|
|
3186
|
+
);
|
|
2194
3187
|
io.out("The classic table was not modified.");
|
|
2195
3188
|
return 0;
|
|
2196
3189
|
} catch (exc) {
|
|
@@ -2239,7 +3232,7 @@ async function cmdInfo(v, env, io) {
|
|
|
2239
3232
|
}
|
|
2240
3233
|
if (info.onDisk) {
|
|
2241
3234
|
io.out(
|
|
2242
|
-
target.kind === "postgres" ? ` On-disk size (table + TOAST + indexes): ${info.onDisk}` : ` On-disk size (database file + WAL): ${info.onDisk}`
|
|
3235
|
+
target.kind === "postgres" ? ` On-disk size (table + TOAST + indexes): ${info.onDisk}` : target.kind === "mysql" ? ` On-disk size (data + indexes, approximate): ${info.onDisk}` : ` On-disk size (database file + WAL): ${info.onDisk}`
|
|
2243
3236
|
);
|
|
2244
3237
|
}
|
|
2245
3238
|
if (info.tiered && info.tiered.count > 0) {
|