zerobucket 0.7.0 → 0.8.0

package/dist/index.cjs

@@ -2014,11 +2014,60 @@ var CHUNK2 = `SELECT SUBSTRING(data FROM ? FOR ?) AS chunk FROM zerobucket_image
 var EXISTS3 = `SELECT 1 FROM zerobucket_images WHERE id = ?`;
 var LOCK_ONE = `SELECT storage_backend, object_storage_key FROM zerobucket_images WHERE id = ? FOR UPDATE`;
 var DELETE_ONE2 = `DELETE FROM zerobucket_images WHERE id = ?`;
-var LOCK_MANY = `SELECT id, storage_backend, object_storage_key FROM zerobucket_images WHERE id IN (?) ORDER BY id FOR UPDATE`;
-var DELETE_MANY2 = `DELETE FROM zerobucket_images WHERE id IN (?)`;
+var LOCK_MANY = `SELECT id, storage_backend, object_storage_key FROM zerobucket_images FORCE INDEX (PRIMARY) WHERE id IN (?) ORDER BY id FOR UPDATE`;
 var SELECT_FOR_TIERING3 = `SELECT data, mime_type, size_bytes, storage_backend FROM zerobucket_images WHERE id = ? FOR UPDATE`;
 var UPDATE_AFTER_TIERING3 = `UPDATE zerobucket_images SET data = NULL, storage_backend = 'object_storage', object_storage_bucket = ?, object_storage_key = ?, updated_at = ? WHERE id = ?`;
 var LOCAL = "mysql";
+var DEDUP_BLOBS_SCHEMA = `
+CREATE TABLE IF NOT EXISTS zerobucket_blobs (
+    checksum_sha256     CHAR(64) PRIMARY KEY,
+    data                LONGBLOB NOT NULL,
+    mime_type           VARCHAR(255) NOT NULL,
+    size_bytes          INT NOT NULL,
+    width               INT,
+    height              INT,
+    ref_count           INT NOT NULL DEFAULT 0,
+    created_at          DATETIME(6) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
+var DEDUP_REFS_SCHEMA = `
+CREATE TABLE IF NOT EXISTS zerobucket_image_refs (
+    id                  CHAR(36) PRIMARY KEY,
+    checksum_sha256     CHAR(64) NOT NULL,
+    original_filename   TEXT,
+    created_at          DATETIME(6) NOT NULL,
+    updated_at          DATETIME(6) NOT NULL,
+    INDEX idx_zerobucket_image_refs_checksum (checksum_sha256),
+    INDEX idx_zerobucket_image_refs_created_at (created_at),
+    FOREIGN KEY (checksum_sha256) REFERENCES zerobucket_blobs(checksum_sha256)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
+function upsertBlobSql(n) {
+  if (!Number.isInteger(n) || n < 1) throw new RangeError("invalid ref count");
+  return `INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, width, height, ref_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ${n}, ?) ON DUPLICATE KEY UPDATE ref_count = ref_count + ${n}`;
+}
+var D_INSERT_REF3 = `INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at, updated_at) VALUES (?, ?, ?, ?, ?)`;
+var D_COLS3 = `r.id, b.data, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256, '${LOCAL}' AS storage_backend, NULL AS object_storage_bucket, NULL AS object_storage_key`;
+var D_FROM3 = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
+var D_LOCK_REF = `SELECT checksum_sha256 FROM zerobucket_image_refs WHERE id = ? FOR UPDATE`;
+var D_DELETE_REF3 = `DELETE FROM zerobucket_image_refs WHERE id = ?`;
+var D_LOCK_REFS = `SELECT id, checksum_sha256 FROM zerobucket_image_refs FORCE INDEX (PRIMARY) WHERE id IN (?) ORDER BY id FOR UPDATE`;
+var D_DECREMENT3 = `UPDATE zerobucket_blobs SET ref_count = ref_count - ? WHERE checksum_sha256 = ?`;
+var D_DELETE_EMPTY3 = `DELETE FROM zerobucket_blobs WHERE checksum_sha256 = ? AND ref_count <= 0`;
+var CLASSIC_SQL2 = {
+  selectFull: SELECT_FULL3,
+  selectMany: SELECT_MANY2,
+  selectMetadata: SELECT_METADATA3,
+  streamInfo: STREAM_INFO2,
+  chunk: CHUNK2,
+  exists: EXISTS3
+};
+var DEDUP_SQL2 = {
+  selectFull: `SELECT ${D_COLS3} ${D_FROM3} WHERE r.id = ?`,
+  selectMany: `SELECT ${D_COLS3} ${D_FROM3} WHERE r.id IN (?)`,
+  selectMetadata: `SELECT r.id, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256 ${D_FROM3} WHERE r.id = ?`,
+  streamInfo: `SELECT b.size_bytes, '${LOCAL}' AS storage_backend, NULL AS object_storage_key ${D_FROM3} WHERE r.id = ?`,
+  chunk: `SELECT SUBSTRING(b.data FROM ? FOR ?) AS chunk ${D_FROM3} WHERE r.id = ?`,
+  exists: `SELECT 1 FROM zerobucket_image_refs WHERE id = ?`
+};
 var RETRYABLE_ERRNOS = /* @__PURE__ */ new Set([
   1205,
   // ER_LOCK_WAIT_TIMEOUT
@@ -2108,11 +2157,13 @@ var MySQLBackend = class {
         "MySQLBackend requires either `connectionString` or `pool`"
       );
     }
-    if (opts.dedup) {
+    this.dedup = opts.dedup ?? false;
+    if (this.dedup && opts.objectStorage) {
       throw new TypeError(
-        "dedup mode is not available for MySQL in the npm package yet (the Python package supports it). Use classic mode, or Postgres/SQLite for dedup."
+        "objectStorage is not supported together with dedup: true (same restriction as the Python package). Tiering moves one image's bytes, but a deduplicated blob can be shared by many images."
       );
     }
+    this.sql = this.dedup ? DEDUP_SQL2 : CLASSIC_SQL2;
     this.ownsPool = !opts.pool;
     this.pool = opts.pool;
     this.maxRetries = opts.maxRetries ?? 3;
@@ -2129,6 +2180,8 @@ var MySQLBackend = class {
   retryBaseDelayMs;
   poolTimeoutMs;
   autoMigrate;
+  dedup;
+  sql;
   // ---- lifecycle -------------------------------------------------------
   ready() {
     if (this.closed)
@@ -2194,6 +2247,10 @@ var MySQLBackend = class {
       );
     }
     try {
+      if (this.dedup) {
+        await this.migrateDedup(q);
+        return;
+      }
       const tableExists = (await q.query(
         "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images'"
       ))[0].length > 0;
@@ -2235,6 +2292,16 @@ var MySQLBackend = class {
       });
     }
   }
+  /** Dedup tables, created only if missing (same no-needless-DDL rule as the classic table). */
+  async migrateDedup(q) {
+    const have = new Set(
+      (await q.query(
+        "SELECT TABLE_NAME AS t FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN ('zerobucket_blobs', 'zerobucket_image_refs')"
+      ))[0].map((r) => r.t)
+    );
+    if (!have.has("zerobucket_blobs")) await q.query(DEDUP_BLOBS_SCHEMA);
+    if (!have.has("zerobucket_image_refs")) await q.query(DEDUP_REFS_SCHEMA);
+  }
   async close() {
     this.closed = true;
     const pool = this.pool;
@@ -2400,6 +2467,19 @@ var MySQLBackend = class {
   async put(row, { connection } = {}) {
     try {
       const id = (0, import_node_crypto2.randomUUID)();
+      if (this.dedup) {
+        await this.run(
+          "put",
+          connection,
+          async (q) => {
+            const now = mysqlNow();
+            await q.execute(upsertBlobSql(1), blobParams2(row, now));
+            await q.execute(D_INSERT_REF3, refParams(id, row, now));
+          },
+          { tx: true }
+        );
+        return id;
+      }
       await this.run("put", connection, async (q) => {
         await q.execute(INSERT3, rowParams2(id, row, mysqlNow()));
       });
@@ -2417,6 +2497,21 @@ var MySQLBackend = class {
         connection,
         async (q) => {
           const now = mysqlNow();
+          if (this.dedup) {
+            const groups = /* @__PURE__ */ new Map();
+            for (const r of rows) {
+              const g = groups.get(r.checksumSha256);
+              if (g) g.n += 1;
+              else groups.set(r.checksumSha256, { row: r, n: 1 });
+            }
+            for (const checksum of [...groups.keys()].sort()) {
+              const g = groups.get(checksum);
+              await q.execute(upsertBlobSql(g.n), blobParams2(g.row, now));
+            }
+            for (let i = 0; i < rows.length; i++)
+              await q.execute(D_INSERT_REF3, refParams(ids[i], rows[i], now));
+            return;
+          }
           for (let i = 0; i < rows.length; i++)
             await q.execute(INSERT3, rowParams2(ids[i], rows[i], now));
         },
@@ -2434,7 +2529,7 @@ var MySQLBackend = class {
       row = await this.run(
         "get",
         connection,
-        async (q) => (await q.query(SELECT_FULL3, [id]))[0][0]
+        async (q) => (await q.query(this.sql.selectFull, [id]))[0][0]
       );
     } catch (exc) {
       throw wrap("Failed to retrieve image", exc);
@@ -2449,7 +2544,7 @@ var MySQLBackend = class {
       rows = await this.run(
         "get_many",
         connection,
-        async (q) => (await q.query(SELECT_MANY2, [valid]))[0]
+        async (q) => (await q.query(this.sql.selectMany, [valid]))[0]
       );
     } catch (exc) {
       throw wrap("Failed to retrieve image batch", exc);
@@ -2463,7 +2558,7 @@ var MySQLBackend = class {
       row = await this.run(
         "get_metadata",
         connection,
-        async (q) => (await q.query(SELECT_METADATA3, [id]))[0][0]
+        async (q) => (await q.query(this.sql.selectMetadata, [id]))[0][0]
       );
     } catch (exc) {
       throw wrap("Failed to retrieve image metadata", exc);
@@ -2489,7 +2584,7 @@ var MySQLBackend = class {
       info = await this.run(
         "get_metadata",
         connection,
-        async (q) => (await q.query(STREAM_INFO2, [id]))[0][0]
+        async (q) => (await q.query(this.sql.streamInfo, [id]))[0][0]
       );
     } catch (exc) {
       throw wrap("Failed to retrieve image metadata", exc);
@@ -2522,7 +2617,7 @@ var MySQLBackend = class {
           row = await self.run(
             "get_stream",
             connection,
-            async (q) => (await q.query(CHUNK2, [offset + 1, length, id]))[0][0]
+            async (q) => (await q.query(self.sql.chunk, [offset + 1, length, id]))[0][0]
           );
         } catch (exc) {
           throw wrap("Failed to stream image", exc);
@@ -2547,6 +2642,7 @@ var MySQLBackend = class {
   }
   async delete(id, { connection } = {}) {
     if (!isValidId(id)) return false;
+    if (this.dedup) return this.deleteDedup(id, connection);
     try {
       const row = await this.run(
         "delete",
@@ -2569,14 +2665,14 @@ var MySQLBackend = class {
   async deleteMany(ids, { connection } = {}) {
     const valid = [...new Set(ids.filter(isValidId))];
     if (valid.length === 0) return [];
+    if (this.dedup) return this.deleteManyDedup(valid, connection);
     try {
       const rows = await this.run(
         "delete_many",
         connection,
         async (q) => {
           const found = (await q.query(LOCK_MANY, [valid]))[0];
-          if (found.length === 0) return [];
-          await q.query(DELETE_MANY2, [found.map((r) => r.id)]);
+          for (const r of found) await q.query(DELETE_ONE2, [r.id]);
           return found;
         },
         { tx: true }
@@ -2596,12 +2692,163 @@ var MySQLBackend = class {
       return await this.run(
         "exists",
         connection,
-        async (q) => (await q.query(EXISTS3, [id]))[0].length > 0
+        async (q) => (await q.query(this.sql.exists, [id]))[0].length > 0
       );
     } catch (exc) {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- maintenance (CLI) -----------------------------------------------
+  /** Storage statistics, or null if the tables do not exist yet (nothing is created). */
+  async getInfo() {
+    try {
+      return await this.run("info", void 0, async (q) => {
+        const need = this.dedup ? ["zerobucket_image_refs", "zerobucket_blobs"] : ["zerobucket_images"];
+        const present = new Set(
+          (await q.query(
+            "SELECT TABLE_NAME AS t FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?)",
+            [need]
+          ))[0].map((r) => r.t)
+        );
+        if (!need.every((t) => present.has(t))) return null;
+        const j = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
+        const size = this.dedup ? "b.size_bytes" : "size_bytes";
+        const created = this.dedup ? "r.created_at" : "created_at";
+        const mime = this.dedup ? "b.mime_type" : "mime_type";
+        const tot = (await q.query(
+          `SELECT COUNT(*) AS n, COALESCE(SUM(${size}), 0) AS bytes, CAST(MIN(${created}) AS CHAR) AS oldest, CAST(MAX(${created}) AS CHAR) AS newest ${j}`
+        ))[0][0];
+        const disk = (await q.query(
+          "SELECT COALESCE(SUM(DATA_LENGTH + INDEX_LENGTH), 0) AS s FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?)",
+          [need]
+        ))[0][0];
+        const fmt = (await q.query(
+          `SELECT ${mime} AS m, COUNT(*) AS n, COALESCE(SUM(${size}), 0) AS bytes ${j} GROUP BY ${mime} ORDER BY COUNT(*) DESC, ${mime}`
+        ))[0];
+        const utc = (v) => v ? `${v}+00:00` : null;
+        const info = {
+          mode: this.dedup ? "dedup" : "classic",
+          count: Number(tot.n),
+          totalBytes: Number(tot.bytes),
+          oldest: utc(tot.oldest),
+          newest: utc(tot.newest),
+          onDisk: prettySize(Number(disk.s)),
+          byFormat: fmt.map((r) => ({
+            mimeType: r.m,
+            count: Number(r.n),
+            bytes: Number(r.bytes)
+          }))
+        };
+        if (this.dedup) {
+          const b = (await q.query(
+            "SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes FROM zerobucket_blobs"
+          ))[0][0];
+          info.dedup = { blobs: Number(b.n), storedBytes: Number(b.bytes) };
+        } else {
+          const t = (await q.query(
+            "SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
+          ))[0][0];
+          info.tiered = { count: Number(t.n), bytes: Number(t.bytes) };
+        }
+        return info;
+      });
+    } catch (exc) {
+      throw wrap("Failed to read storage info", exc);
+    }
+  }
+  /** Ids matching the filters: oldest first, or a random sample. Used by the CLI's verify and tier. */
+  async listImages(o = {}) {
+    const where = [];
+    const params = [];
+    const created = this.dedup ? "r.created_at" : "created_at";
+    if (o.minSize !== void 0) {
+      where.push(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= ?`);
+      params.push(o.minSize);
+    }
+    if (o.olderThanDays !== void 0) {
+      where.push(`${created} <= UTC_TIMESTAMP(6) - INTERVAL ? DAY`);
+      params.push(o.olderThanDays);
+    }
+    if (o.onlyUntiered && !this.dedup)
+      where.push(`storage_backend = '${LOCAL}'`);
+    const from = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
+    const idCol = this.dedup ? "r.id" : "id";
+    const tierCol = this.dedup ? "0 AS tiered" : "(storage_backend = 'object_storage') AS tiered";
+    const order = o.sample !== void 0 ? "ORDER BY RAND()" : `ORDER BY ${created}, ${idCol}`;
+    const lim = o.sample ?? o.limit;
+    let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
+    if (lim !== void 0) {
+      if (!Number.isInteger(lim) || lim < 0)
+        throw new RangeError("limit/sample must be a non-negative integer");
+      sql += ` LIMIT ${lim}`;
+    }
+    try {
+      const rows = await this.run(
+        "list_images",
+        void 0,
+        async (q) => (await q.query(sql, params))[0]
+      );
+      return rows.map((r) => ({
+        id: String(r.id),
+        tiered: Boolean(Number(r.tiered))
+      }));
+    } catch (exc) {
+      throw wrap("Failed to list images", exc);
+    }
+  }
+  // ---- dedup deletes ---------------------------------------------------
+  /**
+   * Lock the ref row, delete it, then decrement its blob (the UPDATE takes the blob row lock, so two
+   * deletes of different refs to one blob serialize instead of racing) and remove the blob once nothing
+   * references it. All in one transaction: a failure can never leave a ref without its count or the reverse.
+   */
+  async deleteDedup(id, connection) {
+    try {
+      return await this.run(
+        "delete",
+        connection,
+        async (q) => {
+          const ref = (await q.query(D_LOCK_REF, [id]))[0][0];
+          if (!ref) return false;
+          const checksum = ref.checksum_sha256;
+          await q.query(D_DELETE_REF3, [id]);
+          await q.query(D_DECREMENT3, [1, checksum]);
+          await q.query(D_DELETE_EMPTY3, [checksum]);
+          return true;
+        },
+        { tx: true }
+      );
+    } catch (exc) {
+      throw wrap("Failed to delete image", exc);
+    }
+  }
+  async deleteManyDedup(ids, connection) {
+    try {
+      return await this.run(
+        "delete_many",
+        connection,
+        async (q) => {
+          const found = (await q.query(D_LOCK_REFS, [ids]))[0];
+          if (found.length === 0) return [];
+          for (const r of found) await q.query(D_DELETE_REF3, [r.id]);
+          const counts = /* @__PURE__ */ new Map();
+          for (const r of found)
+            counts.set(
+              r.checksum_sha256,
+              (counts.get(r.checksum_sha256) ?? 0) + 1
+            );
+          for (const checksum of [...counts.keys()].sort()) {
+            await q.query(D_DECREMENT3, [counts.get(checksum), checksum]);
+            await q.query(D_DELETE_EMPTY3, [checksum]);
+          }
+          return found.map((r) => String(r.id));
+        },
+        { tx: true }
+      );
+    } catch (exc) {
+      throw wrap("Failed to delete image batch", exc);
+    }
+  }
   // ---- helpers ---------------------------------------------------------
   hydrate(row, id) {
     return hydrateRow(row, id, this.opts.objectStorage);
@@ -2648,6 +2895,11 @@ var MySQLBackend = class {
    * id) makes the re-upload an idempotent overwrite.
    */
   async tierToObjectStorage(id, { connection } = {}) {
+    if (this.dedup) {
+      throw new StorageError(
+        "tierToObjectStorage() is not supported in dedup mode (a blob may be shared by many images)."
+      );
+    }
     const os = this.opts.objectStorage;
     if (!os) {
       throw new StorageError(
@@ -2675,6 +2927,20 @@ var MySQLBackend = class {
     }
   }
 };
+function blobParams2(r, now) {
+  return [
+    r.checksumSha256,
+    r.data,
+    r.mimeType,
+    r.sizeBytes,
+    r.width,
+    r.height,
+    now
+  ];
+}
+function refParams(id, r, now) {
+  return [id, r.checksumSha256, r.originalFilename, now, now];
+}
 function rowParams2(id, r, now) {
   return [
     id,