zerobucket 0.10.0 → 0.11.0

package/dist/index.cjs

@@ -255,17 +255,17 @@ INSERT INTO zerobucket_images
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 RETURNING id;`;
 var SELECT_FULL = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = $1;`;
 var SELECT_MANY = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = ANY($1::uuid[]);`;
 var SELECT_METADATA = `
 SELECT id, mime_type, original_filename, size_bytes, width, height, checksum_sha256
 FROM zerobucket_images WHERE id = $1;`;
-var SELECT_CHUNK = `SELECT substring(data FROM $1::int FOR $2::int) AS chunk FROM zerobucket_images WHERE id = $3;`;
+var SELECT_CHUNK = `SELECT encode(substring(data FROM $1::int FOR $2::int), 'base64') AS chunk FROM zerobucket_images WHERE id = $3;`;
 var SELECT_STREAM_INFO = `SELECT size_bytes, storage_backend, object_storage_key FROM zerobucket_images WHERE id = $1;`;
 var DELETE_RETURNING = `DELETE FROM zerobucket_images WHERE id = $1 RETURNING storage_backend, object_storage_key;`;
 var DELETE_MANY = `DELETE FROM zerobucket_images WHERE id = ANY($1::uuid[]) RETURNING id, storage_backend, object_storage_key;`;
@@ -298,7 +298,7 @@ INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, widt
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 ON CONFLICT (checksum_sha256) DO UPDATE SET ref_count = zerobucket_blobs.ref_count + EXCLUDED.ref_count;`;
 var D_INSERT_REF = `INSERT INTO zerobucket_image_refs (checksum_sha256, original_filename) VALUES ($1, $2) RETURNING id;`;
-var D_COLS = `r.id, b.data, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
+var D_COLS = `r.id, encode(b.data, 'base64') AS data_b64, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
        'postgres'::text AS storage_backend, NULL::text AS object_storage_bucket, NULL::text AS object_storage_key`;
 var D_FROM = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
 var D_DELETE_REF = `DELETE FROM zerobucket_image_refs WHERE id = $1 RETURNING checksum_sha256;`;
@@ -320,7 +320,7 @@ var DEDUP_SQL = {
   selectMany: `SELECT ${D_COLS} ${D_FROM} WHERE r.id = ANY($1::uuid[]);`,
   selectMetadata: `SELECT r.id, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256 ${D_FROM} WHERE r.id = $1;`,
   selectStreamInfo: `SELECT b.size_bytes, 'postgres'::text AS storage_backend, NULL::text AS object_storage_key ${D_FROM} WHERE r.id = $1;`,
-  selectChunk: `SELECT substring(b.data FROM $1::int FOR $2::int) AS chunk ${D_FROM} WHERE r.id = $3;`,
+  selectChunk: `SELECT encode(substring(b.data FROM $1::int FOR $2::int), 'base64') AS chunk ${D_FROM} WHERE r.id = $3;`,
   exists: `SELECT 1 FROM zerobucket_image_refs WHERE id = $1;`
 };
 var RETRYABLE_SQLSTATES = /* @__PURE__ */ new Set([
@@ -777,7 +777,7 @@ var PostgresBackend = class {
             `Image ${JSON.stringify(id)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
           );
         }
-        const chunk = row.chunk;
+        const chunk = row.chunk == null ? null : Buffer.from(row.chunk, "base64");
         if (!chunk || chunk.length === 0) {
           throw new StorageError(
             `Image ${JSON.stringify(id)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
@@ -1101,6 +1101,8 @@ var PostgresBackend = class {
   }
   // ---- helpers ---------------------------------------------------------
   hydrate(row, id) {
+    row.data = row.data_b64 == null ? null : Buffer.from(row.data_b64, "base64");
+    delete row.data_b64;
     return hydrateRow(row, id, this.opts.objectStorage);
   }
   /**
@@ -1798,6 +1800,125 @@ var SQLiteBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- migration: classic -> dedup --------------------------------------
+  /**
+   * One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
+   * dedup instance's tables, preserving every id (external references keep working) and
+   * deduplicating identical content on the way. The classic table is never modified. Same contract
+   * as `PostgresBackend.migrateClassicToDedup()`.
+   *
+   * Rows are copied in small batches (bounded memory instead of loading the whole table);
+   * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+   * already tiered to object storage are refused up front with a clear message (their bytes are
+   * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
+   * All-or-nothing: ONE `BEGIN IMMEDIATE` transaction, so a failure part-way leaves the dedup
+   * tables exactly as they were. That transaction holds SQLite's single write lock for the whole
+   * copy, so other writers to this database file wait until it finishes (readers are not blocked).
+   */
+  async migrateClassicToDedup(opts = {}) {
+    if (!this.dedup) {
+      throw new StorageError(
+        "migrateClassicToDedup() must be called on an instance created with dedup: true."
+      );
+    }
+    try {
+      return await this.execute(
+        "migrate_to_dedup",
+        void 0,
+        "write",
+        (db) => {
+          if (!db.prepare(
+            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'zerobucket_images'"
+          ).get()) {
+            throw new StorageError(
+              "No classic zerobucket_images table found in this database -- nothing to migrate."
+            );
+          }
+          let tieredSkipped = 0;
+          const hasTierCol = db.prepare("PRAGMA table_info(zerobucket_images)").all().some((c) => c.name === "storage_backend");
+          if (hasTierCol) {
+            const tiered = db.prepare(
+              "SELECT count(*) AS n FROM zerobucket_images WHERE storage_backend <> 'sqlite'"
+            ).get().n;
+            if (tiered > 0 && !opts.skipTiered) {
+              throw new StorageError(
+                `${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table, bring those images back first, or pass skipTiered: true (CLI: --skip-tiered) to skip them and migrate the rest.`
+              );
+            }
+            tieredSkipped = tiered;
+          }
+          const onlyInDb = hasTierCol && tieredSkipped > 0;
+          const seen = /* @__PURE__ */ new Set();
+          let migrated = 0;
+          let already = 0;
+          let cursor = null;
+          const BATCH = 20;
+          const columns = "id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at, updated_at";
+          for (; ; ) {
+            const page = cursor ? db.prepare(
+              `SELECT ${columns} FROM zerobucket_images WHERE (created_at, id) > (?, ?)${onlyInDb ? " AND storage_backend = 'sqlite'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
+            ).all(cursor.createdAt, cursor.id) : db.prepare(
+              `SELECT ${columns} FROM zerobucket_images${onlyInDb ? " WHERE storage_backend = 'sqlite'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
+            ).all();
+            if (page.length === 0) break;
+            const last = page[page.length - 1];
+            if (cursor && cursor.createdAt === last.created_at && cursor.id === last.id) {
+              throw new StorageError(
+                "Migration made no progress (keyset cursor did not advance); aborting instead of looping."
+              );
+            }
+            cursor = { createdAt: last.created_at, id: last.id };
+            const have = new Set(
+              db.prepare(
+                `SELECT id FROM zerobucket_image_refs WHERE id IN (${placeholders(page.length)})`
+              ).all(...page.map((r) => r.id)).map((r) => r.id)
+            );
+            const fresh = page.filter((r) => !have.has(r.id));
+            already += page.length - fresh.length;
+            const groups = /* @__PURE__ */ new Map();
+            for (const r of fresh) {
+              const g = groups.get(r.checksum_sha256);
+              if (g) g.n += 1;
+              else groups.set(r.checksum_sha256, { row: r, n: 1 });
+            }
+            for (const checksum of [...groups.keys()].sort()) {
+              const { row, n } = groups.get(checksum);
+              db.prepare(D_UPSERT).run(
+                checksum,
+                row.data,
+                row.mime_type,
+                row.size_bytes,
+                row.width,
+                row.height,
+                n,
+                row.created_at
+              );
+            }
+            for (const r of fresh) {
+              db.prepare(D_INSERT_REF2).run(
+                r.id,
+                r.checksum_sha256,
+                r.original_filename,
+                r.created_at,
+                r.updated_at
+              );
+              seen.add(r.checksum_sha256);
+              migrated += 1;
+            }
+          }
+          return {
+            imagesMigrated: migrated,
+            distinctBlobsCreated: seen.size,
+            duplicateReferencesFound: migrated - seen.size,
+            alreadyMigrated: already,
+            tieredSkipped
+          };
+        }
+      );
+    } catch (exc) {
+      throw wrap("Migration to dedup failed", exc);
+    }
+  }
   // ---- maintenance (CLI) -----------------------------------------------
   async getInfo() {
     try {
@@ -2008,6 +2129,7 @@ CREATE TABLE IF NOT EXISTS zerobucket_images (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
 var TIERING_CHECK = "chk_zerobucket_storage";
 var MIGRATE_LOCK = "zerobucket_migrate";
+var MIGRATE_DEDUP_LOCK = "zerobucket_migrate_dedup";
 var COLS2 = `id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, storage_backend, object_storage_bucket, object_storage_key`;
 var INSERT3 = `INSERT INTO zerobucket_images (id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
 var SELECT_FULL3 = `SELECT ${COLS2} FROM zerobucket_images WHERE id = ?`;
@@ -2367,15 +2489,31 @@ var MySQLBackend = class {
   /**
    * Run `work` on a pooled connection. `tx` wraps it in a transaction. Transient failures retry the
    * WHOLE closure on a fresh connection after a full rollback.
+   *
+   * `lock` serializes this work against other callers holding the same named lock (`GET_LOCK`). The
+   * ORDER matters and is the whole point: the lock is taken BEFORE the transaction begins (so the
+   * transaction's snapshot is taken after any earlier holder has finished) and released AFTER the
+   * transaction commits or rolls back (so a later holder can never start while this one's rows are
+   * still uncommitted and invisible to it). Releasing inside `work`, before the commit, lets a second
+   * holder read a stale snapshot and collide on primary keys.
    */
-  async runPooled(operation, work, tx) {
+  async runPooled(operation, work, tx, lock) {
     const start = performance.now();
     let attempt = 0;
     for (; ; ) {
       let conn;
       let destroy = false;
+      let held = false;
       try {
         conn = await this.acquire();
+        if (lock) {
+          const got = (await conn.query("SELECT GET_LOCK(?, ?) AS got", [
+            lock.name,
+            lock.timeoutSec
+          ]))[0][0]?.got;
+          if (got !== 1) throw new StorageError(lock.timeoutMessage);
+          held = true;
+        }
         if (tx) await conn.beginTransaction();
         const result = await work(conn);
         if (tx) await conn.commit();
@@ -2397,6 +2535,13 @@ var MySQLBackend = class {
         }
       } finally {
         if (conn) {
+          if (held && lock && !destroy) {
+            try {
+              await conn.query("SELECT RELEASE_LOCK(?)", [lock.name]);
+            } catch {
+              destroy = true;
+            }
+          }
           if (destroy) conn.destroy();
           else conn.release();
         }
@@ -2410,7 +2555,10 @@ var MySQLBackend = class {
    * connection given  -> run once on it; no retry, and no commit/rollback of the caller's transaction.
    * connection absent -> pooled, retried (see runPooled).
    */
-  async run(operation, connection, work, { tx = false } = {}) {
+  async run(operation, connection, work, {
+    tx = false,
+    lock
+  } = {}) {
     if (connection) {
       if (!isMysqlConnection(connection)) {
         throw new TypeError(
@@ -2428,7 +2576,7 @@ var MySQLBackend = class {
       }
     }
     await this.ready();
-    return this.runPooled(operation, work, tx);
+    return this.runPooled(operation, work, tx, lock);
   }
   /**
    * Make multi-statement work all-or-nothing on a CALLER-supplied connection.
@@ -2702,6 +2850,139 @@ var MySQLBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- migration: classic -> dedup --------------------------------------
+  /**
+   * One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
+   * dedup instance's tables, preserving every id (external references keep working) and
+   * deduplicating identical content on the way. The classic table is never modified. Same contract
+   * as `PostgresBackend.migrateClassicToDedup()`.
+   *
+   * Rows are copied in small batches (bounded memory instead of loading the whole table);
+   * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+   * already tiered to object storage are refused up front with a clear message (their bytes are
+   * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
+   * All-or-nothing: ONE transaction, so a failure part-way leaves the dedup tables exactly as they
+   * were. A named lock (`GET_LOCK`) serializes two migrations started at once; it is taken before the
+   * transaction and released only after it commits (see `runPooled`), so a second migration always sees
+   * the first one's rows.
+   */
+  async migrateClassicToDedup(opts = {}) {
+    if (!this.dedup) {
+      throw new StorageError(
+        "migrateClassicToDedup() must be called on an instance created with dedup: true."
+      );
+    }
+    try {
+      return await this.run(
+        "migrate_to_dedup",
+        void 0,
+        (q) => this.copyClassicToDedup(q, opts),
+        {
+          tx: true,
+          lock: {
+            name: MIGRATE_DEDUP_LOCK,
+            timeoutSec: 30,
+            timeoutMessage: "Timed out waiting for another migration to dedup to finish."
+          }
+        }
+      );
+    } catch (exc) {
+      throw wrap("Migration to dedup failed", exc);
+    }
+  }
+  async copyClassicToDedup(q, opts) {
+    const rowsOf = async (sql, params) => (await q.query(sql, params))[0];
+    if ((await rowsOf(
+      "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images'"
+    )).length === 0) {
+      throw new StorageError(
+        "No classic zerobucket_images table found in this database -- nothing to migrate."
+      );
+    }
+    let tieredSkipped = 0;
+    const hasTierCol = (await rowsOf(
+      "SELECT 1 FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images' AND COLUMN_NAME = 'storage_backend'"
+    )).length > 0;
+    if (hasTierCol) {
+      const tiered = Number(
+        (await rowsOf(
+          `SELECT COUNT(*) AS n FROM zerobucket_images WHERE storage_backend <> '${LOCAL}'`
+        ))[0].n
+      );
+      if (tiered > 0 && !opts.skipTiered) {
+        throw new StorageError(
+          `${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table, bring those images back first, or pass skipTiered: true (CLI: --skip-tiered) to skip them and migrate the rest.`
+        );
+      }
+      tieredSkipped = tiered;
+    }
+    const onlyInDb = hasTierCol && tieredSkipped > 0;
+    const seen = /* @__PURE__ */ new Set();
+    let migrated = 0;
+    let already = 0;
+    let cursor = null;
+    const BATCH = 20;
+    const columns = "id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, CAST(created_at AS CHAR) AS created_at_txt, CAST(updated_at AS CHAR) AS updated_at_txt";
+    for (; ; ) {
+      const page = cursor ? await rowsOf(
+        `SELECT ${columns} FROM zerobucket_images WHERE (created_at, id) > (?, ?)${onlyInDb ? ` AND storage_backend = '${LOCAL}'` : ""} ORDER BY created_at, id LIMIT ${BATCH}`,
+        [cursor.createdAt, cursor.id]
+      ) : await rowsOf(
+        `SELECT ${columns} FROM zerobucket_images${onlyInDb ? ` WHERE storage_backend = '${LOCAL}'` : ""} ORDER BY created_at, id LIMIT ${BATCH}`
+      );
+      if (page.length === 0) break;
+      const last = page[page.length - 1];
+      if (cursor && cursor.createdAt === last.created_at_txt && cursor.id === last.id) {
+        throw new StorageError(
+          "Migration made no progress (keyset cursor did not advance); aborting instead of looping."
+        );
+      }
+      cursor = { createdAt: last.created_at_txt, id: last.id };
+      const have = new Set(
+        (await rowsOf("SELECT id FROM zerobucket_image_refs WHERE id IN (?)", [
+          page.map((r) => r.id)
+        ])).map((r) => String(r.id))
+      );
+      const fresh = page.filter((r) => !have.has(String(r.id)));
+      already += page.length - fresh.length;
+      const groups = /* @__PURE__ */ new Map();
+      for (const r of fresh) {
+        const g = groups.get(r.checksum_sha256);
+        if (g) g.n += 1;
+        else groups.set(r.checksum_sha256, { row: r, n: 1 });
+      }
+      for (const checksum of [...groups.keys()].sort()) {
+        const { row, n } = groups.get(checksum);
+        await q.query(upsertBlobSql(n), [
+          checksum,
+          row.data,
+          row.mime_type,
+          row.size_bytes,
+          row.width,
+          row.height,
+          row.created_at_txt
+        ]);
+      }
+      for (const r of fresh) {
+        await q.query(D_INSERT_REF3, [
+          r.id,
+          r.checksum_sha256,
+          r.original_filename,
+          r.created_at_txt,
+          r.updated_at_txt
+        ]);
+        seen.add(r.checksum_sha256);
+        migrated += 1;
+      }
+    }
+    return {
+      imagesMigrated: migrated,
+      distinctBlobsCreated: seen.size,
+      duplicateReferencesFound: migrated - seen.size,
+      alreadyMigrated: already,
+      tieredSkipped
+    };
+  }
   // ---- maintenance (CLI) -----------------------------------------------
   /** Storage statistics, or null if the tables do not exist yet (nothing is created). */
   async getInfo() {