zerobucket 0.10.1 → 0.11.0

package/dist/index.d.cts

@@ -250,6 +250,21 @@ declare class SQLiteBackend implements StorageBackend {
     delete(id: string, { connection }?: CallOptions): Promise<boolean>;
     deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
     exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    /**
+     * One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
+     * dedup instance's tables, preserving every id (external references keep working) and
+     * deduplicating identical content on the way. The classic table is never modified. Same contract
+     * as `PostgresBackend.migrateClassicToDedup()`.
+     *
+     * Rows are copied in small batches (bounded memory instead of loading the whole table);
+     * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+     * already tiered to object storage are refused up front with a clear message (their bytes are
+     * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
+     * All-or-nothing: ONE `BEGIN IMMEDIATE` transaction, so a failure part-way leaves the dedup
+     * tables exactly as they were. That transaction holds SQLite's single write lock for the whole
+     * copy, so other writers to this database file wait until it finishes (readers are not blocked).
+     */
+    migrateClassicToDedup(opts?: MigrateToDedupOptions): Promise<MigrateToDedupResult>;
     getInfo(): Promise<StorageInfo | null>;
     listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
     /**
@@ -327,6 +342,13 @@ declare class MySQLBackend implements StorageBackend {
     /**
      * Run `work` on a pooled connection. `tx` wraps it in a transaction. Transient failures retry the
      * WHOLE closure on a fresh connection after a full rollback.
+     *
+     * `lock` serializes this work against other callers holding the same named lock (`GET_LOCK`). The
+     * ORDER matters and is the whole point: the lock is taken BEFORE the transaction begins (so the
+     * transaction's snapshot is taken after any earlier holder has finished) and released AFTER the
+     * transaction commits or rolls back (so a later holder can never start while this one's rows are
+     * still uncommitted and invisible to it). Releasing inside `work`, before the commit, lets a second
+     * holder read a stale snapshot and collide on primary keys.
      */
     private runPooled;
     /**
@@ -354,6 +376,23 @@ declare class MySQLBackend implements StorageBackend {
     delete(id: string, { connection }?: CallOptions): Promise<boolean>;
     deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
     exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    /**
+     * One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
+     * dedup instance's tables, preserving every id (external references keep working) and
+     * deduplicating identical content on the way. The classic table is never modified. Same contract
+     * as `PostgresBackend.migrateClassicToDedup()`.
+     *
+     * Rows are copied in small batches (bounded memory instead of loading the whole table);
+     * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+     * already tiered to object storage are refused up front with a clear message (their bytes are
+     * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
+     * All-or-nothing: ONE transaction, so a failure part-way leaves the dedup tables exactly as they
+     * were. A named lock (`GET_LOCK`) serializes two migrations started at once; it is taken before the
+     * transaction and released only after it commits (see `runPooled`), so a second migration always sees
+     * the first one's rows.
+     */
+    migrateClassicToDedup(opts?: MigrateToDedupOptions): Promise<MigrateToDedupResult>;
+    private copyClassicToDedup;
     /** Storage statistics, or null if the tables do not exist yet (nothing is created). */
     getInfo(): Promise<StorageInfo | null>;
     /** Ids matching the filters: oldest first, or a random sample. Used by the CLI's verify and tier. */

package/dist/index.d.ts

@@ -250,6 +250,21 @@ declare class SQLiteBackend implements StorageBackend {
     delete(id: string, { connection }?: CallOptions): Promise<boolean>;
     deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
     exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    /**
+     * One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
+     * dedup instance's tables, preserving every id (external references keep working) and
+     * deduplicating identical content on the way. The classic table is never modified. Same contract
+     * as `PostgresBackend.migrateClassicToDedup()`.
+     *
+     * Rows are copied in small batches (bounded memory instead of loading the whole table);
+     * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+     * already tiered to object storage are refused up front with a clear message (their bytes are
+     * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
+     * All-or-nothing: ONE `BEGIN IMMEDIATE` transaction, so a failure part-way leaves the dedup
+     * tables exactly as they were. That transaction holds SQLite's single write lock for the whole
+     * copy, so other writers to this database file wait until it finishes (readers are not blocked).
+     */
+    migrateClassicToDedup(opts?: MigrateToDedupOptions): Promise<MigrateToDedupResult>;
     getInfo(): Promise<StorageInfo | null>;
     listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
     /**
@@ -327,6 +342,13 @@ declare class MySQLBackend implements StorageBackend {
     /**
      * Run `work` on a pooled connection. `tx` wraps it in a transaction. Transient failures retry the
      * WHOLE closure on a fresh connection after a full rollback.
+     *
+     * `lock` serializes this work against other callers holding the same named lock (`GET_LOCK`). The
+     * ORDER matters and is the whole point: the lock is taken BEFORE the transaction begins (so the
+     * transaction's snapshot is taken after any earlier holder has finished) and released AFTER the
+     * transaction commits or rolls back (so a later holder can never start while this one's rows are
+     * still uncommitted and invisible to it). Releasing inside `work`, before the commit, lets a second
+     * holder read a stale snapshot and collide on primary keys.
      */
     private runPooled;
     /**
@@ -354,6 +376,23 @@ declare class MySQLBackend implements StorageBackend {
     delete(id: string, { connection }?: CallOptions): Promise<boolean>;
     deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
     exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    /**
+     * One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
+     * dedup instance's tables, preserving every id (external references keep working) and
+     * deduplicating identical content on the way. The classic table is never modified. Same contract
+     * as `PostgresBackend.migrateClassicToDedup()`.
+     *
+     * Rows are copied in small batches (bounded memory instead of loading the whole table);
+     * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+     * already tiered to object storage are refused up front with a clear message (their bytes are
+     * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
+     * All-or-nothing: ONE transaction, so a failure part-way leaves the dedup tables exactly as they
+     * were. A named lock (`GET_LOCK`) serializes two migrations started at once; it is taken before the
+     * transaction and released only after it commits (see `runPooled`), so a second migration always sees
+     * the first one's rows.
+     */
+    migrateClassicToDedup(opts?: MigrateToDedupOptions): Promise<MigrateToDedupResult>;
+    private copyClassicToDedup;
     /** Storage statistics, or null if the tables do not exist yet (nothing is created). */
     getInfo(): Promise<StorageInfo | null>;
     /** Ids matching the filters: oldest first, or a random sample. Used by the CLI's verify and tier. */

package/dist/index.js

@@ -1695,6 +1695,125 @@ var SQLiteBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- migration: classic -> dedup --------------------------------------
+  /**
+   * One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
+   * dedup instance's tables, preserving every id (external references keep working) and
+   * deduplicating identical content on the way. The classic table is never modified. Same contract
+   * as `PostgresBackend.migrateClassicToDedup()`.
+   *
+   * Rows are copied in small batches (bounded memory instead of loading the whole table);
+   * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+   * already tiered to object storage are refused up front with a clear message (their bytes are
+   * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
+   * All-or-nothing: ONE `BEGIN IMMEDIATE` transaction, so a failure part-way leaves the dedup
+   * tables exactly as they were. That transaction holds SQLite's single write lock for the whole
+   * copy, so other writers to this database file wait until it finishes (readers are not blocked).
+   */
+  async migrateClassicToDedup(opts = {}) {
+    if (!this.dedup) {
+      throw new StorageError(
+        "migrateClassicToDedup() must be called on an instance created with dedup: true."
+      );
+    }
+    try {
+      return await this.execute(
+        "migrate_to_dedup",
+        void 0,
+        "write",
+        (db) => {
+          if (!db.prepare(
+            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'zerobucket_images'"
+          ).get()) {
+            throw new StorageError(
+              "No classic zerobucket_images table found in this database -- nothing to migrate."
+            );
+          }
+          let tieredSkipped = 0;
+          const hasTierCol = db.prepare("PRAGMA table_info(zerobucket_images)").all().some((c) => c.name === "storage_backend");
+          if (hasTierCol) {
+            const tiered = db.prepare(
+              "SELECT count(*) AS n FROM zerobucket_images WHERE storage_backend <> 'sqlite'"
+            ).get().n;
+            if (tiered > 0 && !opts.skipTiered) {
+              throw new StorageError(
+                `${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table, bring those images back first, or pass skipTiered: true (CLI: --skip-tiered) to skip them and migrate the rest.`
+              );
+            }
+            tieredSkipped = tiered;
+          }
+          const onlyInDb = hasTierCol && tieredSkipped > 0;
+          const seen = /* @__PURE__ */ new Set();
+          let migrated = 0;
+          let already = 0;
+          let cursor = null;
+          const BATCH = 20;
+          const columns = "id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at, updated_at";
+          for (; ; ) {
+            const page = cursor ? db.prepare(
+              `SELECT ${columns} FROM zerobucket_images WHERE (created_at, id) > (?, ?)${onlyInDb ? " AND storage_backend = 'sqlite'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
+            ).all(cursor.createdAt, cursor.id) : db.prepare(
+              `SELECT ${columns} FROM zerobucket_images${onlyInDb ? " WHERE storage_backend = 'sqlite'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
+            ).all();
+            if (page.length === 0) break;
+            const last = page[page.length - 1];
+            if (cursor && cursor.createdAt === last.created_at && cursor.id === last.id) {
+              throw new StorageError(
+                "Migration made no progress (keyset cursor did not advance); aborting instead of looping."
+              );
+            }
+            cursor = { createdAt: last.created_at, id: last.id };
+            const have = new Set(
+              db.prepare(
+                `SELECT id FROM zerobucket_image_refs WHERE id IN (${placeholders(page.length)})`
+              ).all(...page.map((r) => r.id)).map((r) => r.id)
+            );
+            const fresh = page.filter((r) => !have.has(r.id));
+            already += page.length - fresh.length;
+            const groups = /* @__PURE__ */ new Map();
+            for (const r of fresh) {
+              const g = groups.get(r.checksum_sha256);
+              if (g) g.n += 1;
+              else groups.set(r.checksum_sha256, { row: r, n: 1 });
+            }
+            for (const checksum of [...groups.keys()].sort()) {
+              const { row, n } = groups.get(checksum);
+              db.prepare(D_UPSERT).run(
+                checksum,
+                row.data,
+                row.mime_type,
+                row.size_bytes,
+                row.width,
+                row.height,
+                n,
+                row.created_at
+              );
+            }
+            for (const r of fresh) {
+              db.prepare(D_INSERT_REF2).run(
+                r.id,
+                r.checksum_sha256,
+                r.original_filename,
+                r.created_at,
+                r.updated_at
+              );
+              seen.add(r.checksum_sha256);
+              migrated += 1;
+            }
+          }
+          return {
+            imagesMigrated: migrated,
+            distinctBlobsCreated: seen.size,
+            duplicateReferencesFound: migrated - seen.size,
+            alreadyMigrated: already,
+            tieredSkipped
+          };
+        }
+      );
+    } catch (exc) {
+      throw wrap("Migration to dedup failed", exc);
+    }
+  }
   // ---- maintenance (CLI) -----------------------------------------------
   async getInfo() {
     try {
@@ -1905,6 +2024,7 @@ CREATE TABLE IF NOT EXISTS zerobucket_images (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`;
 var TIERING_CHECK = "chk_zerobucket_storage";
 var MIGRATE_LOCK = "zerobucket_migrate";
+var MIGRATE_DEDUP_LOCK = "zerobucket_migrate_dedup";
 var COLS2 = `id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, storage_backend, object_storage_bucket, object_storage_key`;
 var INSERT3 = `INSERT INTO zerobucket_images (id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
 var SELECT_FULL3 = `SELECT ${COLS2} FROM zerobucket_images WHERE id = ?`;
@@ -2264,15 +2384,31 @@ var MySQLBackend = class {
   /**
    * Run `work` on a pooled connection. `tx` wraps it in a transaction. Transient failures retry the
    * WHOLE closure on a fresh connection after a full rollback.
+   *
+   * `lock` serializes this work against other callers holding the same named lock (`GET_LOCK`). The
+   * ORDER matters and is the whole point: the lock is taken BEFORE the transaction begins (so the
+   * transaction's snapshot is taken after any earlier holder has finished) and released AFTER the
+   * transaction commits or rolls back (so a later holder can never start while this one's rows are
+   * still uncommitted and invisible to it). Releasing inside `work`, before the commit, lets a second
+   * holder read a stale snapshot and collide on primary keys.
    */
-  async runPooled(operation, work, tx) {
+  async runPooled(operation, work, tx, lock) {
     const start = performance.now();
     let attempt = 0;
     for (; ; ) {
       let conn;
       let destroy = false;
+      let held = false;
       try {
         conn = await this.acquire();
+        if (lock) {
+          const got = (await conn.query("SELECT GET_LOCK(?, ?) AS got", [
+            lock.name,
+            lock.timeoutSec
+          ]))[0][0]?.got;
+          if (got !== 1) throw new StorageError(lock.timeoutMessage);
+          held = true;
+        }
         if (tx) await conn.beginTransaction();
         const result = await work(conn);
         if (tx) await conn.commit();
@@ -2294,6 +2430,13 @@ var MySQLBackend = class {
         }
       } finally {
         if (conn) {
+          if (held && lock && !destroy) {
+            try {
+              await conn.query("SELECT RELEASE_LOCK(?)", [lock.name]);
+            } catch {
+              destroy = true;
+            }
+          }
           if (destroy) conn.destroy();
           else conn.release();
         }
@@ -2307,7 +2450,10 @@ var MySQLBackend = class {
    * connection given  -> run once on it; no retry, and no commit/rollback of the caller's transaction.
    * connection absent -> pooled, retried (see runPooled).
    */
-  async run(operation, connection, work, { tx = false } = {}) {
+  async run(operation, connection, work, {
+    tx = false,
+    lock
+  } = {}) {
     if (connection) {
       if (!isMysqlConnection(connection)) {
         throw new TypeError(
@@ -2325,7 +2471,7 @@ var MySQLBackend = class {
       }
     }
     await this.ready();
-    return this.runPooled(operation, work, tx);
+    return this.runPooled(operation, work, tx, lock);
   }
   /**
    * Make multi-statement work all-or-nothing on a CALLER-supplied connection.
@@ -2599,6 +2745,139 @@ var MySQLBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- migration: classic -> dedup --------------------------------------
+  /**
+   * One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
+   * dedup instance's tables, preserving every id (external references keep working) and
+   * deduplicating identical content on the way. The classic table is never modified. Same contract
+   * as `PostgresBackend.migrateClassicToDedup()`.
+   *
+   * Rows are copied in small batches (bounded memory instead of loading the whole table);
+   * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+   * already tiered to object storage are refused up front with a clear message (their bytes are
+   * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
+   * All-or-nothing: ONE transaction, so a failure part-way leaves the dedup tables exactly as they
+   * were. A named lock (`GET_LOCK`) serializes two migrations started at once; it is taken before the
+   * transaction and released only after it commits (see `runPooled`), so a second migration always sees
+   * the first one's rows.
+   */
+  async migrateClassicToDedup(opts = {}) {
+    if (!this.dedup) {
+      throw new StorageError(
+        "migrateClassicToDedup() must be called on an instance created with dedup: true."
+      );
+    }
+    try {
+      return await this.run(
+        "migrate_to_dedup",
+        void 0,
+        (q) => this.copyClassicToDedup(q, opts),
+        {
+          tx: true,
+          lock: {
+            name: MIGRATE_DEDUP_LOCK,
+            timeoutSec: 30,
+            timeoutMessage: "Timed out waiting for another migration to dedup to finish."
+          }
+        }
+      );
+    } catch (exc) {
+      throw wrap("Migration to dedup failed", exc);
+    }
+  }
+  async copyClassicToDedup(q, opts) {
+    const rowsOf = async (sql, params) => (await q.query(sql, params))[0];
+    if ((await rowsOf(
+      "SELECT 1 FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images'"
+    )).length === 0) {
+      throw new StorageError(
+        "No classic zerobucket_images table found in this database -- nothing to migrate."
+      );
+    }
+    let tieredSkipped = 0;
+    const hasTierCol = (await rowsOf(
+      "SELECT 1 FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'zerobucket_images' AND COLUMN_NAME = 'storage_backend'"
+    )).length > 0;
+    if (hasTierCol) {
+      const tiered = Number(
+        (await rowsOf(
+          `SELECT COUNT(*) AS n FROM zerobucket_images WHERE storage_backend <> '${LOCAL}'`
+        ))[0].n
+      );
+      if (tiered > 0 && !opts.skipTiered) {
+        throw new StorageError(
+          `${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table, bring those images back first, or pass skipTiered: true (CLI: --skip-tiered) to skip them and migrate the rest.`
+        );
+      }
+      tieredSkipped = tiered;
+    }
+    const onlyInDb = hasTierCol && tieredSkipped > 0;
+    const seen = /* @__PURE__ */ new Set();
+    let migrated = 0;
+    let already = 0;
+    let cursor = null;
+    const BATCH = 20;
+    const columns = "id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, CAST(created_at AS CHAR) AS created_at_txt, CAST(updated_at AS CHAR) AS updated_at_txt";
+    for (; ; ) {
+      const page = cursor ? await rowsOf(
+        `SELECT ${columns} FROM zerobucket_images WHERE (created_at, id) > (?, ?)${onlyInDb ? ` AND storage_backend = '${LOCAL}'` : ""} ORDER BY created_at, id LIMIT ${BATCH}`,
+        [cursor.createdAt, cursor.id]
+      ) : await rowsOf(
+        `SELECT ${columns} FROM zerobucket_images${onlyInDb ? ` WHERE storage_backend = '${LOCAL}'` : ""} ORDER BY created_at, id LIMIT ${BATCH}`
+      );
+      if (page.length === 0) break;
+      const last = page[page.length - 1];
+      if (cursor && cursor.createdAt === last.created_at_txt && cursor.id === last.id) {
+        throw new StorageError(
+          "Migration made no progress (keyset cursor did not advance); aborting instead of looping."
+        );
+      }
+      cursor = { createdAt: last.created_at_txt, id: last.id };
+      const have = new Set(
+        (await rowsOf("SELECT id FROM zerobucket_image_refs WHERE id IN (?)", [
+          page.map((r) => r.id)
+        ])).map((r) => String(r.id))
+      );
+      const fresh = page.filter((r) => !have.has(String(r.id)));
+      already += page.length - fresh.length;
+      const groups = /* @__PURE__ */ new Map();
+      for (const r of fresh) {
+        const g = groups.get(r.checksum_sha256);
+        if (g) g.n += 1;
+        else groups.set(r.checksum_sha256, { row: r, n: 1 });
+      }
+      for (const checksum of [...groups.keys()].sort()) {
+        const { row, n } = groups.get(checksum);
+        await q.query(upsertBlobSql(n), [
+          checksum,
+          row.data,
+          row.mime_type,
+          row.size_bytes,
+          row.width,
+          row.height,
+          row.created_at_txt
+        ]);
+      }
+      for (const r of fresh) {
+        await q.query(D_INSERT_REF3, [
+          r.id,
+          r.checksum_sha256,
+          r.original_filename,
+          r.created_at_txt,
+          r.updated_at_txt
+        ]);
+        seen.add(r.checksum_sha256);
+        migrated += 1;
+      }
+    }
+    return {
+      imagesMigrated: migrated,
+      distinctBlobsCreated: seen.size,
+      duplicateReferencesFound: migrated - seen.size,
+      alreadyMigrated: already,
+      tieredSkipped
+    };
+  }
   // ---- maintenance (CLI) -----------------------------------------------
   /** Storage statistics, or null if the tables do not exist yet (nothing is created). */
   async getInfo() {