zerobucket 0.8.0 → 0.8.2

package/dist/index.d.cts

@@ -1,5 +1,5 @@
-import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, M as MysqlPoolLike, g as ContentValidator, V as ValidatedContent } from './client-XQ22b1l3.cjs';
-export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as MysqlPoolConnectionLike, q as MysqlQueryable, r as ObjectStorage, s as ObjectStorageOptions, t as OptimizationResult, u as OptimizeOptions, v as PutManyOptions, w as PutOptions, Q as Queryable, x as SqliteDatabaseLike, y as StreamOptions, W as WritableLike, Z as ZeroBucket, z as ZeroBucketOptions, A as optimizeImage, E as readInput } from './client-XQ22b1l3.cjs';
+import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, M as MigrateToDedupOptions, g as MigrateToDedupResult, h as MysqlPoolLike, i as ContentValidator, V as ValidatedContent } from './client-l4jPIFJI.cjs';
+export { B as BatchDeleteResult, j as BatchGetResult, k as BatchPutResult, l as BeforeGetHook, m as BeforePutHook, n as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, o as Image, p as ImageInput, q as ImageMetadata, r as MysqlPoolConnectionLike, s as MysqlQueryable, t as ObjectStorage, u as ObjectStorageOptions, v as OptimizationResult, w as OptimizeOptions, x as PutManyOptions, y as PutOptions, Q as Queryable, z as SqliteDatabaseLike, A as StreamOptions, W as WritableLike, Z as ZeroBucket, E as ZeroBucketOptions, F as optimizeImage, H as readInput } from './client-l4jPIFJI.cjs';
 import { Pool } from 'pg';
 
 /**
@@ -158,18 +158,13 @@ declare class PostgresBackend implements StorageBackend {
      * dedup instance's tables, preserving every id (external references keep working) and
      * deduplicating identical content on the way. The classic table is never modified.
      *
-     * Differences from the Python version, all deliberate: rows are copied in small batches (bounded
-     * memory instead of loading the whole table); re-running is safe (already-migrated ids are
-     * skipped and reported, not an error); and classic rows already tiered to object storage are
-     * refused up front with a clear message (their bytes are not in the database to copy).
+     * Rows are copied in small batches (bounded memory instead of loading the whole table);
+     * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+     * already tiered to object storage are refused up front with a clear message (their bytes are
+     * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
      * Still all-or-nothing: one transaction.
      */
-    migrateClassicToDedup(): Promise<{
-        imagesMigrated: number;
-        distinctBlobsCreated: number;
-        duplicateReferencesFound: number;
-        alreadyMigrated: number;
-    }>;
+    migrateClassicToDedup(opts?: MigrateToDedupOptions): Promise<MigrateToDedupResult>;
     private hydrate;
     /**
      * Remove tiered objects AFTER their rows are gone. Ordering is deliberate:
@@ -434,4 +429,4 @@ declare class PDFValidator implements ContentValidator {
     }): ValidatedContent;
 }
 
-export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, MySQLBackend, type MySQLBackendOptions, MysqlPoolLike, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };
+export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, MigrateToDedupOptions, MigrateToDedupResult, MySQLBackend, type MySQLBackendOptions, MysqlPoolLike, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, M as MysqlPoolLike, g as ContentValidator, V as ValidatedContent } from './client-XQ22b1l3.js';
-export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as MysqlPoolConnectionLike, q as MysqlQueryable, r as ObjectStorage, s as ObjectStorageOptions, t as OptimizationResult, u as OptimizeOptions, v as PutManyOptions, w as PutOptions, Q as Queryable, x as SqliteDatabaseLike, y as StreamOptions, W as WritableLike, Z as ZeroBucket, z as ZeroBucketOptions, A as optimizeImage, E as readInput } from './client-XQ22b1l3.js';
+import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, M as MigrateToDedupOptions, g as MigrateToDedupResult, h as MysqlPoolLike, i as ContentValidator, V as ValidatedContent } from './client-l4jPIFJI.js';
+export { B as BatchDeleteResult, j as BatchGetResult, k as BatchPutResult, l as BeforeGetHook, m as BeforePutHook, n as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, o as Image, p as ImageInput, q as ImageMetadata, r as MysqlPoolConnectionLike, s as MysqlQueryable, t as ObjectStorage, u as ObjectStorageOptions, v as OptimizationResult, w as OptimizeOptions, x as PutManyOptions, y as PutOptions, Q as Queryable, z as SqliteDatabaseLike, A as StreamOptions, W as WritableLike, Z as ZeroBucket, E as ZeroBucketOptions, F as optimizeImage, H as readInput } from './client-l4jPIFJI.js';
 import { Pool } from 'pg';
 
 /**
@@ -158,18 +158,13 @@ declare class PostgresBackend implements StorageBackend {
      * dedup instance's tables, preserving every id (external references keep working) and
      * deduplicating identical content on the way. The classic table is never modified.
      *
-     * Differences from the Python version, all deliberate: rows are copied in small batches (bounded
-     * memory instead of loading the whole table); re-running is safe (already-migrated ids are
-     * skipped and reported, not an error); and classic rows already tiered to object storage are
-     * refused up front with a clear message (their bytes are not in the database to copy).
+     * Rows are copied in small batches (bounded memory instead of loading the whole table);
+     * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+     * already tiered to object storage are refused up front with a clear message (their bytes are
+     * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
      * Still all-or-nothing: one transaction.
      */
-    migrateClassicToDedup(): Promise<{
-        imagesMigrated: number;
-        distinctBlobsCreated: number;
-        duplicateReferencesFound: number;
-        alreadyMigrated: number;
-    }>;
+    migrateClassicToDedup(opts?: MigrateToDedupOptions): Promise<MigrateToDedupResult>;
     private hydrate;
     /**
      * Remove tiered objects AFTER their rows are gone. Ordering is deliberate:
@@ -434,4 +429,4 @@ declare class PDFValidator implements ContentValidator {
     }): ValidatedContent;
 }
 
-export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, MySQLBackend, type MySQLBackendOptions, MysqlPoolLike, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };
+export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, MigrateToDedupOptions, MigrateToDedupResult, MySQLBackend, type MySQLBackendOptions, MysqlPoolLike, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };

package/dist/index.js

@@ -875,13 +875,13 @@ var PostgresBackend = class {
    * dedup instance's tables, preserving every id (external references keep working) and
    * deduplicating identical content on the way. The classic table is never modified.
    *
-   * Differences from the Python version, all deliberate: rows are copied in small batches (bounded
-   * memory instead of loading the whole table); re-running is safe (already-migrated ids are
-   * skipped and reported, not an error); and classic rows already tiered to object storage are
-   * refused up front with a clear message (their bytes are not in the database to copy).
+   * Rows are copied in small batches (bounded memory instead of loading the whole table);
+   * re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
+   * already tiered to object storage are refused up front with a clear message (their bytes are
+   * not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
    * Still all-or-nothing: one transaction.
    */
-  async migrateClassicToDedup() {
+  async migrateClassicToDedup(opts = {}) {
     if (!this.dedup) {
       throw new StorageError(
         "migrateClassicToDedup() must be called on an instance created with dedup: true."
@@ -900,6 +900,7 @@ var PostgresBackend = class {
           "No classic zerobucket_images table found in this database -- nothing to migrate."
         );
       }
+      let tieredSkipped = 0;
       const hasTierCol = (await client.query(
         "SELECT 1 FROM information_schema.columns WHERE table_name = 'zerobucket_images' AND column_name = 'storage_backend'"
       )).rowCount;
@@ -907,12 +908,14 @@ var PostgresBackend = class {
         const tiered = (await client.query(
           "SELECT count(*)::int AS n FROM zerobucket_images WHERE storage_backend <> 'postgres'"
         )).rows[0].n;
-        if (tiered > 0) {
+        if (tiered > 0 && !opts.skipTiered) {
           throw new StorageError(
-            `${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table, or bring those images back first.`
+            `${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table, bring those images back first, or pass skipTiered: true (CLI: --skip-tiered) to skip them and migrate the rest.`
           );
         }
+        tieredSkipped = tiered;
       }
+      const onlyInDb = hasTierCol && tieredSkipped > 0;
       const seen = /* @__PURE__ */ new Set();
       let migrated = 0;
       let already = 0;
@@ -921,11 +924,11 @@ var PostgresBackend = class {
       for (; ; ) {
         const page = cursor ? await client.query(
           `SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
-               FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid) ORDER BY created_at, id LIMIT ${BATCH}`,
+               FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid)${onlyInDb ? " AND storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`,
           [cursor.createdAt, cursor.id]
         ) : await client.query(
           `SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
-               FROM zerobucket_images ORDER BY created_at, id LIMIT ${BATCH}`
+               FROM zerobucket_images${onlyInDb ? " WHERE storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
         );
         if (page.rows.length === 0) break;
         const last = page.rows[page.rows.length - 1];
@@ -975,7 +978,8 @@ var PostgresBackend = class {
         imagesMigrated: migrated,
         distinctBlobsCreated: seen.size,
         duplicateReferencesFound: migrated - seen.size,
-        alreadyMigrated: already
+        alreadyMigrated: already,
+        tieredSkipped
       };
     } catch (exc) {
       try {
@@ -3634,13 +3638,13 @@ var ZeroBucket = class _ZeroBucket {
    * One-time, non-destructive copy of classic-mode images into a dedup instance's tables, keeping every
    * id. Call it on an instance created with `dedup: true`. Safe to re-run. See PostgresBackend for details.
    */
-  async migrateClassicToDedup() {
+  async migrateClassicToDedup(opts) {
     if (!this.backend.migrateClassicToDedup) {
       throw new StorageError(
         "This backend does not support migrateClassicToDedup()."
       );
     }
-    return this.backend.migrateClassicToDedup();
+    return this.backend.migrateClassicToDedup(opts);
   }
   /** Release the pool (if ZeroBucket created it). */
   close() {