zerobucket 0.6.1 → 0.8.0

package/dist/index.d.cts

@@ -1,5 +1,5 @@
-import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, g as ContentValidator, V as ValidatedContent } from './client-C7xs1dlr.cjs';
-export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as ObjectStorage, q as ObjectStorageOptions, r as OptimizationResult, s as OptimizeOptions, t as PutManyOptions, u as PutOptions, Q as Queryable, v as SqliteDatabaseLike, w as StreamOptions, W as WritableLike, Z as ZeroBucket, x as ZeroBucketOptions, y as optimizeImage, z as readInput } from './client-C7xs1dlr.cjs';
+import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, M as MysqlPoolLike, g as ContentValidator, V as ValidatedContent } from './client-XQ22b1l3.cjs';
+export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as MysqlPoolConnectionLike, q as MysqlQueryable, r as ObjectStorage, s as ObjectStorageOptions, t as OptimizationResult, u as OptimizeOptions, v as PutManyOptions, w as PutOptions, Q as Queryable, x as SqliteDatabaseLike, y as StreamOptions, W as WritableLike, Z as ZeroBucket, z as ZeroBucketOptions, A as optimizeImage, E as readInput } from './client-XQ22b1l3.cjs';
 import { Pool } from 'pg';
 
 /**
@@ -275,6 +275,132 @@ declare class SQLiteBackend implements StorageBackend {
     tierToObjectStorage(id: string, { connection }?: CallOptions): Promise<boolean | null>;
 }
 
+interface MySQLBackendOptions {
+    /** `mysql://user:password@host:3306/database` (`mariadb://` is accepted too). Required unless `pool` is given. */
+    connectionString?: string;
+    /** Bring your own `mysql2/promise` Pool (not closed by `close()`). Pool sizing options are then ignored. */
+    pool?: MysqlPoolLike;
+    /** Create the schema on first use. Default true. */
+    autoMigrate?: boolean;
+    /** Retries for transient errors on pooled calls. Default 3; 0 disables. */
+    maxRetries?: number;
+    /** Base for exponential backoff with jitter (capped at 2s). Default 100. */
+    retryBaseDelayMs?: number;
+    /** Accepted for parity with the other backends. mysql2 opens connections on demand, so there is no minimum to hold. */
+    poolMinSize?: number;
+    /** Maximum simultaneous connections. Default 5. */
+    poolMaxSize?: number;
+    /** Max wait for a pooled connection (also the TCP connect timeout). Default 10000. */
+    poolTimeoutMs?: number;
+    onOperation?: (event: OperationEvent) => void;
+    /** Enables transparent reads of tiered rows and `tierToObjectStorage()`. */
+    objectStorage?: ObjectStorageLike;
+    /**
+     * Content-addressed storage: identical bytes are stored once and shared by every image that has them
+     * (reference counted). Same tables as the Python package's dedup mode. Cannot be combined with
+     * `objectStorage` (a blob may be shared by many images).
+     */
+    dedup?: boolean;
+}
+declare class MySQLBackend implements StorageBackend {
+    private readonly opts;
+    private pool;
+    private readonly ownsPool;
+    private readyPromise;
+    private closed;
+    private readonly maxRetries;
+    private readonly retryBaseDelayMs;
+    private readonly poolTimeoutMs;
+    private readonly autoMigrate;
+    private readonly dedup;
+    private readonly sql;
+    constructor(opts: MySQLBackendOptions);
+    ready(): Promise<void>;
+    private init;
+    /**
+     * Create the schema if missing, under a server-side named lock so concurrent first callers (in this
+     * or any other process) run the DDL one at a time. Every step checks before it changes anything, so
+     * re-running is a no-op.
+     */
+    private migrate;
+    /** Dedup tables, created only if missing (same no-needless-DDL rule as the classic table). */
+    private migrateDedup;
+    close(): Promise<void>;
+    private emit;
+    /** Wait for a pool slot, but never longer than poolTimeoutMs. */
+    private acquire;
+    /**
+     * Run `work` on a pooled connection. `tx` wraps it in a transaction. Transient failures retry the
+     * WHOLE closure on a fresh connection after a full rollback.
+     */
+    private runPooled;
+    /**
+     * Run `work` and return its result, emitting exactly one OperationEvent.
+     *
+     * connection given  -> run once on it; no retry, and no commit/rollback of the caller's transaction.
+     * connection absent -> pooled, retried (see runPooled).
+     */
+    private run;
+    /**
+     * Make multi-statement work all-or-nothing on a CALLER-supplied connection.
+     *
+     * Is a transaction open? SAVEPOINT followed by ROLLBACK TO SAVEPOINT succeeds only inside one (in
+     * autocommit mode the savepoint is gone before the next statement and the rollback fails with 1305).
+     *  - Open  -> reuse that SAVEPOINT: a failure undoes only our work; success joins the caller's transaction.
+     *  - None  -> START TRANSACTION and COMMIT our own, so a failure can never leave half-applied state.
+     */
+    private atomic;
+    put(row: PreparedRow, { connection }?: CallOptions): Promise<string>;
+    putMany(rows: PreparedRow[], { connection }?: CallOptions): Promise<string[]>;
+    get(id: string, { connection }?: CallOptions): Promise<StoredRecord | null>;
+    getMany(ids: string[], { connection }?: CallOptions): Promise<StoredRecord[]>;
+    getMetadata(id: string, { connection }?: CallOptions): Promise<StoredRecordMetadata | null>;
+    getStream(id: string, opts: StreamOptions): Promise<AsyncIterable<Buffer> | null>;
+    delete(id: string, { connection }?: CallOptions): Promise<boolean>;
+    deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
+    exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    /** Storage statistics, or null if the tables do not exist yet (nothing is created). */
+    getInfo(): Promise<StorageInfo | null>;
+    /** Ids matching the filters: oldest first, or a random sample. Used by the CLI's verify and tier. */
+    listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
+    /**
+     * Lock the ref row, delete it, then decrement its blob (the UPDATE takes the blob row lock, so two
+     * deletes of different refs to one blob serialize instead of racing) and remove the blob once nothing
+     * references it. All in one transaction: a failure can never leave a ref without its count or the reverse.
+     */
+    private deleteDedup;
+    private deleteManyDedup;
+    private hydrate;
+    /**
+     * Remove tiered objects AFTER their rows are gone: once the row is deleted the image is correctly
+     * "not found" to everyone, so a failed object delete only leaves a harmless orphan, whereas deleting
+     * the object first and then failing the row delete would leave a row pointing at nothing.
+     *
+     * Never runs inside a caller-owned transaction (`connection`): we cannot know whether it will
+     * commit. In that case the object stays until you remove it yourself after committing. Failures are
+     * reported through onOperation and never fail the delete itself.
+     */
+    private cleanupObject;
+    /**
+     * Move an image's bytes out of the database into object storage, replacing the row's `data` with a
+     * pointer. Returns null (no such image), false (already tiered: a safe no-op, so backfills can be
+     * re-run) or true (tiered just now).
+     *
+     * SAFETY: the upload happens INSIDE the same transaction as the row lock (SELECT ... FOR UPDATE)
+     * and the UPDATE that flips the row. If the upload fails the transaction rolls back and the row is
+     * untouched: there is no moment when the bytes exist nowhere, or when a row claims to be tiered
+     * without a completed upload.
+     *
+     * TRADEOFF: InnoDB row-locks only THIS row for the duration of the upload. Concurrent writes to this
+     * image wait (up to innodb_lock_wait_timeout, 50s by default); every other row is unaffected and
+     * plain reads are never blocked. Meant for explicit maintenance, not hot paths.
+     *
+     * Retry is safe: a transient error replays the whole closure, and the deterministic key (the image
+     * id) makes the re-upload an idempotent overwrite.
+     */
+    tierToObjectStorage(id: string, { connection }?: CallOptions): Promise<boolean | null>;
+}
+
 declare const SUPPORTED_FORMATS: readonly string[];
 /** Reject images that would decode to more pixels than this, regardless of compressed size (~89 MP). */
 declare const DEFAULT_MAX_PIXELS = 89000000;
@@ -308,4 +434,4 @@ declare class PDFValidator implements ContentValidator {
     }): ValidatedContent;
 }
 
-export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };
+export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, MySQLBackend, type MySQLBackendOptions, MysqlPoolLike, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, g as ContentValidator, V as ValidatedContent } from './client-C7xs1dlr.js';
-export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as ObjectStorage, q as ObjectStorageOptions, r as OptimizationResult, s as OptimizeOptions, t as PutManyOptions, u as PutOptions, Q as Queryable, v as SqliteDatabaseLike, w as StreamOptions, W as WritableLike, Z as ZeroBucket, x as ZeroBucketOptions, y as optimizeImage, z as readInput } from './client-C7xs1dlr.js';
+import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, M as MysqlPoolLike, g as ContentValidator, V as ValidatedContent } from './client-XQ22b1l3.js';
+export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as MysqlPoolConnectionLike, q as MysqlQueryable, r as ObjectStorage, s as ObjectStorageOptions, t as OptimizationResult, u as OptimizeOptions, v as PutManyOptions, w as PutOptions, Q as Queryable, x as SqliteDatabaseLike, y as StreamOptions, W as WritableLike, Z as ZeroBucket, z as ZeroBucketOptions, A as optimizeImage, E as readInput } from './client-XQ22b1l3.js';
 import { Pool } from 'pg';
 
 /**
@@ -275,6 +275,132 @@ declare class SQLiteBackend implements StorageBackend {
     tierToObjectStorage(id: string, { connection }?: CallOptions): Promise<boolean | null>;
 }
 
+interface MySQLBackendOptions {
+    /** `mysql://user:password@host:3306/database` (`mariadb://` is accepted too). Required unless `pool` is given. */
+    connectionString?: string;
+    /** Bring your own `mysql2/promise` Pool (not closed by `close()`). Pool sizing options are then ignored. */
+    pool?: MysqlPoolLike;
+    /** Create the schema on first use. Default true. */
+    autoMigrate?: boolean;
+    /** Retries for transient errors on pooled calls. Default 3; 0 disables. */
+    maxRetries?: number;
+    /** Base for exponential backoff with jitter (capped at 2s). Default 100. */
+    retryBaseDelayMs?: number;
+    /** Accepted for parity with the other backends. mysql2 opens connections on demand, so there is no minimum to hold. */
+    poolMinSize?: number;
+    /** Maximum simultaneous connections. Default 5. */
+    poolMaxSize?: number;
+    /** Max wait for a pooled connection (also the TCP connect timeout). Default 10000. */
+    poolTimeoutMs?: number;
+    onOperation?: (event: OperationEvent) => void;
+    /** Enables transparent reads of tiered rows and `tierToObjectStorage()`. */
+    objectStorage?: ObjectStorageLike;
+    /**
+     * Content-addressed storage: identical bytes are stored once and shared by every image that has them
+     * (reference counted). Same tables as the Python package's dedup mode. Cannot be combined with
+     * `objectStorage` (a blob may be shared by many images).
+     */
+    dedup?: boolean;
+}
+declare class MySQLBackend implements StorageBackend {
+    private readonly opts;
+    private pool;
+    private readonly ownsPool;
+    private readyPromise;
+    private closed;
+    private readonly maxRetries;
+    private readonly retryBaseDelayMs;
+    private readonly poolTimeoutMs;
+    private readonly autoMigrate;
+    private readonly dedup;
+    private readonly sql;
+    constructor(opts: MySQLBackendOptions);
+    ready(): Promise<void>;
+    private init;
+    /**
+     * Create the schema if missing, under a server-side named lock so concurrent first callers (in this
+     * or any other process) run the DDL one at a time. Every step checks before it changes anything, so
+     * re-running is a no-op.
+     */
+    private migrate;
+    /** Dedup tables, created only if missing (same no-needless-DDL rule as the classic table). */
+    private migrateDedup;
+    close(): Promise<void>;
+    private emit;
+    /** Wait for a pool slot, but never longer than poolTimeoutMs. */
+    private acquire;
+    /**
+     * Run `work` on a pooled connection. `tx` wraps it in a transaction. Transient failures retry the
+     * WHOLE closure on a fresh connection after a full rollback.
+     */
+    private runPooled;
+    /**
+     * Run `work` and return its result, emitting exactly one OperationEvent.
+     *
+     * connection given  -> run once on it; no retry, and no commit/rollback of the caller's transaction.
+     * connection absent -> pooled, retried (see runPooled).
+     */
+    private run;
+    /**
+     * Make multi-statement work all-or-nothing on a CALLER-supplied connection.
+     *
+     * Is a transaction open? SAVEPOINT followed by ROLLBACK TO SAVEPOINT succeeds only inside one (in
+     * autocommit mode the savepoint is gone before the next statement and the rollback fails with 1305).
+     *  - Open  -> reuse that SAVEPOINT: a failure undoes only our work; success joins the caller's transaction.
+     *  - None  -> START TRANSACTION and COMMIT our own, so a failure can never leave half-applied state.
+     */
+    private atomic;
+    put(row: PreparedRow, { connection }?: CallOptions): Promise<string>;
+    putMany(rows: PreparedRow[], { connection }?: CallOptions): Promise<string[]>;
+    get(id: string, { connection }?: CallOptions): Promise<StoredRecord | null>;
+    getMany(ids: string[], { connection }?: CallOptions): Promise<StoredRecord[]>;
+    getMetadata(id: string, { connection }?: CallOptions): Promise<StoredRecordMetadata | null>;
+    getStream(id: string, opts: StreamOptions): Promise<AsyncIterable<Buffer> | null>;
+    delete(id: string, { connection }?: CallOptions): Promise<boolean>;
+    deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
+    exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    /** Storage statistics, or null if the tables do not exist yet (nothing is created). */
+    getInfo(): Promise<StorageInfo | null>;
+    /** Ids matching the filters: oldest first, or a random sample. Used by the CLI's verify and tier. */
+    listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
+    /**
+     * Lock the ref row, delete it, then decrement its blob (the UPDATE takes the blob row lock, so two
+     * deletes of different refs to one blob serialize instead of racing) and remove the blob once nothing
+     * references it. All in one transaction: a failure can never leave a ref without its count or the reverse.
+     */
+    private deleteDedup;
+    private deleteManyDedup;
+    private hydrate;
+    /**
+     * Remove tiered objects AFTER their rows are gone: once the row is deleted the image is correctly
+     * "not found" to everyone, so a failed object delete only leaves a harmless orphan, whereas deleting
+     * the object first and then failing the row delete would leave a row pointing at nothing.
+     *
+     * Never runs inside a caller-owned transaction (`connection`): we cannot know whether it will
+     * commit. In that case the object stays until you remove it yourself after committing. Failures are
+     * reported through onOperation and never fail the delete itself.
+     */
+    private cleanupObject;
+    /**
+     * Move an image's bytes out of the database into object storage, replacing the row's `data` with a
+     * pointer. Returns null (no such image), false (already tiered: a safe no-op, so backfills can be
+     * re-run) or true (tiered just now).
+     *
+     * SAFETY: the upload happens INSIDE the same transaction as the row lock (SELECT ... FOR UPDATE)
+     * and the UPDATE that flips the row. If the upload fails the transaction rolls back and the row is
+     * untouched: there is no moment when the bytes exist nowhere, or when a row claims to be tiered
+     * without a completed upload.
+     *
+     * TRADEOFF: InnoDB row-locks only THIS row for the duration of the upload. Concurrent writes to this
+     * image wait (up to innodb_lock_wait_timeout, 50s by default); every other row is unaffected and
+     * plain reads are never blocked. Meant for explicit maintenance, not hot paths.
+     *
+     * Retry is safe: a transient error replays the whole closure, and the deterministic key (the image
+     * id) makes the re-upload an idempotent overwrite.
+     */
+    tierToObjectStorage(id: string, { connection }?: CallOptions): Promise<boolean | null>;
+}
+
 declare const SUPPORTED_FORMATS: readonly string[];
 /** Reject images that would decode to more pixels than this, regardless of compressed size (~89 MP). */
 declare const DEFAULT_MAX_PIXELS = 89000000;
@@ -308,4 +434,4 @@ declare class PDFValidator implements ContentValidator {
     }): ValidatedContent;
 }
 
-export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };
+export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, MySQLBackend, type MySQLBackendOptions, MysqlPoolLike, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };