zerobucket 0.9.1 → 0.10.1

package/README.md

@@ -456,25 +456,26 @@ same corpus of valid and corrupt files identically.
 Python column = zerobucket on PyPI **0.26.0 or later** (several of its behaviours below were fixed in 0.25.1 and 0.26.0; older
 versions differ).
 
-|                                              | Python                                                                | npm                                                                                                                                 |
-| -------------------------------------------- | --------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
-| API                                          | sync + async clients                                                  | one async client                                                                                                                    |
-| `optimize`                                   | Pillow; EXIF stripped, not applied                                    | sharp; EXIF **applied then stripped**; transparency → **white** for JPEG                                                            |
-| JPEG encoder in `optimize`                   | Pillow, progressive + optimized                                       | sharp with `mozjpeg`, progressive: on smooth photos roughly **half the size** at about the same SSIM (measured); WebP and PNG match |
-| `putMany` inserts                            | pipelined                                                             | one transaction, statements run sequentially                                                                                        |
-| Formats                                      | JPEG, PNG, WebP, HEIC (`[heic]` extra)                                | JPEG, PNG, WebP, HEIC (optional `libheif-js`)                                                                                       |
-| Convert TO HEIC with `optimize`              | yes (pillow-heif)                                                     | no (no HEVC encoder for Node); converting FROM HEIC works; `optimize: true` on a HEIC needs an explicit `format`                    |
-| Event duration                               | `duration_seconds`                                                    | `durationMs`                                                                                                                        |
-| `migrateClassicToDedup`, tiered classic rows | always **skips** them and reports `tiered_rows_skipped`               | **refuses** by default (changes nothing); `skipTiered: true` / `--skip-tiered` skips and reports like Python                        |
-| `migrateClassicToDedup`, other behaviour     | batched, safe to re-run (server-side cursor)                          | batched, safe to re-run (keyset paging, advisory lock so concurrent runs cannot race)                                               |
-| SQLite API                                   | synchronous `SQLiteBackend` + separate async backend                  | one async client; the sync driver runs on the event loop, lock waits are async (never block it)                                     |
-| SQLite `onOperation` / busy retry            | none                                                                  | `onOperation` events and async `SQLITE_BUSY` retry (`busyTimeoutMs`)                                                                |
-| SQLite stream deleted mid-read               | sync adapter keeps reading a WAL snapshot (async adapter raises)      | raises `StorageError`, like Postgres; use `connection` in a transaction for a snapshot                                              |
-| MySQL `onOperation` / retry                  | sync `MySQLBackend` yes; `AsyncMySQLBackend` none                     | yes (same client for everything)                                                                                                    |
-| CLI                                          | PostgreSQL, classic mode only; `verify` skips tiered rows with a note | PostgreSQL, SQLite **and** MySQL/MariaDB, classic **and** dedup; `verify --bucket` also checks tiered images; `migrate --to-dedup`  |
-| `connection=` without an open transaction    | multi-statement ops rely on the driver's implicit transaction         | always atomic (own transaction or savepoint)                                                                                        |
-| S3 delete failure after `delete()`           | raises after the row is gone                                          | never fails the delete; reported via `onOperation`                                                                                  |
-| Options                                      | snake_case kwargs                                                     | camelCase options object                                                                                                            |
+|                                                                | Python                                                                | npm                                                                                                                                 |
+| -------------------------------------------------------------- | --------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
+| API                                                            | sync + async clients                                                  | one async client                                                                                                                    |
+| `optimize`                                                     | Pillow; EXIF stripped, not applied                                    | sharp; EXIF **applied then stripped**; transparency → **white** for JPEG                                                            |
+| JPEG encoder in `optimize`                                     | Pillow, progressive + optimized                                       | sharp with `mozjpeg`, progressive: on smooth photos roughly **half the size** at about the same SSIM (measured); WebP and PNG match |
+| JPEG missing only its last 1-2 bytes (the end-of-image marker) | accepted (Pillow reads every pixel)                                   | **rejected** (libvips treats a premature end as an error); truncated interlaced PNGs are rejected by both since npm 0.10.0          |
+| `putMany` inserts                                              | pipelined                                                             | one transaction, statements run sequentially                                                                                        |
+| Formats                                                        | JPEG, PNG, WebP, HEIC (`[heic]` extra)                                | JPEG, PNG, WebP, HEIC (optional `libheif-js`)                                                                                       |
+| Convert TO HEIC with `optimize`                                | yes (pillow-heif)                                                     | no (no HEVC encoder for Node); converting FROM HEIC works; `optimize: true` on a HEIC needs an explicit `format`                    |
+| Event duration                                                 | `duration_seconds`                                                    | `durationMs`                                                                                                                        |
+| `migrateClassicToDedup`, tiered classic rows                   | always **skips** them and reports `tiered_rows_skipped`               | **refuses** by default (changes nothing); `skipTiered: true` / `--skip-tiered` skips and reports like Python                        |
+| `migrateClassicToDedup`, other behaviour                       | batched, safe to re-run (server-side cursor)                          | batched, safe to re-run (keyset paging, advisory lock so concurrent runs cannot race)                                               |
+| SQLite API                                                     | synchronous `SQLiteBackend` + separate async backend                  | one async client; the sync driver runs on the event loop, lock waits are async (never block it)                                     |
+| SQLite `onOperation` / busy retry                              | none                                                                  | `onOperation` events and async `SQLITE_BUSY` retry (`busyTimeoutMs`)                                                                |
+| SQLite stream deleted mid-read                                 | sync adapter keeps reading a WAL snapshot (async adapter raises)      | raises `StorageError`, like Postgres; use `connection` in a transaction for a snapshot                                              |
+| MySQL `onOperation` / retry                                    | sync `MySQLBackend` yes; `AsyncMySQLBackend` none                     | yes (same client for everything)                                                                                                    |
+| CLI                                                            | PostgreSQL, classic mode only; `verify` skips tiered rows with a note | PostgreSQL, SQLite **and** MySQL/MariaDB, classic **and** dedup; `verify --bucket` also checks tiered images; `migrate --to-dedup`  |
+| `connection=` without an open transaction                      | multi-statement ops rely on the driver's implicit transaction         | always atomic (own transaction or savepoint)                                                                                        |
+| S3 delete failure after `delete()`                             | raises after the row is gone                                          | never fails the delete; reported via `onOperation`                                                                                  |
+| Options                                                        | snake_case kwargs                                                     | camelCase options object                                                                                                            |
 
 Both packages treat a malformed id as "not found" on every backend (Python 0.26.0+, npm 0.9.0+) and never pass one to a
 `before_get`/`beforeGet` hook. Both packages now leave a tiered image's object in place on `delete(id, connection=...)` (a rolled-back transaction can
@@ -484,19 +485,26 @@ MySQL `ALTER TABLE` at startup when the schema is current.
 ## Performance
 
 Measured on one single-core container, so treat it as directional; the full tables (PostgreSQL, SQLite, MariaDB, per size,
-memory, event-loop impact, throughput) are in the
+memory, event-loop impact, throughput, and the before and after of each release) are in the
 [benchmark results](https://github.com/KedarGhadyalji/ZeroBucket/blob/main/benchmarks/node/RESULTS.md).
 
-| Image size   | `put()` (PostgreSQL) | `get()`     | Notes                                                               |
-| ------------ | -------------------- | ----------- | ------------------------------------------------------------------- |
-| up to ~500KB | under 60ms           | under 5ms   | Feels like any other write.                                         |
-| 0.5-1.5MB    | about 60-160ms       | under 10ms  | Fine for uploads.                                                   |
-| about 5MB    | about 0.6s           | about 45ms  | About 58MB of transient memory per upload.                          |
-| about 10MB   | about 1.2-1.4s       | about 120ms | About 130MB of transient memory per upload. Default limit is 8 MiB. |
-
-- `put()` time is mostly image validation (a full decode), so it follows the image's pixel count, not the database you pick.
-- The decode runs off the main thread: the event loop stalled at most about 16ms during a 10MB `put()`.
-- Memory is the decoded image, roughly 11x the file size for a 45-megapixel JPEG. Cap `maxBytes` and `maxPixels` on public endpoints.
+| Image size   | `put()` (PostgreSQL) | `get()` (PostgreSQL) | Notes                                                             |
+| ------------ | -------------------- | -------------------- | ----------------------------------------------------------------- |
+| up to ~500KB | under 20ms           | under 5ms            | Feels like any other write.                                       |
+| 0.5-1.5MB    | about 20-45ms        | under 10ms           | Fine for uploads, including in a loop.                            |
+| about 5MB    | about 170ms          | about 35ms           | A few MB of transient memory to write a baseline JPEG or PNG.     |
+| about 10MB   | about 350ms          | about 70ms           | Default limit is 8 MiB. Memory depends on the format (see below). |
+
+- Since 0.10.0, `put()` is 2.5-4x faster than 0.9.x and about as fast as the Python package for large images. Validation is now
+  roughly half of `put()` on PostgreSQL.
+- Since 0.10.1, reading from PostgreSQL asks for base64 instead of PostgreSQL's default hex text: about 30% faster for images over
+  2MB, and `get()` needs 37-47% less memory (about 6.5 times the image size, down from about 10). Streaming, which is how
+  `createImageHandler` serves images, is bounded in memory either way. Between roughly 0.5MB and 1.3MB there is no reliable
+  difference (a cost of under a millisecond may exist). SQLite and MySQL/MariaDB were already fast readers and are unchanged.
+- The decode runs off the main thread: the event loop stalled at most about 14ms during a 10MB `put()`.
+- Transient memory depends on the format. A baseline JPEG or non-interlaced PNG validates in streaming fashion (about 11MB for a
+  45-megapixel JPEG, down from about 148MB in 0.9.x). WebP, progressive JPEG and interlaced PNG still need roughly the decoded
+  image in memory, so cap `maxBytes` and `maxPixels` on public endpoints.
 - Stored size equals the file size: JPEG, PNG and WebP are already compressed, so the database cannot shrink them further.
 - Compression quality of `optimize` is measured in the
   [compression results](https://github.com/KedarGhadyalji/ZeroBucket/blob/main/benchmarks/node/COMPRESSION_RESULTS.md).

package/dist/bin.js

@@ -150,17 +150,17 @@ INSERT INTO zerobucket_images
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 RETURNING id;`;
 var SELECT_FULL = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = $1;`;
 var SELECT_MANY = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = ANY($1::uuid[]);`;
 var SELECT_METADATA = `
 SELECT id, mime_type, original_filename, size_bytes, width, height, checksum_sha256
 FROM zerobucket_images WHERE id = $1;`;
-var SELECT_CHUNK = `SELECT substring(data FROM $1::int FOR $2::int) AS chunk FROM zerobucket_images WHERE id = $3;`;
+var SELECT_CHUNK = `SELECT encode(substring(data FROM $1::int FOR $2::int), 'base64') AS chunk FROM zerobucket_images WHERE id = $3;`;
 var SELECT_STREAM_INFO = `SELECT size_bytes, storage_backend, object_storage_key FROM zerobucket_images WHERE id = $1;`;
 var DELETE_RETURNING = `DELETE FROM zerobucket_images WHERE id = $1 RETURNING storage_backend, object_storage_key;`;
 var DELETE_MANY = `DELETE FROM zerobucket_images WHERE id = ANY($1::uuid[]) RETURNING id, storage_backend, object_storage_key;`;
@@ -193,7 +193,7 @@ INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, widt
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 ON CONFLICT (checksum_sha256) DO UPDATE SET ref_count = zerobucket_blobs.ref_count + EXCLUDED.ref_count;`;
 var D_INSERT_REF = `INSERT INTO zerobucket_image_refs (checksum_sha256, original_filename) VALUES ($1, $2) RETURNING id;`;
-var D_COLS = `r.id, b.data, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
+var D_COLS = `r.id, encode(b.data, 'base64') AS data_b64, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
        'postgres'::text AS storage_backend, NULL::text AS object_storage_bucket, NULL::text AS object_storage_key`;
 var D_FROM = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
 var D_DELETE_REF = `DELETE FROM zerobucket_image_refs WHERE id = $1 RETURNING checksum_sha256;`;
@@ -215,7 +215,7 @@ var DEDUP_SQL = {
   selectMany: `SELECT ${D_COLS} ${D_FROM} WHERE r.id = ANY($1::uuid[]);`,
   selectMetadata: `SELECT r.id, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256 ${D_FROM} WHERE r.id = $1;`,
   selectStreamInfo: `SELECT b.size_bytes, 'postgres'::text AS storage_backend, NULL::text AS object_storage_key ${D_FROM} WHERE r.id = $1;`,
-  selectChunk: `SELECT substring(b.data FROM $1::int FOR $2::int) AS chunk ${D_FROM} WHERE r.id = $3;`,
+  selectChunk: `SELECT encode(substring(b.data FROM $1::int FOR $2::int), 'base64') AS chunk ${D_FROM} WHERE r.id = $3;`,
   exists: `SELECT 1 FROM zerobucket_image_refs WHERE id = $1;`
 };
 var RETRYABLE_SQLSTATES = /* @__PURE__ */ new Set([
@@ -672,7 +672,7 @@ var PostgresBackend = class {
             `Image ${JSON.stringify(id)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
           );
         }
-        const chunk = row.chunk;
+        const chunk = row.chunk == null ? null : Buffer.from(row.chunk, "base64");
         if (!chunk || chunk.length === 0) {
           throw new StorageError(
             `Image ${JSON.stringify(id)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
@@ -996,6 +996,8 @@ var PostgresBackend = class {
   }
   // ---- helpers ---------------------------------------------------------
   hydrate(row, id) {
+    row.data = row.data_b64 == null ? null : Buffer.from(row.data_b64, "base64");
+    delete row.data_b64;
     return hydrateRow(row, id, this.opts.objectStorage);
   }
   /**

package/dist/index.cjs

@@ -255,17 +255,17 @@ INSERT INTO zerobucket_images
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 RETURNING id;`;
 var SELECT_FULL = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = $1;`;
 var SELECT_MANY = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = ANY($1::uuid[]);`;
 var SELECT_METADATA = `
 SELECT id, mime_type, original_filename, size_bytes, width, height, checksum_sha256
 FROM zerobucket_images WHERE id = $1;`;
-var SELECT_CHUNK = `SELECT substring(data FROM $1::int FOR $2::int) AS chunk FROM zerobucket_images WHERE id = $3;`;
+var SELECT_CHUNK = `SELECT encode(substring(data FROM $1::int FOR $2::int), 'base64') AS chunk FROM zerobucket_images WHERE id = $3;`;
 var SELECT_STREAM_INFO = `SELECT size_bytes, storage_backend, object_storage_key FROM zerobucket_images WHERE id = $1;`;
 var DELETE_RETURNING = `DELETE FROM zerobucket_images WHERE id = $1 RETURNING storage_backend, object_storage_key;`;
 var DELETE_MANY = `DELETE FROM zerobucket_images WHERE id = ANY($1::uuid[]) RETURNING id, storage_backend, object_storage_key;`;
@@ -298,7 +298,7 @@ INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, widt
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 ON CONFLICT (checksum_sha256) DO UPDATE SET ref_count = zerobucket_blobs.ref_count + EXCLUDED.ref_count;`;
 var D_INSERT_REF = `INSERT INTO zerobucket_image_refs (checksum_sha256, original_filename) VALUES ($1, $2) RETURNING id;`;
-var D_COLS = `r.id, b.data, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
+var D_COLS = `r.id, encode(b.data, 'base64') AS data_b64, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
        'postgres'::text AS storage_backend, NULL::text AS object_storage_bucket, NULL::text AS object_storage_key`;
 var D_FROM = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
 var D_DELETE_REF = `DELETE FROM zerobucket_image_refs WHERE id = $1 RETURNING checksum_sha256;`;
@@ -320,7 +320,7 @@ var DEDUP_SQL = {
   selectMany: `SELECT ${D_COLS} ${D_FROM} WHERE r.id = ANY($1::uuid[]);`,
   selectMetadata: `SELECT r.id, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256 ${D_FROM} WHERE r.id = $1;`,
   selectStreamInfo: `SELECT b.size_bytes, 'postgres'::text AS storage_backend, NULL::text AS object_storage_key ${D_FROM} WHERE r.id = $1;`,
-  selectChunk: `SELECT substring(b.data FROM $1::int FOR $2::int) AS chunk ${D_FROM} WHERE r.id = $3;`,
+  selectChunk: `SELECT encode(substring(b.data FROM $1::int FOR $2::int), 'base64') AS chunk ${D_FROM} WHERE r.id = $3;`,
   exists: `SELECT 1 FROM zerobucket_image_refs WHERE id = $1;`
 };
 var RETRYABLE_SQLSTATES = /* @__PURE__ */ new Set([
@@ -777,7 +777,7 @@ var PostgresBackend = class {
             `Image ${JSON.stringify(id)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
           );
         }
-        const chunk = row.chunk;
+        const chunk = row.chunk == null ? null : Buffer.from(row.chunk, "base64");
         if (!chunk || chunk.length === 0) {
           throw new StorageError(
             `Image ${JSON.stringify(id)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
@@ -1101,6 +1101,8 @@ var PostgresBackend = class {
   }
   // ---- helpers ---------------------------------------------------------
   hydrate(row, id) {
+    row.data = row.data_b64 == null ? null : Buffer.from(row.data_b64, "base64");
+    delete row.data_b64;
     return hydrateRow(row, id, this.opts.objectStorage);
   }
   /**
@@ -3197,14 +3199,16 @@ async function validateImage(data, opts) {
   if (!width || !height)
     throw new CorruptedImageError("Image has no readable dimensions");
   try {
-    await make().stats();
+    await sharp(data, {
+      limitInputPixels: maxPixels,
+      failOn: "error",
+      sequentialRead: true
+    }).extract({ left: 0, top: 0, width: 1, height }).raw().toBuffer();
   } catch (exc) {
     if (isPixelLimit(exc)) throw new ImageTooLargeError(sizeBytes, maxBytes);
     throw new CorruptedImageError(
-      `Image data is truncated or corrupted: ${errMessage(exc)}`,
-      {
-        cause: exc
-      }
+      `Image data is truncated or corrupted: ${decodeDetail(exc)}`,
+      { cause: exc }
     );
   }
   return { mimeType: FORMAT_TO_MIME[format], width, height, sizeBytes };
@@ -3248,6 +3252,10 @@ async function validateHeic(data, {
   }
   return { mimeType: FORMAT_TO_MIME.heic, width, height, sizeBytes };
 }
+function decodeDetail(exc) {
+  const msg3 = errMessage(exc).trim();
+  return /^unknown error$/i.test(msg3) ? "the decoder stopped before the end of the image" : msg3;
+}
 function isPixelLimit(exc) {
   return /pixel limit/i.test(errMessage(exc));
 }