zerobucket 0.10.0 → 0.10.1

package/README.md

@@ -485,18 +485,22 @@ MySQL `ALTER TABLE` at startup when the schema is current.
 ## Performance
 
 Measured on one single-core container, so treat it as directional; the full tables (PostgreSQL, SQLite, MariaDB, per size,
-memory, event-loop impact, throughput, and the 0.9.0 to 0.10.0 before and after) are in the
+memory, event-loop impact, throughput, and the before and after of each release) are in the
 [benchmark results](https://github.com/KedarGhadyalji/ZeroBucket/blob/main/benchmarks/node/RESULTS.md).
 
-| Image size   | `put()` (PostgreSQL) | `get()`     | Notes                                                             |
-| ------------ | -------------------- | ----------- | ----------------------------------------------------------------- |
-| up to ~500KB | under 20ms           | under 5ms   | Feels like any other write.                                       |
-| 0.5-1.5MB    | about 20-45ms        | under 10ms  | Fine for uploads, including in a loop.                            |
-| about 5MB    | about 170ms          | about 50ms  | A few MB of transient memory for baseline JPEG and PNG.           |
-| about 10MB   | about 350ms          | about 135ms | Default limit is 8 MiB. Memory depends on the format (see below). |
+| Image size   | `put()` (PostgreSQL) | `get()` (PostgreSQL) | Notes                                                             |
+| ------------ | -------------------- | -------------------- | ----------------------------------------------------------------- |
+| up to ~500KB | under 20ms           | under 5ms            | Feels like any other write.                                       |
+| 0.5-1.5MB    | about 20-45ms        | under 10ms           | Fine for uploads, including in a loop.                            |
+| about 5MB    | about 170ms          | about 35ms           | A few MB of transient memory to write a baseline JPEG or PNG.     |
+| about 10MB   | about 350ms          | about 70ms           | Default limit is 8 MiB. Memory depends on the format (see below). |
 
 - Since 0.10.0, `put()` is 2.5-4x faster than 0.9.x and about as fast as the Python package for large images. Validation is now
   roughly half of `put()` on PostgreSQL.
+- Since 0.10.1, reading from PostgreSQL asks for base64 instead of PostgreSQL's default hex text: about 30% faster for images over
+  2MB, and `get()` needs 37-47% less memory (about 6.5 times the image size, down from about 10). Streaming, which is how
+  `createImageHandler` serves images, is bounded in memory either way. Between roughly 0.5MB and 1.3MB there is no reliable
+  difference (a cost of under a millisecond may exist). SQLite and MySQL/MariaDB were already fast readers and are unchanged.
 - The decode runs off the main thread: the event loop stalled at most about 14ms during a 10MB `put()`.
 - Transient memory depends on the format. A baseline JPEG or non-interlaced PNG validates in streaming fashion (about 11MB for a
   45-megapixel JPEG, down from about 148MB in 0.9.x). WebP, progressive JPEG and interlaced PNG still need roughly the decoded

package/dist/bin.js

@@ -150,17 +150,17 @@ INSERT INTO zerobucket_images
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 RETURNING id;`;
 var SELECT_FULL = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = $1;`;
 var SELECT_MANY = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = ANY($1::uuid[]);`;
 var SELECT_METADATA = `
 SELECT id, mime_type, original_filename, size_bytes, width, height, checksum_sha256
 FROM zerobucket_images WHERE id = $1;`;
-var SELECT_CHUNK = `SELECT substring(data FROM $1::int FOR $2::int) AS chunk FROM zerobucket_images WHERE id = $3;`;
+var SELECT_CHUNK = `SELECT encode(substring(data FROM $1::int FOR $2::int), 'base64') AS chunk FROM zerobucket_images WHERE id = $3;`;
 var SELECT_STREAM_INFO = `SELECT size_bytes, storage_backend, object_storage_key FROM zerobucket_images WHERE id = $1;`;
 var DELETE_RETURNING = `DELETE FROM zerobucket_images WHERE id = $1 RETURNING storage_backend, object_storage_key;`;
 var DELETE_MANY = `DELETE FROM zerobucket_images WHERE id = ANY($1::uuid[]) RETURNING id, storage_backend, object_storage_key;`;
@@ -193,7 +193,7 @@ INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, widt
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 ON CONFLICT (checksum_sha256) DO UPDATE SET ref_count = zerobucket_blobs.ref_count + EXCLUDED.ref_count;`;
 var D_INSERT_REF = `INSERT INTO zerobucket_image_refs (checksum_sha256, original_filename) VALUES ($1, $2) RETURNING id;`;
-var D_COLS = `r.id, b.data, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
+var D_COLS = `r.id, encode(b.data, 'base64') AS data_b64, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
        'postgres'::text AS storage_backend, NULL::text AS object_storage_bucket, NULL::text AS object_storage_key`;
 var D_FROM = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
 var D_DELETE_REF = `DELETE FROM zerobucket_image_refs WHERE id = $1 RETURNING checksum_sha256;`;
@@ -215,7 +215,7 @@ var DEDUP_SQL = {
   selectMany: `SELECT ${D_COLS} ${D_FROM} WHERE r.id = ANY($1::uuid[]);`,
   selectMetadata: `SELECT r.id, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256 ${D_FROM} WHERE r.id = $1;`,
   selectStreamInfo: `SELECT b.size_bytes, 'postgres'::text AS storage_backend, NULL::text AS object_storage_key ${D_FROM} WHERE r.id = $1;`,
-  selectChunk: `SELECT substring(b.data FROM $1::int FOR $2::int) AS chunk ${D_FROM} WHERE r.id = $3;`,
+  selectChunk: `SELECT encode(substring(b.data FROM $1::int FOR $2::int), 'base64') AS chunk ${D_FROM} WHERE r.id = $3;`,
   exists: `SELECT 1 FROM zerobucket_image_refs WHERE id = $1;`
 };
 var RETRYABLE_SQLSTATES = /* @__PURE__ */ new Set([
@@ -672,7 +672,7 @@ var PostgresBackend = class {
             `Image ${JSON.stringify(id)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
           );
         }
-        const chunk = row.chunk;
+        const chunk = row.chunk == null ? null : Buffer.from(row.chunk, "base64");
         if (!chunk || chunk.length === 0) {
           throw new StorageError(
             `Image ${JSON.stringify(id)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
@@ -996,6 +996,8 @@ var PostgresBackend = class {
   }
   // ---- helpers ---------------------------------------------------------
   hydrate(row, id) {
+    row.data = row.data_b64 == null ? null : Buffer.from(row.data_b64, "base64");
+    delete row.data_b64;
     return hydrateRow(row, id, this.opts.objectStorage);
   }
   /**

package/dist/index.cjs

@@ -255,17 +255,17 @@ INSERT INTO zerobucket_images
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 RETURNING id;`;
 var SELECT_FULL = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = $1;`;
 var SELECT_MANY = `
-SELECT id, data, mime_type, original_filename, size_bytes, width, height,
+SELECT id, encode(data, 'base64') AS data_b64, mime_type, original_filename, size_bytes, width, height,
        checksum_sha256, storage_backend, object_storage_bucket, object_storage_key
 FROM zerobucket_images WHERE id = ANY($1::uuid[]);`;
 var SELECT_METADATA = `
 SELECT id, mime_type, original_filename, size_bytes, width, height, checksum_sha256
 FROM zerobucket_images WHERE id = $1;`;
-var SELECT_CHUNK = `SELECT substring(data FROM $1::int FOR $2::int) AS chunk FROM zerobucket_images WHERE id = $3;`;
+var SELECT_CHUNK = `SELECT encode(substring(data FROM $1::int FOR $2::int), 'base64') AS chunk FROM zerobucket_images WHERE id = $3;`;
 var SELECT_STREAM_INFO = `SELECT size_bytes, storage_backend, object_storage_key FROM zerobucket_images WHERE id = $1;`;
 var DELETE_RETURNING = `DELETE FROM zerobucket_images WHERE id = $1 RETURNING storage_backend, object_storage_key;`;
 var DELETE_MANY = `DELETE FROM zerobucket_images WHERE id = ANY($1::uuid[]) RETURNING id, storage_backend, object_storage_key;`;
@@ -298,7 +298,7 @@ INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, widt
 VALUES ($1, $2, $3, $4, $5, $6, $7)
 ON CONFLICT (checksum_sha256) DO UPDATE SET ref_count = zerobucket_blobs.ref_count + EXCLUDED.ref_count;`;
 var D_INSERT_REF = `INSERT INTO zerobucket_image_refs (checksum_sha256, original_filename) VALUES ($1, $2) RETURNING id;`;
-var D_COLS = `r.id, b.data, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
+var D_COLS = `r.id, encode(b.data, 'base64') AS data_b64, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
        'postgres'::text AS storage_backend, NULL::text AS object_storage_bucket, NULL::text AS object_storage_key`;
 var D_FROM = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
 var D_DELETE_REF = `DELETE FROM zerobucket_image_refs WHERE id = $1 RETURNING checksum_sha256;`;
@@ -320,7 +320,7 @@ var DEDUP_SQL = {
   selectMany: `SELECT ${D_COLS} ${D_FROM} WHERE r.id = ANY($1::uuid[]);`,
   selectMetadata: `SELECT r.id, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256 ${D_FROM} WHERE r.id = $1;`,
   selectStreamInfo: `SELECT b.size_bytes, 'postgres'::text AS storage_backend, NULL::text AS object_storage_key ${D_FROM} WHERE r.id = $1;`,
-  selectChunk: `SELECT substring(b.data FROM $1::int FOR $2::int) AS chunk ${D_FROM} WHERE r.id = $3;`,
+  selectChunk: `SELECT encode(substring(b.data FROM $1::int FOR $2::int), 'base64') AS chunk ${D_FROM} WHERE r.id = $3;`,
   exists: `SELECT 1 FROM zerobucket_image_refs WHERE id = $1;`
 };
 var RETRYABLE_SQLSTATES = /* @__PURE__ */ new Set([
@@ -777,7 +777,7 @@ var PostgresBackend = class {
             `Image ${JSON.stringify(id)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
           );
         }
-        const chunk = row.chunk;
+        const chunk = row.chunk == null ? null : Buffer.from(row.chunk, "base64");
         if (!chunk || chunk.length === 0) {
           throw new StorageError(
             `Image ${JSON.stringify(id)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
@@ -1101,6 +1101,8 @@ var PostgresBackend = class {
   }
   // ---- helpers ---------------------------------------------------------
   hydrate(row, id) {
+    row.data = row.data_b64 == null ? null : Buffer.from(row.data_b64, "base64");
+    delete row.data_b64;
     return hydrateRow(row, id, this.opts.objectStorage);
   }
   /**