zerobucket 0.1.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -39,8 +39,10 @@ __export(src_exports, {
39
39
  ImageNotFoundError: () => ImageNotFoundError,
40
40
  ImageTooLargeError: () => ImageTooLargeError,
41
41
  ImageValidationError: () => ImageValidationError,
42
+ ObjectStorage: () => ObjectStorage,
42
43
  PDFValidator: () => PDFValidator,
43
44
  PostgresBackend: () => PostgresBackend,
45
+ SQLiteBackend: () => SQLiteBackend,
44
46
  SUPPORTED_FORMATS: () => SUPPORTED_FORMATS,
45
47
  StorageError: () => StorageError,
46
48
  UnsupportedFormatError: () => UnsupportedFormatError,
@@ -53,7 +55,7 @@ __export(src_exports, {
53
55
  module.exports = __toCommonJS(src_exports);
54
56
 
55
57
  // src/client.ts
56
- var import_node_crypto = require("crypto");
58
+ var import_node_crypto2 = require("crypto");
57
59
  var import_promises = require("fs/promises");
58
60
  var import_node_path = require("path");
59
61
 
@@ -113,8 +115,91 @@ var AccessDeniedError = class extends ZeroBucketError {
113
115
  imageId;
114
116
  };
115
117
 
116
- // src/adapters/postgres.ts
118
+ // src/adapters/shared.ts
117
119
  var DEFAULT_STREAM_CHUNK_SIZE = 1024 * 1024;
120
+ var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
121
+ function isValidId(id) {
122
+ return typeof id === "string" && UUID_RE.test(id);
123
+ }
124
+ function msg(exc) {
125
+ return exc instanceof Error ? exc.message : String(exc);
126
+ }
127
+ function wrap(prefix, exc) {
128
+ return exc instanceof StorageError ? exc : new StorageError(`${prefix}: ${msg(exc)}`, { cause: exc });
129
+ }
130
+ function sleep(ms) {
131
+ return new Promise((resolve) => setTimeout(resolve, ms));
132
+ }
133
+ async function mapLimit(items, limit, fn) {
134
+ const out = new Array(items.length);
135
+ let next = 0;
136
+ await Promise.all(
137
+ Array.from({ length: Math.min(limit, items.length) }, async () => {
138
+ while (next < items.length) {
139
+ const i = next++;
140
+ out[i] = await fn(items[i]);
141
+ }
142
+ })
143
+ );
144
+ return out;
145
+ }
146
+ function tieredWithoutStorage(id, key) {
147
+ return new StorageError(
148
+ `Image ${JSON.stringify(id)} is stored in object storage (key=${JSON.stringify(key)}) but this ZeroBucket was constructed without \`objectStorage\`. Configure it with the same bucket and credentials that were used to tier this image.`
149
+ );
150
+ }
151
+ async function hydrateRow(row, id, os) {
152
+ let data;
153
+ if (row.storage_backend === "object_storage") {
154
+ if (!os) throw tieredWithoutStorage(id, row.object_storage_key);
155
+ data = await os.download(row.object_storage_key);
156
+ if (data.length !== row.size_bytes) {
157
+ throw new StorageError(
158
+ `Object ${JSON.stringify(row.object_storage_key)} for image ${JSON.stringify(id)} has ${data.length} bytes but the database records ${row.size_bytes}. The stored object is corrupted or was replaced.`
159
+ );
160
+ }
161
+ } else {
162
+ data = row.data;
163
+ }
164
+ return {
165
+ id: String(row.id),
166
+ data,
167
+ mimeType: row.mime_type,
168
+ originalFilename: row.original_filename,
169
+ sizeBytes: row.size_bytes,
170
+ width: row.width,
171
+ height: row.height,
172
+ checksumSha256: row.checksum_sha256
173
+ };
174
+ }
175
+ async function* streamFromObjectStorage(os, id, key, start, end, chunkSize) {
176
+ let offset = start;
177
+ while (offset <= end) {
178
+ const last = Math.min(offset + chunkSize - 1, end);
179
+ const expected = last - offset + 1;
180
+ const chunk = await os.downloadRange(key, offset, last);
181
+ if (chunk.length !== expected) {
182
+ throw new StorageError(
183
+ `Object ${JSON.stringify(key)} for image ${JSON.stringify(id)} returned ${chunk.length} bytes for a ${expected}-byte range; the stored object is shorter than the database records.`
184
+ );
185
+ }
186
+ yield chunk;
187
+ offset = last + 1;
188
+ }
189
+ }
190
+ function prettySize(bytes) {
191
+ if (bytes < 1024) return `${bytes} bytes`;
192
+ const units = ["kB", "MB", "GB", "TB"];
193
+ let v = bytes / 1024;
194
+ let i = 0;
195
+ while (v >= 1024 && i < units.length - 1) {
196
+ v /= 1024;
197
+ i += 1;
198
+ }
199
+ return `${v.toFixed(1)} ${units[i]}`;
200
+ }
201
+
202
+ // src/adapters/postgres.ts
118
203
  var SCHEMA = `
119
204
  CREATE TABLE IF NOT EXISTS zerobucket_images (
120
205
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
@@ -180,10 +265,59 @@ var SELECT_STREAM_INFO = `SELECT size_bytes, storage_backend, object_storage_key
180
265
  var DELETE_RETURNING = `DELETE FROM zerobucket_images WHERE id = $1 RETURNING storage_backend, object_storage_key;`;
181
266
  var DELETE_MANY = `DELETE FROM zerobucket_images WHERE id = ANY($1::uuid[]) RETURNING id, storage_backend, object_storage_key;`;
182
267
  var EXISTS = `SELECT 1 FROM zerobucket_images WHERE id = $1;`;
183
- var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
184
- function isValidId(id) {
185
- return typeof id === "string" && UUID_RE.test(id);
186
- }
268
+ var SELECT_FOR_TIERING = `SELECT data, mime_type, size_bytes, storage_backend FROM zerobucket_images WHERE id = $1 FOR UPDATE;`;
269
+ var UPDATE_AFTER_TIERING = `UPDATE zerobucket_images SET data = NULL, storage_backend = 'object_storage', object_storage_bucket = $1, object_storage_key = $2, updated_at = now() WHERE id = $3;`;
270
+ var DEDUP_SCHEMA = `
271
+ CREATE TABLE IF NOT EXISTS zerobucket_blobs (
272
+ checksum_sha256 CHAR(64) PRIMARY KEY,
273
+ data BYTEA NOT NULL,
274
+ mime_type TEXT NOT NULL,
275
+ size_bytes INTEGER NOT NULL,
276
+ width INTEGER,
277
+ height INTEGER,
278
+ ref_count INTEGER NOT NULL DEFAULT 0,
279
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now()
280
+ );
281
+ CREATE TABLE IF NOT EXISTS zerobucket_image_refs (
282
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
283
+ checksum_sha256 CHAR(64) NOT NULL REFERENCES zerobucket_blobs(checksum_sha256),
284
+ original_filename TEXT,
285
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
286
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
287
+ );
288
+ CREATE INDEX IF NOT EXISTS idx_zerobucket_image_refs_checksum ON zerobucket_image_refs (checksum_sha256);
289
+ CREATE INDEX IF NOT EXISTS idx_zerobucket_image_refs_created_at ON zerobucket_image_refs (created_at);
290
+ `;
291
+ var D_UPSERT_BLOB = `
292
+ INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, width, height, ref_count)
293
+ VALUES ($1, $2, $3, $4, $5, $6, $7)
294
+ ON CONFLICT (checksum_sha256) DO UPDATE SET ref_count = zerobucket_blobs.ref_count + EXCLUDED.ref_count;`;
295
+ var D_INSERT_REF = `INSERT INTO zerobucket_image_refs (checksum_sha256, original_filename) VALUES ($1, $2) RETURNING id;`;
296
+ var D_COLS = `r.id, b.data, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256,
297
+ 'postgres'::text AS storage_backend, NULL::text AS object_storage_bucket, NULL::text AS object_storage_key`;
298
+ var D_FROM = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
299
+ var D_DELETE_REF = `DELETE FROM zerobucket_image_refs WHERE id = $1 RETURNING checksum_sha256;`;
300
+ var D_DELETE_REFS = `DELETE FROM zerobucket_image_refs WHERE id = ANY($1::uuid[]) RETURNING id, checksum_sha256;`;
301
+ var D_DECREMENT = `UPDATE zerobucket_blobs SET ref_count = ref_count - $1 WHERE checksum_sha256 = $2 RETURNING ref_count;`;
302
+ var D_DELETE_EMPTY = `DELETE FROM zerobucket_blobs WHERE checksum_sha256 = ANY($1::text[]) AND ref_count <= 0;`;
303
+ var CLASSIC_SQL = {
304
+ schema: SCHEMA,
305
+ selectFull: SELECT_FULL,
306
+ selectMany: SELECT_MANY,
307
+ selectMetadata: SELECT_METADATA,
308
+ selectStreamInfo: SELECT_STREAM_INFO,
309
+ selectChunk: SELECT_CHUNK,
310
+ exists: EXISTS
311
+ };
312
+ var DEDUP_SQL = {
313
+ schema: DEDUP_SCHEMA,
314
+ selectFull: `SELECT ${D_COLS} ${D_FROM} WHERE r.id = $1;`,
315
+ selectMany: `SELECT ${D_COLS} ${D_FROM} WHERE r.id = ANY($1::uuid[]);`,
316
+ selectMetadata: `SELECT r.id, b.mime_type, r.original_filename, b.size_bytes, b.width, b.height, r.checksum_sha256 ${D_FROM} WHERE r.id = $1;`,
317
+ selectStreamInfo: `SELECT b.size_bytes, 'postgres'::text AS storage_backend, NULL::text AS object_storage_key ${D_FROM} WHERE r.id = $1;`,
318
+ selectChunk: `SELECT substring(b.data FROM $1::int FOR $2::int) AS chunk ${D_FROM} WHERE r.id = $3;`,
319
+ exists: `SELECT 1 FROM zerobucket_image_refs WHERE id = $1;`
320
+ };
187
321
  var RETRYABLE_SQLSTATES = /* @__PURE__ */ new Set([
188
322
  "40001",
189
323
  // serialization_failure
@@ -254,6 +388,13 @@ var PostgresBackend = class {
254
388
  this.maxRetries = opts.maxRetries ?? 3;
255
389
  this.retryBaseDelayMs = opts.retryBaseDelayMs ?? 100;
256
390
  this.autoMigrate = opts.autoMigrate ?? true;
391
+ this.dedup = opts.dedup ?? false;
392
+ if (this.dedup && opts.objectStorage) {
393
+ throw new TypeError(
394
+ "objectStorage is not supported together with dedup: true (same restriction as the Python package). Tiering moves one image's bytes, but a deduplicated blob can be shared by many images."
395
+ );
396
+ }
397
+ this.sql = this.dedup ? DEDUP_SQL : CLASSIC_SQL;
257
398
  }
258
399
  opts;
259
400
  pool;
@@ -263,6 +404,8 @@ var PostgresBackend = class {
263
404
  maxRetries;
264
405
  retryBaseDelayMs;
265
406
  autoMigrate;
407
+ dedup;
408
+ sql;
266
409
  // ---- lifecycle -------------------------------------------------------
267
410
  ready() {
268
411
  if (this.closed)
@@ -327,7 +470,7 @@ var PostgresBackend = class {
327
470
  await client.query(
328
471
  "SELECT pg_advisory_xact_lock(hashtext('zerobucket_migrate'))"
329
472
  );
330
- await client.query(SCHEMA);
473
+ await client.query(this.sql.schema);
331
474
  await client.query("COMMIT");
332
475
  client.release();
333
476
  } catch (exc) {
@@ -360,123 +503,1119 @@ var PostgresBackend = class {
360
503
  }
361
504
  }
362
505
  /**
363
- * Run `work` and return its result, emitting exactly one OperationEvent.
364
- *
365
- * connection given -> run once on it, no BEGIN/COMMIT, no retry.
366
- * connection absent -> pooled; `tx` wraps the work in BEGIN/COMMIT; transient
367
- * failures retry the WHOLE closure on a fresh connection.
506
+ * Run `work` and return its result, emitting exactly one OperationEvent.
507
+ *
508
+ * connection given -> run once on it, no BEGIN/COMMIT, no retry.
509
+ * connection absent -> pooled; `tx` wraps the work in BEGIN/COMMIT; transient
510
+ * failures retry the WHOLE closure on a fresh connection.
511
+ */
512
+ async run(operation, connection, work, { tx = false } = {}) {
513
+ if (connection) {
514
+ if (!isQueryable(connection)) {
515
+ throw new TypeError(
516
+ "PostgresBackend expects a pg client/pool client as `connection` (an object with query()), not a SQLite Database."
517
+ );
518
+ }
519
+ const start2 = performance.now();
520
+ try {
521
+ const result = tx ? await this.atomic(connection, work) : await work(connection);
522
+ this.emit(operation, start2, true, null, 0);
523
+ return result;
524
+ } catch (exc) {
525
+ this.emit(operation, start2, false, msg(exc), 0);
526
+ throw exc;
527
+ }
528
+ }
529
+ await this.ready();
530
+ const start = performance.now();
531
+ let attempt = 0;
532
+ for (; ; ) {
533
+ let client;
534
+ let destroy = false;
535
+ try {
536
+ client = await this.pool.connect();
537
+ if (tx) await client.query("BEGIN");
538
+ const result = await work(client);
539
+ if (tx) await client.query("COMMIT");
540
+ this.emit(operation, start, true, null, attempt);
541
+ return result;
542
+ } catch (exc) {
543
+ if (client && tx) {
544
+ try {
545
+ await client.query("ROLLBACK");
546
+ } catch {
547
+ destroy = true;
548
+ }
549
+ }
550
+ if (isConnectionLevel(exc)) destroy = true;
551
+ attempt += 1;
552
+ if (attempt > this.maxRetries || !isRetryable(exc)) {
553
+ this.emit(operation, start, false, msg(exc), attempt - 1);
554
+ throw exc;
555
+ }
556
+ } finally {
557
+ client?.release(destroy);
558
+ }
559
+ await sleep(backoffDelayMs(attempt, this.retryBaseDelayMs));
560
+ }
561
+ }
562
+ /**
563
+ * Make multi-statement work all-or-nothing on a CALLER-supplied connection.
564
+ * - Caller has an open transaction -> SAVEPOINT: a failure rolls back only our work and the
565
+ * caller's transaction stays usable; success joins their transaction.
566
+ * - No open transaction (an autocommit pg.Client) -> open and commit our own, so a failure
567
+ * can never leave half-applied state (e.g. an inflated dedup ref_count).
568
+ */
569
+ async atomic(conn, work) {
570
+ let ownTx = false;
571
+ try {
572
+ await conn.query("SAVEPOINT zerobucket_atomic");
573
+ } catch (exc) {
574
+ if (exc.code !== "25P01") throw exc;
575
+ ownTx = true;
576
+ await conn.query("BEGIN");
577
+ }
578
+ try {
579
+ const result = await work(conn);
580
+ await conn.query(
581
+ ownTx ? "COMMIT" : "RELEASE SAVEPOINT zerobucket_atomic"
582
+ );
583
+ return result;
584
+ } catch (exc) {
585
+ try {
586
+ await conn.query(
587
+ ownTx ? "ROLLBACK" : "ROLLBACK TO SAVEPOINT zerobucket_atomic"
588
+ );
589
+ } catch {
590
+ }
591
+ throw exc;
592
+ }
593
+ }
594
+ // ---- StorageBackend --------------------------------------------------
595
+ async put(row, { connection } = {}) {
596
+ try {
597
+ if (this.dedup) {
598
+ return await this.run(
599
+ "put",
600
+ connection,
601
+ async (q) => {
602
+ await q.query(D_UPSERT_BLOB, blobParams(row, 1));
603
+ return String(
604
+ (await q.query(D_INSERT_REF, [
605
+ row.checksumSha256,
606
+ row.originalFilename
607
+ ])).rows[0].id
608
+ );
609
+ },
610
+ { tx: true }
611
+ );
612
+ }
613
+ return await this.run("put", connection, async (q) => {
614
+ const res = await q.query(INSERT, rowParams(row));
615
+ return String(res.rows[0].id);
616
+ });
617
+ } catch (exc) {
618
+ throw wrap("Failed to store image", exc);
619
+ }
620
+ }
621
+ async putMany(rows, { connection } = {}) {
622
+ if (rows.length === 0) return [];
623
+ try {
624
+ if (this.dedup) {
625
+ return await this.run(
626
+ "put_many",
627
+ connection,
628
+ async (q) => {
629
+ const groups = /* @__PURE__ */ new Map();
630
+ for (const r of rows) {
631
+ const g = groups.get(r.checksumSha256);
632
+ if (g) g.n += 1;
633
+ else groups.set(r.checksumSha256, { row: r, n: 1 });
634
+ }
635
+ for (const checksum of [...groups.keys()].sort()) {
636
+ const g = groups.get(checksum);
637
+ await q.query(D_UPSERT_BLOB, blobParams(g.row, g.n));
638
+ }
639
+ const ids = [];
640
+ for (const r of rows) {
641
+ ids.push(
642
+ String(
643
+ (await q.query(D_INSERT_REF, [
644
+ r.checksumSha256,
645
+ r.originalFilename
646
+ ])).rows[0].id
647
+ )
648
+ );
649
+ }
650
+ return ids;
651
+ },
652
+ { tx: true }
653
+ );
654
+ }
655
+ return await this.run(
656
+ "put_many",
657
+ connection,
658
+ async (q) => {
659
+ const ids = [];
660
+ for (const r of rows)
661
+ ids.push(String((await q.query(INSERT, rowParams(r))).rows[0].id));
662
+ return ids;
663
+ },
664
+ { tx: true }
665
+ );
666
+ } catch (exc) {
667
+ throw wrap("Failed to store image batch", exc);
668
+ }
669
+ }
670
+ async get(id, { connection } = {}) {
671
+ if (!isValidId(id)) return null;
672
+ let row;
673
+ try {
674
+ row = await this.run(
675
+ "get",
676
+ connection,
677
+ async (q) => (await q.query(this.sql.selectFull, [id])).rows[0]
678
+ );
679
+ } catch (exc) {
680
+ throw wrap("Failed to retrieve image", exc);
681
+ }
682
+ return row ? this.hydrate(row, id) : null;
683
+ }
684
+ async getMany(ids, { connection } = {}) {
685
+ const valid = ids.filter(isValidId);
686
+ if (valid.length === 0) return [];
687
+ let rows;
688
+ try {
689
+ rows = await this.run(
690
+ "get_many",
691
+ connection,
692
+ async (q) => (await q.query(this.sql.selectMany, [valid])).rows
693
+ );
694
+ } catch (exc) {
695
+ throw wrap("Failed to retrieve image batch", exc);
696
+ }
697
+ return mapLimit(rows, 4, (r) => this.hydrate(r, String(r.id)));
698
+ }
699
+ async getMetadata(id, { connection } = {}) {
700
+ if (!isValidId(id)) return null;
701
+ let row;
702
+ try {
703
+ row = await this.run(
704
+ "get_metadata",
705
+ connection,
706
+ async (q) => (await q.query(this.sql.selectMetadata, [id])).rows[0]
707
+ );
708
+ } catch (exc) {
709
+ throw wrap("Failed to retrieve image metadata", exc);
710
+ }
711
+ if (!row) return null;
712
+ return {
713
+ id: String(row.id),
714
+ mimeType: row.mime_type,
715
+ originalFilename: row.original_filename,
716
+ sizeBytes: row.size_bytes,
717
+ width: row.width,
718
+ height: row.height,
719
+ checksumSha256: row.checksum_sha256
720
+ };
721
+ }
722
+ async getStream(id, opts) {
723
+ if (!isValidId(id)) return null;
724
+ const { connection, chunkSize } = opts;
725
+ if (!Number.isInteger(chunkSize) || chunkSize <= 0)
726
+ throw new RangeError("chunkSize must be a positive integer");
727
+ let info;
728
+ try {
729
+ info = await this.run(
730
+ "get_metadata",
731
+ connection,
732
+ async (q) => (await q.query(this.sql.selectStreamInfo, [id])).rows[0]
733
+ );
734
+ } catch (exc) {
735
+ throw wrap("Failed to retrieve image metadata", exc);
736
+ }
737
+ if (!info) return null;
738
+ const totalSize = info.size_bytes;
739
+ const start = opts.range?.start ?? 0;
740
+ const end = Math.min(opts.range?.end ?? totalSize - 1, totalSize - 1);
741
+ if (info.storage_backend === "object_storage") {
742
+ const os = this.opts.objectStorage;
743
+ if (!os) throw tieredWithoutStorage(id, info.object_storage_key);
744
+ return streamFromObjectStorage(
745
+ os,
746
+ id,
747
+ info.object_storage_key,
748
+ start,
749
+ end,
750
+ chunkSize
751
+ );
752
+ }
753
+ const self = this;
754
+ async function* generate() {
755
+ let offset = start;
756
+ let delivered = 0;
757
+ const wanted = Math.max(0, end - start + 1);
758
+ while (delivered < wanted) {
759
+ const length = Math.min(chunkSize, wanted - delivered);
760
+ let row;
761
+ try {
762
+ row = await self.run(
763
+ "get_stream",
764
+ connection,
765
+ async (q) => (await q.query(self.sql.selectChunk, [offset + 1, length, id])).rows[0]
766
+ );
767
+ } catch (exc) {
768
+ throw wrap("Failed to stream image", exc);
769
+ }
770
+ if (!row) {
771
+ throw new StorageError(
772
+ `Image ${JSON.stringify(id)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
773
+ );
774
+ }
775
+ const chunk = row.chunk;
776
+ if (!chunk || chunk.length === 0) {
777
+ throw new StorageError(
778
+ `Image ${JSON.stringify(id)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
779
+ );
780
+ }
781
+ yield chunk;
782
+ offset += chunk.length;
783
+ delivered += chunk.length;
784
+ }
785
+ }
786
+ return generate();
787
+ }
788
+ async delete(id, { connection } = {}) {
789
+ if (!isValidId(id)) return false;
790
+ if (this.dedup) return this.deleteDedup(id, connection);
791
+ try {
792
+ const row = await this.run(
793
+ "delete",
794
+ connection,
795
+ async (q) => (await q.query(DELETE_RETURNING, [id])).rows[0]
796
+ );
797
+ if (row?.storage_backend === "object_storage")
798
+ await this.cleanupObject([row.object_storage_key], connection);
799
+ return row !== void 0;
800
+ } catch (exc) {
801
+ throw wrap("Failed to delete image", exc);
802
+ }
803
+ }
804
+ async deleteMany(ids, { connection } = {}) {
805
+ const valid = ids.filter(isValidId);
806
+ if (valid.length === 0) return [];
807
+ if (this.dedup) return this.deleteManyDedup(valid, connection);
808
+ try {
809
+ const rows = await this.run(
810
+ "delete_many",
811
+ connection,
812
+ async (q) => (await q.query(DELETE_MANY, [valid])).rows
813
+ );
814
+ await this.cleanupObject(
815
+ rows.filter((r) => r.storage_backend === "object_storage").map((r) => r.object_storage_key),
816
+ connection
817
+ );
818
+ return rows.map((r) => String(r.id));
819
+ } catch (exc) {
820
+ throw wrap("Failed to delete image batch", exc);
821
+ }
822
+ }
823
+ async exists(id, { connection } = {}) {
824
+ if (!isValidId(id)) return false;
825
+ try {
826
+ return await this.run(
827
+ "exists",
828
+ connection,
829
+ async (q) => ((await q.query(this.sql.exists, [id])).rowCount ?? 0) > 0
830
+ );
831
+ } catch (exc) {
832
+ throw wrap("Failed to check image existence", exc);
833
+ }
834
+ }
835
+ // ---- maintenance (CLI) -----------------------------------------------
836
+ async getInfo() {
837
+ try {
838
+ return await this.run("info", void 0, async (q) => {
839
+ const imgTable = this.dedup ? "zerobucket_image_refs" : "zerobucket_images";
840
+ const present = await q.query(
841
+ "SELECT to_regclass($1) AS a, to_regclass('zerobucket_blobs') AS b",
842
+ [imgTable]
843
+ );
844
+ if (!present.rows[0].a || this.dedup && !present.rows[0].b)
845
+ return null;
846
+ const j = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
847
+ const size = this.dedup ? "b.size_bytes" : "size_bytes";
848
+ const created = this.dedup ? "r.created_at" : "created_at";
849
+ const mime = this.dedup ? "b.mime_type" : "mime_type";
850
+ const tot = (await q.query(
851
+ `SELECT count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes, min(${created})::text AS oldest, max(${created})::text AS newest ${j}`
852
+ )).rows[0];
853
+ const disk = (await q.query(
854
+ this.dedup ? "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_blobs') + pg_total_relation_size('zerobucket_image_refs')) AS s" : "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_images')) AS s"
855
+ )).rows[0].s;
856
+ const fmt = (await q.query(
857
+ `SELECT ${mime} AS m, count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
858
+ )).rows;
859
+ const info = {
860
+ mode: this.dedup ? "dedup" : "classic",
861
+ count: tot.n,
862
+ totalBytes: Number(tot.bytes),
863
+ // Postgres prints the offset as "+00"; Python's CLI (and ISO-8601) print "+00:00". Normalise so output is identical.
864
+ oldest: tot.oldest && tot.oldest.replace(/([+-]\d\d)$/, "$1:00"),
865
+ newest: tot.newest && tot.newest.replace(/([+-]\d\d)$/, "$1:00"),
866
+ onDisk: disk,
867
+ byFormat: fmt.map((r) => ({
868
+ mimeType: r.m,
869
+ count: r.n,
870
+ bytes: Number(r.bytes)
871
+ }))
872
+ };
873
+ if (this.dedup) {
874
+ const b = (await q.query(
875
+ "SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_blobs"
876
+ )).rows[0];
877
+ info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
878
+ } else {
879
+ const t = (await q.query(
880
+ "SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
881
+ )).rows[0];
882
+ info.tiered = { count: t.n, bytes: Number(t.bytes) };
883
+ }
884
+ return info;
885
+ });
886
+ } catch (exc) {
887
+ throw wrap("Failed to read storage info", exc);
888
+ }
889
+ }
890
+ async listImages(o = {}) {
891
+ const where = [];
892
+ const params = [];
893
+ const add = (sql2, v) => (params.push(v), where.push(sql2.replace("$?", `$${params.length}`)));
894
+ if (o.minSize !== void 0)
895
+ add(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= $?`, o.minSize);
896
+ if (o.olderThanDays !== void 0)
897
+ add(
898
+ `${this.dedup ? "r.created_at" : "created_at"} <= now() - make_interval(days => $?::int)`,
899
+ o.olderThanDays
900
+ );
901
+ if (o.onlyUntiered && !this.dedup)
902
+ where.push("storage_backend = 'postgres'");
903
+ const from = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
904
+ const idCol = this.dedup ? "r.id" : "id";
905
+ const tierCol = this.dedup ? "false AS tiered" : "(storage_backend = 'object_storage') AS tiered";
906
+ const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
907
+ const lim = o.sample ?? o.limit;
908
+ let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
909
+ if (lim !== void 0) {
910
+ params.push(lim);
911
+ sql += ` LIMIT $${params.length}`;
912
+ }
913
+ try {
914
+ const rows = await this.run(
915
+ "list_images",
916
+ void 0,
917
+ async (q) => (await q.query(sql, params)).rows
918
+ );
919
+ return rows.map((r) => ({ id: String(r.id), tiered: Boolean(r.tiered) }));
920
+ } catch (exc) {
921
+ throw wrap("Failed to list images", exc);
922
+ }
923
+ }
924
+ // ---- dedup deletes / migration ---------------------------------------
925
+ async deleteDedup(id, connection) {
926
+ try {
927
+ return await this.run(
928
+ "delete",
929
+ connection,
930
+ async (q) => {
931
+ const ref = (await q.query(D_DELETE_REF, [id])).rows[0];
932
+ if (!ref) return false;
933
+ const checksum = ref.checksum_sha256;
934
+ const dec = (await q.query(D_DECREMENT, [1, checksum])).rows[0];
935
+ if (dec && dec.ref_count <= 0)
936
+ await q.query(D_DELETE_EMPTY, [[checksum]]);
937
+ return true;
938
+ },
939
+ { tx: true }
940
+ );
941
+ } catch (exc) {
942
+ throw wrap("Failed to delete image", exc);
943
+ }
944
+ }
945
+ async deleteManyDedup(ids, connection) {
946
+ try {
947
+ return await this.run(
948
+ "delete_many",
949
+ connection,
950
+ async (q) => {
951
+ const deleted = (await q.query(D_DELETE_REFS, [ids])).rows;
952
+ if (deleted.length === 0) return [];
953
+ const counts = /* @__PURE__ */ new Map();
954
+ for (const r of deleted)
955
+ counts.set(
956
+ r.checksum_sha256,
957
+ (counts.get(r.checksum_sha256) ?? 0) + 1
958
+ );
959
+ const emptied = [];
960
+ for (const checksum of [...counts.keys()].sort()) {
961
+ const dec = (await q.query(D_DECREMENT, [counts.get(checksum), checksum])).rows[0];
962
+ if (dec && dec.ref_count <= 0) emptied.push(checksum);
963
+ }
964
+ if (emptied.length > 0) await q.query(D_DELETE_EMPTY, [emptied]);
965
+ return deleted.map((r) => String(r.id));
966
+ },
967
+ { tx: true }
968
+ );
969
+ } catch (exc) {
970
+ throw wrap("Failed to delete image batch", exc);
971
+ }
972
+ }
973
+ /**
974
+ * One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
975
+ * dedup instance's tables, preserving every id (external references keep working) and
976
+ * deduplicating identical content on the way. The classic table is never modified.
977
+ *
978
+ * Differences from the Python version, all deliberate: rows are copied in small batches (bounded
979
+ * memory instead of loading the whole table); re-running is safe (already-migrated ids are
980
+ * skipped and reported, not an error); and classic rows already tiered to object storage are
981
+ * refused up front with a clear message (their bytes are not in the database to copy).
982
+ * Still all-or-nothing: one transaction.
983
+ */
984
+ async migrateClassicToDedup() {
985
+ if (!this.dedup) {
986
+ throw new StorageError(
987
+ "migrateClassicToDedup() must be called on an instance created with dedup: true."
988
+ );
989
+ }
990
+ await this.ready();
991
+ const client = await this.pool.connect();
992
+ let destroy = false;
993
+ try {
994
+ await client.query("BEGIN");
995
+ await client.query(
996
+ "SELECT pg_advisory_xact_lock(hashtext('zerobucket_migrate_dedup'))"
997
+ );
998
+ if (!(await client.query("SELECT to_regclass('zerobucket_images') AS t")).rows[0].t) {
999
+ throw new StorageError(
1000
+ "No classic zerobucket_images table found in this database -- nothing to migrate."
1001
+ );
1002
+ }
1003
+ const hasTierCol = (await client.query(
1004
+ "SELECT 1 FROM information_schema.columns WHERE table_name = 'zerobucket_images' AND column_name = 'storage_backend'"
1005
+ )).rowCount;
1006
+ if (hasTierCol) {
1007
+ const tiered = (await client.query(
1008
+ "SELECT count(*)::int AS n FROM zerobucket_images WHERE storage_backend <> 'postgres'"
1009
+ )).rows[0].n;
1010
+ if (tiered > 0) {
1011
+ throw new StorageError(
1012
+ `${tiered} classic image(s) are tiered to object storage, so their bytes are not in the database to migrate. Dedup mode does not support tiering. Migrate only an un-tiered table, or bring those images back first.`
1013
+ );
1014
+ }
1015
+ }
1016
+ const seen = /* @__PURE__ */ new Set();
1017
+ let migrated = 0;
1018
+ let already = 0;
1019
+ let cursor = null;
1020
+ const BATCH = 20;
1021
+ for (; ; ) {
1022
+ const page = cursor ? await client.query(
1023
+ `SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
1024
+ FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid) ORDER BY created_at, id LIMIT ${BATCH}`,
1025
+ [cursor.createdAt, cursor.id]
1026
+ ) : await client.query(
1027
+ `SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
1028
+ FROM zerobucket_images ORDER BY created_at, id LIMIT ${BATCH}`
1029
+ );
1030
+ if (page.rows.length === 0) break;
1031
+ const last = page.rows[page.rows.length - 1];
1032
+ if (cursor && cursor.createdAt === last.created_at_txt && cursor.id === last.id) {
1033
+ throw new StorageError(
1034
+ "Migration made no progress (keyset cursor did not advance); aborting instead of looping."
1035
+ );
1036
+ }
1037
+ cursor = { createdAt: last.created_at_txt, id: last.id };
1038
+ const have = new Set(
1039
+ (await client.query(
1040
+ "SELECT id FROM zerobucket_image_refs WHERE id = ANY($1::uuid[])",
1041
+ [page.rows.map((r) => r.id)]
1042
+ )).rows.map((r) => String(r.id))
1043
+ );
1044
+ const fresh = page.rows.filter((r) => !have.has(String(r.id)));
1045
+ already += page.rows.length - fresh.length;
1046
+ const groups = /* @__PURE__ */ new Map();
1047
+ for (const r of fresh) {
1048
+ const g = groups.get(r.checksum_sha256);
1049
+ if (g) g.n += 1;
1050
+ else groups.set(r.checksum_sha256, { row: r, n: 1 });
1051
+ }
1052
+ for (const checksum of [...groups.keys()].sort()) {
1053
+ const { row, n } = groups.get(checksum);
1054
+ await client.query(D_UPSERT_BLOB, [
1055
+ checksum,
1056
+ row.data,
1057
+ row.mime_type,
1058
+ row.size_bytes,
1059
+ row.width,
1060
+ row.height,
1061
+ n
1062
+ ]);
1063
+ }
1064
+ for (const r of fresh) {
1065
+ await client.query(
1066
+ "INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at) VALUES ($1, $2, $3, $4::timestamptz)",
1067
+ [r.id, r.checksum_sha256, r.original_filename, r.created_at_txt]
1068
+ );
1069
+ seen.add(r.checksum_sha256);
1070
+ migrated += 1;
1071
+ }
1072
+ }
1073
+ await client.query("COMMIT");
1074
+ return {
1075
+ imagesMigrated: migrated,
1076
+ distinctBlobsCreated: seen.size,
1077
+ duplicateReferencesFound: migrated - seen.size,
1078
+ alreadyMigrated: already
1079
+ };
1080
+ } catch (exc) {
1081
+ try {
1082
+ await client.query("ROLLBACK");
1083
+ } catch {
1084
+ destroy = true;
1085
+ }
1086
+ throw exc instanceof StorageError ? exc : new StorageError(`Migration to dedup failed: ${msg(exc)}`, {
1087
+ cause: exc
1088
+ });
1089
+ } finally {
1090
+ client.release(destroy);
1091
+ }
1092
+ }
1093
+ // ---- helpers ---------------------------------------------------------
1094
+ hydrate(row, id) {
1095
+ return hydrateRow(row, id, this.opts.objectStorage);
1096
+ }
1097
+ /**
1098
+ * Remove tiered objects AFTER their rows are gone. Ordering is deliberate:
1099
+ * once the row is deleted the image is correctly "not found" to everyone, so
1100
+ * a failed object delete only leaves a harmless orphan, whereas deleting the
1101
+ * object first and then failing the row delete would leave a row pointing at nothing.
1102
+ *
1103
+ * Never runs inside a caller-owned transaction (`connection`): we cannot know
1104
+ * whether it will commit, and destroying data before the commit outcome is known
1105
+ * could leave a rolled-back row pointing at a deleted object. In that case the
1106
+ * object stays until you remove it yourself after committing (objectStorage.delete(id)).
1107
+ * Failures are reported through onOperation and never fail the delete itself.
1108
+ */
1109
+ async cleanupObject(keys, connection) {
1110
+ const os = this.opts.objectStorage;
1111
+ if (!os || connection || keys.length === 0) return;
1112
+ await Promise.all(
1113
+ keys.map(async (key) => {
1114
+ const start = performance.now();
1115
+ try {
1116
+ await os.delete(key);
1117
+ this.emit("object_storage_delete", start, true, null, 0);
1118
+ } catch (exc) {
1119
+ this.emit("object_storage_delete", start, false, msg(exc), 0);
1120
+ }
1121
+ })
1122
+ );
1123
+ }
1124
+ /**
1125
+ * Move an image's bytes out of Postgres into object storage, replacing the
1126
+ * row's `data` with a pointer. Returns:
1127
+ * null - no such image
1128
+ * false - already tiered (a safe no-op, so backfills can be re-run)
1129
+ * true - tiered just now
1130
+ *
1131
+ * SAFETY: the upload happens INSIDE the same transaction as the row lock
1132
+ * (SELECT ... FOR UPDATE) and the UPDATE that flips the row. If the upload
1133
+ * fails, the transaction rolls back and the row is untouched: there is no
1134
+ * moment when the bytes exist nowhere, or when a row claims to be tiered
1135
+ * without a completed upload.
1136
+ *
1137
+ * TRADEOFF: the row lock is held for the whole upload. Concurrent writes
1138
+ * to THIS image (delete, tier) wait; every other row is unaffected, and plain
1139
+ * reads are never blocked (Postgres MVCC). Meant for explicit maintenance, not hot paths.
1140
+ *
1141
+ * Retry is safe: a transient error replays the whole closure, and the
1142
+ * deterministic key (the image id) makes the re-upload an idempotent overwrite.
1143
+ */
1144
+ async tierToObjectStorage(id, { connection } = {}) {
1145
+ if (this.dedup) {
1146
+ throw new StorageError(
1147
+ "tierToObjectStorage() is not supported in dedup mode (a blob may be shared by many images)."
1148
+ );
1149
+ }
1150
+ const os = this.opts.objectStorage;
1151
+ if (!os) {
1152
+ throw new StorageError(
1153
+ "tierToObjectStorage() requires objectStorage to be configured (new ZeroBucket({ objectStorage: new ObjectStorage({ bucket }) }))."
1154
+ );
1155
+ }
1156
+ if (!isValidId(id)) return null;
1157
+ const key = id.toLowerCase();
1158
+ try {
1159
+ return await this.run(
1160
+ "tier_to_object_storage",
1161
+ connection,
1162
+ async (q) => {
1163
+ const row = (await q.query(SELECT_FOR_TIERING, [id])).rows[0];
1164
+ if (!row) return null;
1165
+ if (row.storage_backend !== "postgres") return false;
1166
+ await os.upload(key, row.data, { mimeType: row.mime_type });
1167
+ await q.query(UPDATE_AFTER_TIERING, [os.bucket, key, id]);
1168
+ return true;
1169
+ },
1170
+ { tx: true }
1171
+ );
1172
+ } catch (exc) {
1173
+ throw wrap("Failed to tier image to object storage", exc);
1174
+ }
1175
+ }
1176
+ };
1177
+ function blobParams(r, refs) {
1178
+ return [
1179
+ r.checksumSha256,
1180
+ r.data,
1181
+ r.mimeType,
1182
+ r.sizeBytes,
1183
+ r.width,
1184
+ r.height,
1185
+ refs
1186
+ ];
1187
+ }
1188
+ function rowParams(r) {
1189
+ return [
1190
+ r.data,
1191
+ r.mimeType,
1192
+ r.originalFilename,
1193
+ r.sizeBytes,
1194
+ r.width,
1195
+ r.height,
1196
+ r.checksumSha256
1197
+ ];
1198
+ }
1199
+ function isQueryable(c) {
1200
+ return typeof c.query === "function";
1201
+ }
1202
+
1203
+ // src/adapters/sqlite.ts
1204
+ var import_node_crypto = require("crypto");
1205
+ var import_node_fs = require("fs");
1206
+ var SCHEMA2 = `
1207
+ CREATE TABLE IF NOT EXISTS zerobucket_images (
1208
+ id TEXT PRIMARY KEY,
1209
+ data BLOB,
1210
+ mime_type TEXT NOT NULL,
1211
+ original_filename TEXT,
1212
+ size_bytes INTEGER NOT NULL,
1213
+ width INTEGER,
1214
+ height INTEGER,
1215
+ checksum_sha256 TEXT NOT NULL,
1216
+ created_at TEXT NOT NULL,
1217
+ updated_at TEXT NOT NULL,
1218
+ storage_backend TEXT NOT NULL DEFAULT 'sqlite',
1219
+ object_storage_bucket TEXT,
1220
+ object_storage_key TEXT,
1221
+ CHECK (
1222
+ (storage_backend = 'sqlite'
1223
+ AND data IS NOT NULL
1224
+ AND object_storage_key IS NULL
1225
+ AND object_storage_bucket IS NULL)
1226
+ OR
1227
+ (storage_backend = 'object_storage'
1228
+ AND data IS NULL
1229
+ AND object_storage_key IS NOT NULL
1230
+ AND object_storage_bucket IS NOT NULL)
1231
+ )
1232
+ );
1233
+ CREATE INDEX IF NOT EXISTS idx_zerobucket_checksum ON zerobucket_images (checksum_sha256);
1234
+ CREATE INDEX IF NOT EXISTS idx_zerobucket_created_at ON zerobucket_images (created_at);
1235
+ `;
1236
+ var DEDUP_SCHEMA2 = `
1237
+ CREATE TABLE IF NOT EXISTS zerobucket_blobs (
1238
+ checksum_sha256 TEXT PRIMARY KEY,
1239
+ data BLOB NOT NULL,
1240
+ mime_type TEXT NOT NULL,
1241
+ size_bytes INTEGER NOT NULL,
1242
+ width INTEGER,
1243
+ height INTEGER,
1244
+ ref_count INTEGER NOT NULL DEFAULT 0,
1245
+ created_at TEXT NOT NULL
1246
+ );
1247
+ CREATE TABLE IF NOT EXISTS zerobucket_image_refs (
1248
+ id TEXT PRIMARY KEY,
1249
+ checksum_sha256 TEXT NOT NULL REFERENCES zerobucket_blobs(checksum_sha256),
1250
+ original_filename TEXT,
1251
+ created_at TEXT NOT NULL,
1252
+ updated_at TEXT NOT NULL
1253
+ );
1254
+ CREATE INDEX IF NOT EXISTS idx_zerobucket_image_refs_checksum ON zerobucket_image_refs (checksum_sha256);
1255
+ CREATE INDEX IF NOT EXISTS idx_zerobucket_image_refs_created_at ON zerobucket_image_refs (created_at);
1256
+ `;
1257
+ var COLS = `id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, storage_backend, object_storage_bucket, object_storage_key`;
1258
+ var INSERT2 = `INSERT INTO zerobucket_images (id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
1259
+ var SELECT_FULL2 = `SELECT ${COLS} FROM zerobucket_images WHERE id = ?`;
1260
+ var selectMany = (n) => `SELECT ${COLS} FROM zerobucket_images WHERE id IN (${placeholders(n)})`;
1261
+ var SELECT_METADATA2 = `SELECT id, mime_type, original_filename, size_bytes, width, height, checksum_sha256 FROM zerobucket_images WHERE id = ?`;
1262
+ var STREAM_INFO = `SELECT size_bytes, storage_backend, object_storage_key FROM zerobucket_images WHERE id = ?`;
1263
+ var CHUNK = `SELECT substr(data, ?, ?) AS chunk FROM zerobucket_images WHERE id = ?`;
1264
+ var EXISTS2 = `SELECT 1 FROM zerobucket_images WHERE id = ?`;
1265
+ var DELETE_ONE = `DELETE FROM zerobucket_images WHERE id = ? RETURNING storage_backend, object_storage_key`;
1266
+ var deleteMany = (n) => `DELETE FROM zerobucket_images WHERE id IN (${placeholders(n)}) RETURNING id, storage_backend, object_storage_key`;
1267
+ var SELECT_FOR_TIERING2 = `SELECT data, mime_type, size_bytes, storage_backend FROM zerobucket_images WHERE id = ?`;
1268
+ var UPDATE_AFTER_TIERING2 = `UPDATE zerobucket_images SET data = NULL, storage_backend = 'object_storage', object_storage_bucket = ?, object_storage_key = ?, updated_at = ? WHERE id = ?`;
1269
+ var D_FROM2 = `FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256`;
1270
+ var D_COLS2 = `r.id AS id, b.data AS data, b.mime_type AS mime_type, r.original_filename AS original_filename, b.size_bytes AS size_bytes, b.width AS width, b.height AS height, r.checksum_sha256 AS checksum_sha256, 'sqlite' AS storage_backend, NULL AS object_storage_bucket, NULL AS object_storage_key`;
1271
+ var D_UPSERT = `INSERT INTO zerobucket_blobs (checksum_sha256, data, mime_type, size_bytes, width, height, ref_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (checksum_sha256) DO UPDATE SET ref_count = ref_count + excluded.ref_count`;
1272
+ var D_INSERT_REF2 = `INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at, updated_at) VALUES (?, ?, ?, ?, ?)`;
1273
+ var D_SELECT_FULL = `SELECT ${D_COLS2} ${D_FROM2} WHERE r.id = ?`;
1274
+ var dSelectMany = (n) => `SELECT ${D_COLS2} ${D_FROM2} WHERE r.id IN (${placeholders(n)})`;
1275
+ var D_SELECT_METADATA = `SELECT r.id AS id, b.mime_type AS mime_type, r.original_filename AS original_filename, b.size_bytes AS size_bytes, b.width AS width, b.height AS height, r.checksum_sha256 AS checksum_sha256 ${D_FROM2} WHERE r.id = ?`;
1276
+ var D_STREAM_INFO = `SELECT b.size_bytes AS size_bytes, 'sqlite' AS storage_backend, NULL AS object_storage_key ${D_FROM2} WHERE r.id = ?`;
1277
+ var D_CHUNK = `SELECT substr(b.data, ?, ?) AS chunk ${D_FROM2} WHERE r.id = ?`;
1278
+ var D_EXISTS = `SELECT 1 FROM zerobucket_image_refs WHERE id = ?`;
1279
+ var D_DELETE_REF2 = `DELETE FROM zerobucket_image_refs WHERE id = ? RETURNING checksum_sha256`;
1280
+ var dDeleteRefs = (n) => `DELETE FROM zerobucket_image_refs WHERE id IN (${placeholders(n)}) RETURNING id, checksum_sha256`;
1281
+ var D_DECREMENT2 = `UPDATE zerobucket_blobs SET ref_count = ref_count - ? WHERE checksum_sha256 = ? RETURNING ref_count`;
1282
+ var D_DELETE_EMPTY2 = `DELETE FROM zerobucket_blobs WHERE checksum_sha256 = ? AND ref_count <= 0`;
1283
+ var IN_CHUNK = 500;
1284
+ var ATTEMPT_BUSY_MS = 25;
1285
+ var DEFAULT_BUSY_TIMEOUT_MS = 1e4;
1286
+ function placeholders(n) {
1287
+ return Array.from({ length: n }, () => "?").join(", ");
1288
+ }
1289
+ function isoNow() {
1290
+ return (/* @__PURE__ */ new Date()).toISOString().replace("Z", "000+00:00");
1291
+ }
1292
+ function isBusy(exc) {
1293
+ const code = exc?.code;
1294
+ return typeof code === "string" && (code.startsWith("SQLITE_BUSY") || code.startsWith("SQLITE_LOCKED"));
1295
+ }
1296
+ var Mutex = class {
1297
+ tail = Promise.resolve();
1298
+ run(fn) {
1299
+ const prev = this.tail;
1300
+ let release;
1301
+ this.tail = new Promise((r) => release = r);
1302
+ return prev.then(fn).finally(release);
1303
+ }
1304
+ };
1305
+ var SQLiteBackend = class {
1306
+ constructor(opts) {
1307
+ this.opts = opts;
1308
+ if (!opts.path)
1309
+ throw new TypeError(
1310
+ 'SQLiteBackend requires a `path` (a file path or ":memory:")'
1311
+ );
1312
+ this.dedup = opts.dedup ?? false;
1313
+ this.busyTimeoutMs = opts.busyTimeoutMs ?? DEFAULT_BUSY_TIMEOUT_MS;
1314
+ if (this.dedup && opts.objectStorage) {
1315
+ throw new TypeError(
1316
+ "objectStorage is not supported together with dedup: true (same restriction as the Python package). Tiering moves one image's bytes, but a deduplicated blob can be shared by many images."
1317
+ );
1318
+ }
1319
+ }
1320
+ opts;
1321
+ db;
1322
+ Database;
1323
+ readyPromise;
1324
+ closed = false;
1325
+ dedup;
1326
+ busyTimeoutMs;
1327
+ writeLock = new Mutex();
1328
+ // ---- lifecycle -------------------------------------------------------
1329
+ ready() {
1330
+ if (this.closed)
1331
+ return Promise.reject(
1332
+ new StorageError("This ZeroBucket instance has been closed")
1333
+ );
1334
+ this.readyPromise ??= this.init().catch((exc) => {
1335
+ this.readyPromise = void 0;
1336
+ throw exc;
1337
+ });
1338
+ return this.readyPromise;
1339
+ }
1340
+ async init() {
1341
+ try {
1342
+ const mod = await import("better-sqlite3");
1343
+ this.Database = mod.default ?? mod;
1344
+ } catch (cause) {
1345
+ throw new StorageError(
1346
+ "SQLiteBackend requires the 'better-sqlite3' package. Install it with: npm install better-sqlite3",
1347
+ { cause }
1348
+ );
1349
+ }
1350
+ const db = new this.Database(this.opts.path, { timeout: ATTEMPT_BUSY_MS });
1351
+ try {
1352
+ await this.retrying(() => {
1353
+ if (this.opts.path !== ":memory:" && db.pragma("journal_mode", { simple: true }) !== "wal") {
1354
+ db.pragma("journal_mode = WAL");
1355
+ }
1356
+ db.pragma("foreign_keys = ON");
1357
+ if (this.opts.autoMigrate ?? true)
1358
+ db.exec(this.dedup ? DEDUP_SCHEMA2 : SCHEMA2);
1359
+ });
1360
+ } catch (exc) {
1361
+ try {
1362
+ db.close();
1363
+ } catch {
1364
+ }
1365
+ throw new StorageError(
1366
+ `Could not open SQLite database ${JSON.stringify(this.opts.path)}: ${msg(exc)}`,
1367
+ { cause: exc }
1368
+ );
1369
+ }
1370
+ this.db = db;
1371
+ }
1372
+ async close() {
1373
+ this.closed = true;
1374
+ const db = this.db;
1375
+ this.db = void 0;
1376
+ this.readyPromise = void 0;
1377
+ await this.writeLock.run(async () => {
1378
+ db?.close();
1379
+ });
1380
+ }
1381
+ // ---- execution core --------------------------------------------------
1382
+ emit(operation, start, success, error, retryCount) {
1383
+ const cb = this.opts.onOperation;
1384
+ if (!cb) return;
1385
+ try {
1386
+ cb({
1387
+ operation,
1388
+ durationMs: performance.now() - start,
1389
+ success,
1390
+ error,
1391
+ retryCount
1392
+ });
1393
+ } catch {
1394
+ }
1395
+ }
1396
+ /** Re-run `fn` while another connection holds a lock, sleeping asynchronously between attempts. `fn` must be atomic. */
1397
+ async retrying(fn, onRetry) {
1398
+ const deadline = performance.now() + this.busyTimeoutMs;
1399
+ let attempt = 0;
1400
+ for (; ; ) {
1401
+ try {
1402
+ return fn();
1403
+ } catch (exc) {
1404
+ if (!isBusy(exc) || performance.now() >= deadline) throw exc;
1405
+ attempt += 1;
1406
+ onRetry?.(attempt);
1407
+ await sleep(
1408
+ Math.min(5 * 2 ** Math.min(attempt, 5), 100) + Math.random() * 5
1409
+ );
1410
+ }
1411
+ }
1412
+ }
1413
+ /** BEGIN IMMEDIATE takes the write lock up front, so a BUSY comes before any work and the whole unit is safely re-runnable. */
1414
+ tx(db, work) {
1415
+ db.exec("BEGIN IMMEDIATE");
1416
+ try {
1417
+ const result = work(db);
1418
+ db.exec("COMMIT");
1419
+ return result;
1420
+ } catch (exc) {
1421
+ try {
1422
+ db.exec("ROLLBACK");
1423
+ } catch {
1424
+ }
1425
+ throw exc;
1426
+ }
1427
+ }
1428
+ /**
1429
+ * All-or-nothing work on a CALLER-supplied handle. SAVEPOINT works whether or not the caller has an open
1430
+ * transaction: inside one it nests (a failure rolls back only our work); outside one it begins and, on
1431
+ * RELEASE, commits.
368
1432
  */
369
- async run(operation, connection, work, { tx = false } = {}) {
370
- if (connection) {
371
- const start2 = performance.now();
1433
+ atomic(db, work) {
1434
+ db.exec("SAVEPOINT zerobucket_atomic");
1435
+ try {
1436
+ const result = work(db);
1437
+ db.exec("RELEASE SAVEPOINT zerobucket_atomic");
1438
+ return result;
1439
+ } catch (exc) {
372
1440
  try {
373
- const result = await work(connection);
374
- this.emit(operation, start2, true, null, 0);
375
- return result;
376
- } catch (exc) {
377
- this.emit(operation, start2, false, msg(exc), 0);
378
- throw exc;
1441
+ db.exec("ROLLBACK TO SAVEPOINT zerobucket_atomic");
1442
+ db.exec("RELEASE SAVEPOINT zerobucket_atomic");
1443
+ } catch {
379
1444
  }
1445
+ throw exc;
380
1446
  }
381
- await this.ready();
1447
+ }
1448
+ async execute(operation, connection, mode, work) {
382
1449
  const start = performance.now();
383
- let attempt = 0;
384
- for (; ; ) {
385
- let client;
386
- let destroy = false;
387
- try {
388
- client = await this.pool.connect();
389
- if (tx) await client.query("BEGIN");
390
- const result = await work(client);
391
- if (tx) await client.query("COMMIT");
392
- this.emit(operation, start, true, null, attempt);
393
- return result;
394
- } catch (exc) {
395
- if (client && tx) {
396
- try {
397
- await client.query("ROLLBACK");
398
- } catch {
399
- destroy = true;
400
- }
401
- }
402
- if (isConnectionLevel(exc)) destroy = true;
403
- attempt += 1;
404
- if (attempt > this.maxRetries || !isRetryable(exc)) {
405
- this.emit(operation, start, false, msg(exc), attempt - 1);
406
- throw exc;
1450
+ let retries = 0;
1451
+ try {
1452
+ let result;
1453
+ if (connection) {
1454
+ if (typeof connection.prepare !== "function") {
1455
+ throw new TypeError(
1456
+ "SQLiteBackend expects a better-sqlite3 Database as `connection` (an object with prepare()), not a pg client."
1457
+ );
407
1458
  }
408
- } finally {
409
- client?.release(destroy);
1459
+ const db = connection;
1460
+ result = mode === "write" ? this.atomic(db, work) : work(db);
1461
+ } else {
1462
+ await this.ready();
1463
+ const db = this.db;
1464
+ const onRetry = (n) => retries = n;
1465
+ result = mode === "write" ? await this.writeLock.run(
1466
+ () => this.retrying(() => this.tx(db, work), onRetry)
1467
+ ) : await this.retrying(() => work(db), onRetry);
410
1468
  }
411
- await sleep(backoffDelayMs(attempt, this.retryBaseDelayMs));
1469
+ this.emit(operation, start, true, null, retries);
1470
+ return result;
1471
+ } catch (exc) {
1472
+ this.emit(operation, start, false, msg(exc), retries);
1473
+ throw exc;
412
1474
  }
413
1475
  }
414
1476
  // ---- StorageBackend --------------------------------------------------
415
1477
  async put(row, { connection } = {}) {
416
1478
  try {
417
- return await this.run("put", connection, async (q) => {
418
- const res = await q.query(INSERT, rowParams(row));
419
- return String(res.rows[0].id);
420
- });
1479
+ return await this.execute(
1480
+ "put",
1481
+ connection,
1482
+ "write",
1483
+ (db) => this.insertOne(db, row)
1484
+ );
421
1485
  } catch (exc) {
422
1486
  throw wrap("Failed to store image", exc);
423
1487
  }
424
1488
  }
1489
+ insertOne(db, r) {
1490
+ const id = (0, import_node_crypto.randomUUID)();
1491
+ const now = isoNow();
1492
+ if (this.dedup) {
1493
+ db.prepare(D_UPSERT).run(
1494
+ r.checksumSha256,
1495
+ r.data,
1496
+ r.mimeType,
1497
+ r.sizeBytes,
1498
+ r.width,
1499
+ r.height,
1500
+ 1,
1501
+ now
1502
+ );
1503
+ db.prepare(D_INSERT_REF2).run(
1504
+ id,
1505
+ r.checksumSha256,
1506
+ r.originalFilename,
1507
+ now,
1508
+ now
1509
+ );
1510
+ } else {
1511
+ db.prepare(INSERT2).run(
1512
+ id,
1513
+ r.data,
1514
+ r.mimeType,
1515
+ r.originalFilename,
1516
+ r.sizeBytes,
1517
+ r.width,
1518
+ r.height,
1519
+ r.checksumSha256,
1520
+ now,
1521
+ now
1522
+ );
1523
+ }
1524
+ return id;
1525
+ }
425
1526
  async putMany(rows, { connection } = {}) {
426
1527
  if (rows.length === 0) return [];
427
1528
  try {
428
- return await this.run(
429
- "put_many",
430
- connection,
431
- async (q) => {
432
- const ids = [];
433
- for (const r of rows)
434
- ids.push(String((await q.query(INSERT, rowParams(r))).rows[0].id));
435
- return ids;
436
- },
437
- { tx: true }
438
- );
1529
+ return await this.execute("put_many", connection, "write", (db) => {
1530
+ if (!this.dedup) return rows.map((r) => this.insertOne(db, r));
1531
+ const now = isoNow();
1532
+ const groups = /* @__PURE__ */ new Map();
1533
+ for (const r of rows) {
1534
+ const g = groups.get(r.checksumSha256);
1535
+ if (g) g.n += 1;
1536
+ else groups.set(r.checksumSha256, { row: r, n: 1 });
1537
+ }
1538
+ for (const [checksum, { row, n }] of groups) {
1539
+ db.prepare(D_UPSERT).run(
1540
+ checksum,
1541
+ row.data,
1542
+ row.mimeType,
1543
+ row.sizeBytes,
1544
+ row.width,
1545
+ row.height,
1546
+ n,
1547
+ now
1548
+ );
1549
+ }
1550
+ return rows.map((r) => {
1551
+ const id = (0, import_node_crypto.randomUUID)();
1552
+ db.prepare(D_INSERT_REF2).run(
1553
+ id,
1554
+ r.checksumSha256,
1555
+ r.originalFilename,
1556
+ now,
1557
+ now
1558
+ );
1559
+ return id;
1560
+ });
1561
+ });
439
1562
  } catch (exc) {
440
1563
  throw wrap("Failed to store image batch", exc);
441
1564
  }
442
1565
  }
443
1566
  async get(id, { connection } = {}) {
444
1567
  if (!isValidId(id)) return null;
1568
+ const norm = id.toLowerCase();
445
1569
  let row;
446
1570
  try {
447
- row = await this.run(
1571
+ row = await this.execute(
448
1572
  "get",
449
1573
  connection,
450
- async (q) => (await q.query(SELECT_FULL, [id])).rows[0]
1574
+ "read",
1575
+ (db) => db.prepare(this.dedup ? D_SELECT_FULL : SELECT_FULL2).get(norm)
451
1576
  );
452
1577
  } catch (exc) {
453
1578
  throw wrap("Failed to retrieve image", exc);
454
1579
  }
455
- return row ? this.toRecord(row, id) : null;
1580
+ return row ? hydrateRow(row, norm, this.opts.objectStorage) : null;
456
1581
  }
457
1582
  async getMany(ids, { connection } = {}) {
458
- const valid = ids.filter(isValidId);
1583
+ const valid = [
1584
+ ...new Set(ids.filter(isValidId).map((i) => i.toLowerCase()))
1585
+ ];
459
1586
  if (valid.length === 0) return [];
460
1587
  let rows;
461
1588
  try {
462
- rows = await this.run(
463
- "get_many",
464
- connection,
465
- async (q) => (await q.query(SELECT_MANY, [valid])).rows
466
- );
1589
+ rows = await this.execute("get_many", connection, "read", (db) => {
1590
+ const out = [];
1591
+ for (let i = 0; i < valid.length; i += IN_CHUNK) {
1592
+ const part = valid.slice(i, i + IN_CHUNK);
1593
+ out.push(
1594
+ ...db.prepare(
1595
+ this.dedup ? dSelectMany(part.length) : selectMany(part.length)
1596
+ ).all(...part)
1597
+ );
1598
+ }
1599
+ return out;
1600
+ });
467
1601
  } catch (exc) {
468
1602
  throw wrap("Failed to retrieve image batch", exc);
469
1603
  }
470
- return rows.map((r) => this.toRecord(r, String(r.id)));
1604
+ return mapLimit(
1605
+ rows,
1606
+ 4,
1607
+ (r) => hydrateRow(r, String(r.id), this.opts.objectStorage)
1608
+ );
471
1609
  }
472
1610
  async getMetadata(id, { connection } = {}) {
473
1611
  if (!isValidId(id)) return null;
474
1612
  let row;
475
1613
  try {
476
- row = await this.run(
1614
+ row = await this.execute(
477
1615
  "get_metadata",
478
1616
  connection,
479
- async (q) => (await q.query(SELECT_METADATA, [id])).rows[0]
1617
+ "read",
1618
+ (db) => db.prepare(this.dedup ? D_SELECT_METADATA : SELECT_METADATA2).get(id.toLowerCase())
480
1619
  );
481
1620
  } catch (exc) {
482
1621
  throw wrap("Failed to retrieve image metadata", exc);
@@ -494,25 +1633,37 @@ var PostgresBackend = class {
494
1633
  }
495
1634
  async getStream(id, opts) {
496
1635
  if (!isValidId(id)) return null;
1636
+ const norm = id.toLowerCase();
497
1637
  const { connection, chunkSize } = opts;
498
1638
  if (!Number.isInteger(chunkSize) || chunkSize <= 0)
499
1639
  throw new RangeError("chunkSize must be a positive integer");
500
1640
  let info;
501
1641
  try {
502
- info = await this.run(
1642
+ info = await this.execute(
503
1643
  "get_metadata",
504
1644
  connection,
505
- async (q) => (await q.query(SELECT_STREAM_INFO, [id])).rows[0]
1645
+ "read",
1646
+ (db) => db.prepare(this.dedup ? D_STREAM_INFO : STREAM_INFO).get(norm)
506
1647
  );
507
1648
  } catch (exc) {
508
1649
  throw wrap("Failed to retrieve image metadata", exc);
509
1650
  }
510
1651
  if (!info) return null;
511
- if (info.storage_backend === "object_storage")
512
- throw this.tieredWithoutStorage(id, info.object_storage_key);
513
1652
  const totalSize = info.size_bytes;
514
1653
  const start = opts.range?.start ?? 0;
515
1654
  const end = Math.min(opts.range?.end ?? totalSize - 1, totalSize - 1);
1655
+ if (info.storage_backend === "object_storage") {
1656
+ const os = this.opts.objectStorage;
1657
+ if (!os) throw tieredWithoutStorage(norm, info.object_storage_key);
1658
+ return streamFromObjectStorage(
1659
+ os,
1660
+ norm,
1661
+ info.object_storage_key,
1662
+ start,
1663
+ end,
1664
+ chunkSize
1665
+ );
1666
+ }
516
1667
  const self = this;
517
1668
  async function* generate() {
518
1669
  let offset = start;
@@ -522,23 +1673,24 @@ var PostgresBackend = class {
522
1673
  const length = Math.min(chunkSize, wanted - delivered);
523
1674
  let row;
524
1675
  try {
525
- row = await self.run(
1676
+ row = await self.execute(
526
1677
  "get_stream",
527
1678
  connection,
528
- async (q) => (await q.query(SELECT_CHUNK, [offset + 1, length, id])).rows[0]
1679
+ "read",
1680
+ (db) => db.prepare(self.dedup ? D_CHUNK : CHUNK).get(offset + 1, length, norm)
529
1681
  );
530
1682
  } catch (exc) {
531
1683
  throw wrap("Failed to stream image", exc);
532
1684
  }
533
1685
  if (!row) {
534
1686
  throw new StorageError(
535
- `Image ${JSON.stringify(id)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
1687
+ `Image ${JSON.stringify(norm)} was deleted while streaming (delivered ${delivered} of ${wanted} bytes). Pass connection with your own open transaction if you need a consistent read across concurrent writers.`
536
1688
  );
537
1689
  }
538
1690
  const chunk = row.chunk;
539
1691
  if (!chunk || chunk.length === 0) {
540
1692
  throw new StorageError(
541
- `Image ${JSON.stringify(id)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
1693
+ `Image ${JSON.stringify(norm)} ended early while streaming (delivered ${delivered} of ${wanted} bytes).`
542
1694
  );
543
1695
  }
544
1696
  yield chunk;
@@ -550,25 +1702,74 @@ var PostgresBackend = class {
550
1702
  }
551
1703
  async delete(id, { connection } = {}) {
552
1704
  if (!isValidId(id)) return false;
1705
+ const norm = id.toLowerCase();
553
1706
  try {
554
- const row = await this.run(
1707
+ if (this.dedup) {
1708
+ return await this.execute("delete", connection, "write", (db) => {
1709
+ const ref = db.prepare(D_DELETE_REF2).get(norm);
1710
+ if (!ref) return false;
1711
+ const dec = db.prepare(D_DECREMENT2).get(1, ref.checksum_sha256);
1712
+ if (dec && dec.ref_count <= 0)
1713
+ db.prepare(D_DELETE_EMPTY2).run(ref.checksum_sha256);
1714
+ return true;
1715
+ });
1716
+ }
1717
+ const row = await this.execute(
555
1718
  "delete",
556
1719
  connection,
557
- async (q) => (await q.query(DELETE_RETURNING, [id])).rows[0]
1720
+ "write",
1721
+ (db) => db.prepare(DELETE_ONE).get(norm)
558
1722
  );
1723
+ if (row?.storage_backend === "object_storage")
1724
+ await this.cleanupObjects([row.object_storage_key], connection);
559
1725
  return row !== void 0;
560
1726
  } catch (exc) {
561
1727
  throw wrap("Failed to delete image", exc);
562
1728
  }
563
1729
  }
564
1730
  async deleteMany(ids, { connection } = {}) {
565
- const valid = ids.filter(isValidId);
1731
+ const valid = [
1732
+ ...new Set(ids.filter(isValidId).map((i) => i.toLowerCase()))
1733
+ ];
566
1734
  if (valid.length === 0) return [];
567
1735
  try {
568
- const rows = await this.run(
1736
+ if (this.dedup) {
1737
+ return await this.execute("delete_many", connection, "write", (db) => {
1738
+ const deleted = [];
1739
+ for (let i = 0; i < valid.length; i += IN_CHUNK) {
1740
+ const part = valid.slice(i, i + IN_CHUNK);
1741
+ deleted.push(...db.prepare(dDeleteRefs(part.length)).all(...part));
1742
+ }
1743
+ const counts = /* @__PURE__ */ new Map();
1744
+ for (const r of deleted)
1745
+ counts.set(
1746
+ r.checksum_sha256,
1747
+ (counts.get(r.checksum_sha256) ?? 0) + 1
1748
+ );
1749
+ for (const [checksum, n] of counts) {
1750
+ const dec = db.prepare(D_DECREMENT2).get(n, checksum);
1751
+ if (dec && dec.ref_count <= 0)
1752
+ db.prepare(D_DELETE_EMPTY2).run(checksum);
1753
+ }
1754
+ return deleted.map((r) => String(r.id));
1755
+ });
1756
+ }
1757
+ const rows = await this.execute(
569
1758
  "delete_many",
570
1759
  connection,
571
- async (q) => (await q.query(DELETE_MANY, [valid])).rows
1760
+ "write",
1761
+ (db) => {
1762
+ const out = [];
1763
+ for (let i = 0; i < valid.length; i += IN_CHUNK) {
1764
+ const part = valid.slice(i, i + IN_CHUNK);
1765
+ out.push(...db.prepare(deleteMany(part.length)).all(...part));
1766
+ }
1767
+ return out;
1768
+ }
1769
+ );
1770
+ await this.cleanupObjects(
1771
+ rows.filter((r) => r.storage_backend === "object_storage").map((r) => r.object_storage_key),
1772
+ connection
572
1773
  );
573
1774
  return rows.map((r) => String(r.id));
574
1775
  } catch (exc) {
@@ -578,56 +1779,206 @@ var PostgresBackend = class {
578
1779
  async exists(id, { connection } = {}) {
579
1780
  if (!isValidId(id)) return false;
580
1781
  try {
581
- return await this.run(
1782
+ return await this.execute(
582
1783
  "exists",
583
1784
  connection,
584
- async (q) => ((await q.query(EXISTS, [id])).rowCount ?? 0) > 0
1785
+ "read",
1786
+ (db) => db.prepare(this.dedup ? D_EXISTS : EXISTS2).get(id.toLowerCase()) !== void 0
585
1787
  );
586
1788
  } catch (exc) {
587
1789
  throw wrap("Failed to check image existence", exc);
588
1790
  }
589
1791
  }
590
- // ---- helpers ---------------------------------------------------------
591
- toRecord(row, id) {
592
- if (row.storage_backend === "object_storage")
593
- throw this.tieredWithoutStorage(id, row.object_storage_key);
594
- return {
595
- id: String(row.id),
596
- data: row.data,
597
- mimeType: row.mime_type,
598
- originalFilename: row.original_filename,
599
- sizeBytes: row.size_bytes,
600
- width: row.width,
601
- height: row.height,
602
- checksumSha256: row.checksum_sha256
603
- };
1792
+ // ---- maintenance (CLI) -----------------------------------------------
1793
+ async getInfo() {
1794
+ try {
1795
+ return await this.execute("info", void 0, "read", (db) => {
1796
+ const has = (t) => db.prepare(
1797
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?"
1798
+ ).get(t) !== void 0;
1799
+ if (!has(this.dedup ? "zerobucket_image_refs" : "zerobucket_images") || this.dedup && !has("zerobucket_blobs"))
1800
+ return null;
1801
+ const j = this.dedup ? D_FROM2 : "FROM zerobucket_images";
1802
+ const size = this.dedup ? "b.size_bytes" : "size_bytes";
1803
+ const created = this.dedup ? "r.created_at" : "created_at";
1804
+ const mime = this.dedup ? "b.mime_type" : "mime_type";
1805
+ const tot = db.prepare(
1806
+ `SELECT count(*) AS n, coalesce(sum(${size}), 0) AS bytes, min(${created}) AS oldest, max(${created}) AS newest ${j}`
1807
+ ).get();
1808
+ const fmt = db.prepare(
1809
+ `SELECT ${mime} AS m, count(*) AS n, coalesce(sum(${size}), 0) AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
1810
+ ).all();
1811
+ let onDisk = null;
1812
+ if (this.opts.path !== ":memory:") {
1813
+ let total = (0, import_node_fs.statSync)(this.opts.path).size;
1814
+ for (const suffix of ["-wal", "-shm"])
1815
+ if ((0, import_node_fs.existsSync)(this.opts.path + suffix))
1816
+ total += (0, import_node_fs.statSync)(this.opts.path + suffix).size;
1817
+ onDisk = prettySize(total);
1818
+ }
1819
+ const info = {
1820
+ mode: this.dedup ? "dedup" : "classic",
1821
+ count: tot.n,
1822
+ totalBytes: Number(tot.bytes),
1823
+ oldest: tot.oldest,
1824
+ newest: tot.newest,
1825
+ onDisk,
1826
+ byFormat: fmt.map((r) => ({
1827
+ mimeType: r.m,
1828
+ count: r.n,
1829
+ bytes: Number(r.bytes)
1830
+ }))
1831
+ };
1832
+ if (this.dedup) {
1833
+ const b = db.prepare(
1834
+ "SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_blobs"
1835
+ ).get();
1836
+ info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
1837
+ } else {
1838
+ const t = db.prepare(
1839
+ "SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
1840
+ ).get();
1841
+ info.tiered = { count: t.n, bytes: Number(t.bytes) };
1842
+ }
1843
+ return info;
1844
+ });
1845
+ } catch (exc) {
1846
+ throw wrap("Failed to read storage info", exc);
1847
+ }
604
1848
  }
605
- tieredWithoutStorage(id, key) {
606
- return new StorageError(
607
- `Image ${JSON.stringify(id)} is stored in object storage (key=${JSON.stringify(key)}) but object-storage tiering is not available in this version of the npm package yet. Read it from the Python package (configured with the same bucket), or wait for npm tiering support.`
1849
+ async listImages(o = {}) {
1850
+ const where = [];
1851
+ const params = [];
1852
+ if (o.minSize !== void 0)
1853
+ where.push(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= ?`), params.push(o.minSize);
1854
+ if (o.olderThanDays !== void 0) {
1855
+ const cutoff = new Date(Date.now() - o.olderThanDays * 864e5).toISOString().replace("Z", "000+00:00");
1856
+ where.push(`${this.dedup ? "r.created_at" : "created_at"} <= ?`);
1857
+ params.push(cutoff);
1858
+ }
1859
+ if (o.onlyUntiered && !this.dedup) where.push("storage_backend = 'sqlite'");
1860
+ const from = this.dedup ? D_FROM2 : "FROM zerobucket_images";
1861
+ const idCol = this.dedup ? "r.id" : "id";
1862
+ const tierCol = this.dedup ? "0 AS tiered" : "(storage_backend = 'object_storage') AS tiered";
1863
+ const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
1864
+ const lim = o.sample ?? o.limit;
1865
+ let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
1866
+ if (lim !== void 0) sql += " LIMIT ?", params.push(lim);
1867
+ try {
1868
+ const rows = await this.execute(
1869
+ "list_images",
1870
+ void 0,
1871
+ "read",
1872
+ (db) => db.prepare(sql).all(...params)
1873
+ );
1874
+ return rows.map((r) => ({
1875
+ id: String(r.id),
1876
+ tiered: Boolean(r.tiered)
1877
+ }));
1878
+ } catch (exc) {
1879
+ throw wrap("Failed to list images", exc);
1880
+ }
1881
+ }
1882
+ /**
1883
+ * Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
1884
+ * reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the
1885
+ * commit outcome is unknown. Failures never fail the delete; they are reported through onOperation.
1886
+ */
1887
+ async cleanupObjects(keys, connection) {
1888
+ const os = this.opts.objectStorage;
1889
+ if (!os || connection || keys.length === 0) return;
1890
+ await Promise.all(
1891
+ keys.map(async (key) => {
1892
+ const start = performance.now();
1893
+ try {
1894
+ await os.delete(key);
1895
+ this.emit("object_storage_delete", start, true, null, 0);
1896
+ } catch (exc) {
1897
+ this.emit("object_storage_delete", start, false, msg(exc), 0);
1898
+ }
1899
+ })
608
1900
  );
609
1901
  }
1902
+ /**
1903
+ * Move an image's bytes into object storage. null = no such image, false = already tiered, true = tiered now.
1904
+ *
1905
+ * SAFETY (same guarantee as Postgres): the upload happens INSIDE a BEGIN IMMEDIATE transaction; if it fails the
1906
+ * transaction rolls back and the row is untouched.
1907
+ *
1908
+ * TRADEOFF (coarser than Postgres): SQLite has no row locks, so BEGIN IMMEDIATE locks the WHOLE DATABASE for the
1909
+ * upload. Other processes' writes wait; this process's writes queue behind an async lock; reads are not blocked.
1910
+ */
1911
+ async tierToObjectStorage(id, { connection } = {}) {
1912
+ if (this.dedup)
1913
+ throw new StorageError(
1914
+ "tierToObjectStorage() is not supported in dedup mode (a blob may be shared by many images)."
1915
+ );
1916
+ const os = this.opts.objectStorage;
1917
+ if (!os) {
1918
+ throw new StorageError(
1919
+ "tierToObjectStorage() requires objectStorage to be configured (new ZeroBucket({ sqlite, objectStorage: new ObjectStorage({ bucket }) }))."
1920
+ );
1921
+ }
1922
+ if (connection) {
1923
+ throw new StorageError(
1924
+ "tierToObjectStorage() does not accept `connection` on SQLite: it needs its own dedicated connection for the BEGIN IMMEDIATE transaction."
1925
+ );
1926
+ }
1927
+ if (this.opts.path === ":memory:") {
1928
+ throw new StorageError(
1929
+ "tierToObjectStorage() needs a database file: an in-memory SQLite database cannot be shared with its dedicated tiering connection."
1930
+ );
1931
+ }
1932
+ if (!isValidId(id)) return null;
1933
+ const norm = id.toLowerCase();
1934
+ const start = performance.now();
1935
+ let retries = 0;
1936
+ try {
1937
+ await this.ready();
1938
+ const result = await this.writeLock.run(async () => {
1939
+ const tdb = new this.Database(this.opts.path, {
1940
+ timeout: ATTEMPT_BUSY_MS
1941
+ });
1942
+ try {
1943
+ await this.retrying(
1944
+ () => tdb.exec("BEGIN IMMEDIATE"),
1945
+ (n) => retries = n
1946
+ );
1947
+ try {
1948
+ const row = tdb.prepare(SELECT_FOR_TIERING2).get(norm);
1949
+ if (!row) {
1950
+ tdb.exec("ROLLBACK");
1951
+ return null;
1952
+ }
1953
+ if (row.storage_backend !== "sqlite") {
1954
+ tdb.exec("ROLLBACK");
1955
+ return false;
1956
+ }
1957
+ await os.upload(norm, row.data, {
1958
+ mimeType: row.mime_type
1959
+ });
1960
+ tdb.prepare(UPDATE_AFTER_TIERING2).run(os.bucket, norm, isoNow(), norm);
1961
+ tdb.exec("COMMIT");
1962
+ return true;
1963
+ } catch (exc) {
1964
+ try {
1965
+ tdb.exec("ROLLBACK");
1966
+ } catch {
1967
+ }
1968
+ throw exc;
1969
+ }
1970
+ } finally {
1971
+ tdb.close();
1972
+ }
1973
+ });
1974
+ this.emit("tier_to_object_storage", start, true, null, retries);
1975
+ return result;
1976
+ } catch (exc) {
1977
+ this.emit("tier_to_object_storage", start, false, msg(exc), retries);
1978
+ throw wrap("Failed to tier image to object storage", exc);
1979
+ }
1980
+ }
610
1981
  };
611
- function rowParams(r) {
612
- return [
613
- r.data,
614
- r.mimeType,
615
- r.originalFilename,
616
- r.sizeBytes,
617
- r.width,
618
- r.height,
619
- r.checksumSha256
620
- ];
621
- }
622
- function wrap(prefix, exc) {
623
- return exc instanceof StorageError ? exc : new StorageError(`${prefix}: ${msg(exc)}`, { cause: exc });
624
- }
625
- function msg(exc) {
626
- return exc instanceof Error ? exc.message : String(exc);
627
- }
628
- function sleep(ms) {
629
- return new Promise((resolve) => setTimeout(resolve, ms));
630
- }
631
1982
 
632
1983
  // src/validation.ts
633
1984
  var FORMAT_TO_MIME = {
@@ -771,7 +2122,18 @@ var ZeroBucket = class _ZeroBucket {
771
2122
  beforeGet;
772
2123
  beforePut;
773
2124
  constructor(options = {}) {
774
- this.backend = options.backend ?? new PostgresBackend({
2125
+ if (options.sqlite !== void 0 && (options.connectionString || options.pool)) {
2126
+ throw new TypeError(
2127
+ "Choose one database: `sqlite` OR `connectionString`/`pool`, not both."
2128
+ );
2129
+ }
2130
+ this.backend = options.backend ?? (options.sqlite !== void 0 ? new SQLiteBackend({
2131
+ path: options.sqlite,
2132
+ autoMigrate: options.autoMigrate,
2133
+ dedup: options.dedup,
2134
+ objectStorage: options.objectStorage,
2135
+ onOperation: options.onOperation
2136
+ }) : new PostgresBackend({
775
2137
  connectionString: options.connectionString,
776
2138
  pool: options.pool,
777
2139
  autoMigrate: options.autoMigrate,
@@ -780,8 +2142,10 @@ var ZeroBucket = class _ZeroBucket {
780
2142
  poolMinSize: options.poolMinSize,
781
2143
  poolMaxSize: options.poolMaxSize,
782
2144
  poolTimeoutMs: options.poolTimeoutMs,
783
- onOperation: options.onOperation
784
- });
2145
+ onOperation: options.onOperation,
2146
+ objectStorage: options.objectStorage,
2147
+ dedup: options.dedup
2148
+ }));
785
2149
  this.maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
786
2150
  this.maxPixels = options.maxPixels ?? DEFAULT_MAX_PIXELS;
787
2151
  this.allowedFormats = options.allowedFormats ?? SUPPORTED_FORMATS;
@@ -865,7 +2229,7 @@ var ZeroBucket = class _ZeroBucket {
865
2229
  sizeBytes,
866
2230
  width,
867
2231
  height,
868
- checksumSha256: (0, import_node_crypto.createHash)("sha256").update(finalData).digest("hex")
2232
+ checksumSha256: (0, import_node_crypto2.createHash)("sha256").update(finalData).digest("hex")
869
2233
  };
870
2234
  }
871
2235
  /** Validate, optionally optimize, checksum and store one image. Returns its id (a UUID). */
@@ -902,7 +2266,7 @@ var ZeroBucket = class _ZeroBucket {
902
2266
  }
903
2267
  const results = new Array(images.length);
904
2268
  const prepared = [];
905
- await mapLimit(images, PREPARE_CONCURRENCY, async (image, i) => {
2269
+ await mapLimit2(images, PREPARE_CONCURRENCY, async (image, i) => {
906
2270
  try {
907
2271
  const row = await this.prepareRow(image, {
908
2272
  ...options,
@@ -1118,6 +2482,32 @@ var ZeroBucket = class _ZeroBucket {
1118
2482
  success: true
1119
2483
  }));
1120
2484
  }
2485
+ /**
2486
+ * Move an image's bytes from Postgres into object storage; reads, streams and
2487
+ * deletes keep working transparently. Returns true if tiered now, false if it was
2488
+ * already tiered (safe to re-run in a backfill). Throws ImageNotFoundError if missing.
2489
+ *
2490
+ * The upload runs inside the database transaction: if it fails, the row is left
2491
+ * completely untouched. Explicit and opt-in: put() never tiers automatically.
2492
+ * Not gated by beforeGet/beforePut (it is a maintenance operation, as in the Python package).
2493
+ */
2494
+ async tierToObjectStorage(imageId, options = {}) {
2495
+ const result = await this.backend.tierToObjectStorage(imageId, options);
2496
+ if (result === null) throw new ImageNotFoundError(imageId);
2497
+ return result;
2498
+ }
2499
+ /**
2500
+ * One-time, non-destructive copy of classic-mode images into a dedup instance's tables, keeping every
2501
+ * id. Call it on an instance created with `dedup: true`. Safe to re-run. See PostgresBackend for details.
2502
+ */
2503
+ async migrateClassicToDedup() {
2504
+ if (!this.backend.migrateClassicToDedup) {
2505
+ throw new StorageError(
2506
+ "This backend does not support migrateClassicToDedup()."
2507
+ );
2508
+ }
2509
+ return this.backend.migrateClassicToDedup();
2510
+ }
1121
2511
  /** Release the pool (if ZeroBucket created it). */
1122
2512
  close() {
1123
2513
  return this.backend.close();
@@ -1171,7 +2561,7 @@ async function readInput(image, filename, maxBytes) {
1171
2561
  "Unsupported image input. Pass a Buffer, Uint8Array, file path string, Blob/File, or a readable stream."
1172
2562
  );
1173
2563
  }
1174
- async function mapLimit(items, limit, fn) {
2564
+ async function mapLimit2(items, limit, fn) {
1175
2565
  let next = 0;
1176
2566
  const workers = Array.from(
1177
2567
  { length: Math.min(limit, items.length) },
@@ -1188,6 +2578,119 @@ function errText(exc) {
1188
2578
  return exc instanceof Error ? exc.message : String(exc);
1189
2579
  }
1190
2580
 
2581
+ // src/object-storage.ts
2582
+ var ObjectStorage = class {
2583
+ constructor(opts) {
2584
+ this.opts = opts;
2585
+ if (!opts.bucket)
2586
+ throw new TypeError("ObjectStorage requires a bucket name");
2587
+ this.bucket = opts.bucket;
2588
+ }
2589
+ opts;
2590
+ bucket;
2591
+ clientPromise;
2592
+ connection() {
2593
+ this.clientPromise ??= (async () => {
2594
+ let sdk;
2595
+ try {
2596
+ sdk = await import("@aws-sdk/client-s3");
2597
+ } catch (cause) {
2598
+ throw new StorageError(
2599
+ "Object-storage tiering requires the optional dependency '@aws-sdk/client-s3'. Install it with: npm install @aws-sdk/client-s3",
2600
+ { cause }
2601
+ );
2602
+ }
2603
+ const o = this.opts;
2604
+ const client = o.client ?? new sdk.S3Client({
2605
+ endpoint: o.endpoint,
2606
+ region: o.region ?? process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? "us-east-1",
2607
+ credentials: o.credentials,
2608
+ forcePathStyle: o.forcePathStyle ?? o.endpoint !== void 0
2609
+ });
2610
+ return { sdk, client };
2611
+ })().catch((exc) => {
2612
+ this.clientPromise = void 0;
2613
+ throw exc;
2614
+ });
2615
+ return this.clientPromise;
2616
+ }
2617
+ async upload(key, data, { mimeType }) {
2618
+ const { sdk, client } = await this.connection();
2619
+ try {
2620
+ await client.send(
2621
+ new sdk.PutObjectCommand({
2622
+ Bucket: this.bucket,
2623
+ Key: key,
2624
+ Body: data,
2625
+ ContentType: mimeType
2626
+ })
2627
+ );
2628
+ } catch (exc) {
2629
+ throw new StorageError(
2630
+ `Failed to upload ${JSON.stringify(key)} to bucket ${JSON.stringify(this.bucket)}: ${msg2(exc)}`,
2631
+ { cause: exc }
2632
+ );
2633
+ }
2634
+ }
2635
+ async download(key) {
2636
+ return this.get(key, void 0);
2637
+ }
2638
+ async downloadRange(key, start, endInclusive) {
2639
+ return this.get(key, `bytes=${start}-${endInclusive}`);
2640
+ }
2641
+ async get(key, range) {
2642
+ const { sdk, client } = await this.connection();
2643
+ try {
2644
+ const res = await client.send(
2645
+ new sdk.GetObjectCommand({
2646
+ Bucket: this.bucket,
2647
+ Key: key,
2648
+ Range: range
2649
+ })
2650
+ );
2651
+ return Buffer.from(await res.Body.transformToByteArray());
2652
+ } catch (exc) {
2653
+ throw new StorageError(
2654
+ `Failed to download ${JSON.stringify(key)} from bucket ${JSON.stringify(this.bucket)}: ${msg2(exc)}`,
2655
+ { cause: exc }
2656
+ );
2657
+ }
2658
+ }
2659
+ async delete(key) {
2660
+ const { sdk, client } = await this.connection();
2661
+ try {
2662
+ await client.send(
2663
+ new sdk.DeleteObjectCommand({ Bucket: this.bucket, Key: key })
2664
+ );
2665
+ } catch (exc) {
2666
+ throw new StorageError(
2667
+ `Failed to delete ${JSON.stringify(key)} from bucket ${JSON.stringify(this.bucket)}: ${msg2(exc)}`,
2668
+ { cause: exc }
2669
+ );
2670
+ }
2671
+ }
2672
+ async exists(key) {
2673
+ const { sdk, client } = await this.connection();
2674
+ try {
2675
+ await client.send(
2676
+ new sdk.HeadObjectCommand({ Bucket: this.bucket, Key: key })
2677
+ );
2678
+ return true;
2679
+ } catch (exc) {
2680
+ const e = exc;
2681
+ if (e.name === "NotFound" || e.name === "NoSuchKey" || e.$metadata?.httpStatusCode === 404)
2682
+ return false;
2683
+ throw new StorageError(
2684
+ `Failed to check ${JSON.stringify(key)} in bucket ${JSON.stringify(this.bucket)}: ${msg2(exc)}`,
2685
+ { cause: exc }
2686
+ );
2687
+ }
2688
+ }
2689
+ };
2690
+ function msg2(exc) {
2691
+ return exc instanceof Error ? exc.message : String(exc);
2692
+ }
2693
+
1191
2694
  // src/validators/pdf.ts
1192
2695
  var DEFAULT_MAX_PDF_BYTES = 20 * 1024 * 1024;
1193
2696
  var PDFValidator = class {
@@ -1231,8 +2734,10 @@ var PDFValidator = class {
1231
2734
  ImageNotFoundError,
1232
2735
  ImageTooLargeError,
1233
2736
  ImageValidationError,
2737
+ ObjectStorage,
1234
2738
  PDFValidator,
1235
2739
  PostgresBackend,
2740
+ SQLiteBackend,
1236
2741
  SUPPORTED_FORMATS,
1237
2742
  StorageError,
1238
2743
  UnsupportedFormatError,