zerobucket 0.4.0 → 0.6.0

package/dist/index.cjs

@@ -54,10 +54,14 @@ __export(src_exports, {
 });
 module.exports = __toCommonJS(src_exports);
 
+// node_modules/tsup/assets/cjs_shims.js
+var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.tagName.toUpperCase() === "SCRIPT" ? document.currentScript.src : new URL("main.js", document.baseURI).href;
+var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
+
 // src/client.ts
 var import_node_crypto2 = require("crypto");
 var import_promises = require("fs/promises");
-var import_node_path = require("path");
+var import_node_path2 = require("path");
 
 // src/errors.ts
 var ZeroBucketError = class extends Error {
@@ -187,6 +191,17 @@ async function* streamFromObjectStorage(os, id, key, start, end, chunkSize) {
     offset = last + 1;
   }
 }
+function prettySize(bytes) {
+  if (bytes < 1024) return `${bytes} bytes`;
+  const units = ["kB", "MB", "GB", "TB"];
+  let v = bytes / 1024;
+  let i = 0;
+  while (v >= 1024 && i < units.length - 1) {
+    v /= 1024;
+    i += 1;
+  }
+  return `${v.toFixed(1)} ${units[i]}`;
+}
 
 // src/adapters/postgres.ts
 var SCHEMA = `
@@ -821,6 +836,95 @@ var PostgresBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- maintenance (CLI) -----------------------------------------------
+  async getInfo() {
+    try {
+      return await this.run("info", void 0, async (q) => {
+        const imgTable = this.dedup ? "zerobucket_image_refs" : "zerobucket_images";
+        const present = await q.query(
+          "SELECT to_regclass($1) AS a, to_regclass('zerobucket_blobs') AS b",
+          [imgTable]
+        );
+        if (!present.rows[0].a || this.dedup && !present.rows[0].b)
+          return null;
+        const j = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
+        const size = this.dedup ? "b.size_bytes" : "size_bytes";
+        const created = this.dedup ? "r.created_at" : "created_at";
+        const mime = this.dedup ? "b.mime_type" : "mime_type";
+        const tot = (await q.query(
+          `SELECT count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes, min(${created})::text AS oldest, max(${created})::text AS newest ${j}`
+        )).rows[0];
+        const disk = (await q.query(
+          this.dedup ? "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_blobs') + pg_total_relation_size('zerobucket_image_refs')) AS s" : "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_images')) AS s"
+        )).rows[0].s;
+        const fmt = (await q.query(
+          `SELECT ${mime} AS m, count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
+        )).rows;
+        const info = {
+          mode: this.dedup ? "dedup" : "classic",
+          count: tot.n,
+          totalBytes: Number(tot.bytes),
+          // Postgres prints the offset as "+00"; Python's CLI (and ISO-8601) print "+00:00". Normalise so output is identical.
+          oldest: tot.oldest && tot.oldest.replace(/([+-]\d\d)$/, "$1:00"),
+          newest: tot.newest && tot.newest.replace(/([+-]\d\d)$/, "$1:00"),
+          onDisk: disk,
+          byFormat: fmt.map((r) => ({
+            mimeType: r.m,
+            count: r.n,
+            bytes: Number(r.bytes)
+          }))
+        };
+        if (this.dedup) {
+          const b = (await q.query(
+            "SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_blobs"
+          )).rows[0];
+          info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
+        } else {
+          const t = (await q.query(
+            "SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
+          )).rows[0];
+          info.tiered = { count: t.n, bytes: Number(t.bytes) };
+        }
+        return info;
+      });
+    } catch (exc) {
+      throw wrap("Failed to read storage info", exc);
+    }
+  }
+  async listImages(o = {}) {
+    const where = [];
+    const params = [];
+    const add = (sql2, v) => (params.push(v), where.push(sql2.replace("$?", `$${params.length}`)));
+    if (o.minSize !== void 0)
+      add(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= $?`, o.minSize);
+    if (o.olderThanDays !== void 0)
+      add(
+        `${this.dedup ? "r.created_at" : "created_at"} <= now() - make_interval(days => $?::int)`,
+        o.olderThanDays
+      );
+    if (o.onlyUntiered && !this.dedup)
+      where.push("storage_backend = 'postgres'");
+    const from = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
+    const idCol = this.dedup ? "r.id" : "id";
+    const tierCol = this.dedup ? "false AS tiered" : "(storage_backend = 'object_storage') AS tiered";
+    const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
+    const lim = o.sample ?? o.limit;
+    let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
+    if (lim !== void 0) {
+      params.push(lim);
+      sql += ` LIMIT $${params.length}`;
+    }
+    try {
+      const rows = await this.run(
+        "list_images",
+        void 0,
+        async (q) => (await q.query(sql, params)).rows
+      );
+      return rows.map((r) => ({ id: String(r.id), tiered: Boolean(r.tiered) }));
+    } catch (exc) {
+      throw wrap("Failed to list images", exc);
+    }
+  }
   // ---- dedup deletes / migration ---------------------------------------
   async deleteDedup(id, connection) {
     try {
@@ -1102,6 +1206,7 @@ function isQueryable(c) {
 
 // src/adapters/sqlite.ts
 var import_node_crypto = require("crypto");
+var import_node_fs = require("fs");
 var SCHEMA2 = `
 CREATE TABLE IF NOT EXISTS zerobucket_images (
     id                  TEXT PRIMARY KEY,
@@ -1688,6 +1793,96 @@ var SQLiteBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- maintenance (CLI) -----------------------------------------------
+  async getInfo() {
+    try {
+      return await this.execute("info", void 0, "read", (db) => {
+        const has = (t) => db.prepare(
+          "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?"
+        ).get(t) !== void 0;
+        if (!has(this.dedup ? "zerobucket_image_refs" : "zerobucket_images") || this.dedup && !has("zerobucket_blobs"))
+          return null;
+        const j = this.dedup ? D_FROM2 : "FROM zerobucket_images";
+        const size = this.dedup ? "b.size_bytes" : "size_bytes";
+        const created = this.dedup ? "r.created_at" : "created_at";
+        const mime = this.dedup ? "b.mime_type" : "mime_type";
+        const tot = db.prepare(
+          `SELECT count(*) AS n, coalesce(sum(${size}), 0) AS bytes, min(${created}) AS oldest, max(${created}) AS newest ${j}`
+        ).get();
+        const fmt = db.prepare(
+          `SELECT ${mime} AS m, count(*) AS n, coalesce(sum(${size}), 0) AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
+        ).all();
+        let onDisk = null;
+        if (this.opts.path !== ":memory:") {
+          let total = (0, import_node_fs.statSync)(this.opts.path).size;
+          for (const suffix of ["-wal", "-shm"])
+            if ((0, import_node_fs.existsSync)(this.opts.path + suffix))
+              total += (0, import_node_fs.statSync)(this.opts.path + suffix).size;
+          onDisk = prettySize(total);
+        }
+        const info = {
+          mode: this.dedup ? "dedup" : "classic",
+          count: tot.n,
+          totalBytes: Number(tot.bytes),
+          oldest: tot.oldest,
+          newest: tot.newest,
+          onDisk,
+          byFormat: fmt.map((r) => ({
+            mimeType: r.m,
+            count: r.n,
+            bytes: Number(r.bytes)
+          }))
+        };
+        if (this.dedup) {
+          const b = db.prepare(
+            "SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_blobs"
+          ).get();
+          info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
+        } else {
+          const t = db.prepare(
+            "SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
+          ).get();
+          info.tiered = { count: t.n, bytes: Number(t.bytes) };
+        }
+        return info;
+      });
+    } catch (exc) {
+      throw wrap("Failed to read storage info", exc);
+    }
+  }
+  async listImages(o = {}) {
+    const where = [];
+    const params = [];
+    if (o.minSize !== void 0)
+      where.push(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= ?`), params.push(o.minSize);
+    if (o.olderThanDays !== void 0) {
+      const cutoff = new Date(Date.now() - o.olderThanDays * 864e5).toISOString().replace("Z", "000+00:00");
+      where.push(`${this.dedup ? "r.created_at" : "created_at"} <= ?`);
+      params.push(cutoff);
+    }
+    if (o.onlyUntiered && !this.dedup) where.push("storage_backend = 'sqlite'");
+    const from = this.dedup ? D_FROM2 : "FROM zerobucket_images";
+    const idCol = this.dedup ? "r.id" : "id";
+    const tierCol = this.dedup ? "0 AS tiered" : "(storage_backend = 'object_storage') AS tiered";
+    const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
+    const lim = o.sample ?? o.limit;
+    let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
+    if (lim !== void 0) sql += " LIMIT ?", params.push(lim);
+    try {
+      const rows = await this.execute(
+        "list_images",
+        void 0,
+        "read",
+        (db) => db.prepare(sql).all(...params)
+      );
+      return rows.map((r) => ({
+        id: String(r.id),
+        tiered: Boolean(r.tiered)
+      }));
+    } catch (exc) {
+      throw wrap("Failed to list images", exc);
+    }
+  }
   /**
    * Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
    * reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the
@@ -1789,11 +1984,191 @@ var SQLiteBackend = class {
   }
 };
 
+// src/heic.ts
+var import_node_module = require("module");
+var import_node_path = require("path");
+var import_node_worker_threads = require("worker_threads");
+var HEIC_BRANDS = /* @__PURE__ */ new Set([
+  "heic",
+  "heix",
+  "hevc",
+  "heim",
+  "heis",
+  "mif1",
+  "msf1"
+]);
+function looksLikeHeic(data) {
+  if (data.length < 12) return false;
+  if (String.fromCharCode(data[4], data[5], data[6], data[7]) !== "ftyp")
+    return false;
+  return HEIC_BRANDS.has(
+    String.fromCharCode(data[8], data[9], data[10], data[11])
+  );
+}
+var HEIC_INSTALL_HINT = "npm install libheif-js";
+var WORKER_SOURCE = `
+const { parentPort, workerData } = require("node:worker_threads");
+const libheif = require(workerData.libPath);
+// libheif-js reports decode problems by dumping a raw error object to the console. Capture that text instead:
+// it becomes part of the error we return (the useful diagnosis), and nothing is printed to the host's stderr.
+let diag = "";
+for (const level of ["log", "info", "warn", "error"]) {
+  console[level] = (...args) => {
+    diag = args.map((a) => (a && a.message) || (typeof a === "string" ? a : "")).filter(Boolean).join(" ").trim();
+  };
+}
+const describe = (e) => {
+  const base = String((e && e.message) || e);
+  return diag && !base.includes(diag) ? base + ": " + diag : base;
+};
+function open(bytes) {
+  const images = new libheif.HeifDecoder().decode(bytes);
+  if (!images || images.length === 0) throw new Error("no images found in HEIF container");
+  return images[0];
+}
+parentPort.on("message", async ({ id, op, data }) => {
+  diag = "";
+  try {
+    const img = open(new Uint8Array(data));
+    const width = img.get_width();
+    const height = img.get_height();
+    if (op === "info") return parentPort.postMessage({ id, ok: true, width, height });
+    const out = await new Promise((resolve, reject) =>
+      img.display({ data: new Uint8ClampedArray(width * height * 4), width, height }, (d) =>
+        d ? resolve(d) : reject(new Error("HEIF processing error")),
+      ),
+    );
+    if (op === "validate") return parentPort.postMessage({ id, ok: true, width, height });
+    const rgba = out.data.buffer;
+    parentPort.postMessage({ id, ok: true, width, height, rgba }, [rgba]);
+  } catch (e) {
+    parentPort.postMessage({ id, ok: false, error: describe(e) });
+  }
+});
+`;
+var HeicWorkerError = class extends StorageError {
+};
+var worker;
+var nextId = 1;
+var pending = /* @__PURE__ */ new Map();
+var stats = { info: 0, validate: 0, decode: 0, workersStarted: 0 };
+var resolveLib = () => {
+  const attempts = [
+    () => (0, import_node_module.createRequire)(importMetaUrl).resolve("libheif-js/wasm-bundle"),
+    // next to / above this package
+    () => (0, import_node_module.createRequire)((0, import_node_path.join)(process.cwd(), "noop.js")).resolve(
+      "libheif-js/wasm-bundle"
+    )
+    // the app's own node_modules
+  ];
+  let last;
+  for (const attempt of attempts) {
+    try {
+      return attempt();
+    } catch (e) {
+      last = e;
+    }
+  }
+  throw last;
+};
+function heicAvailable() {
+  try {
+    resolveLib();
+    return true;
+  } catch {
+    return false;
+  }
+}
+function releaseIfIdle(w) {
+  for (const p of pending.values()) if (p.owner === w) return;
+  w.unref();
+}
+function startWorker() {
+  let libPath;
+  try {
+    libPath = resolveLib();
+  } catch (cause) {
+    throw new StorageError(
+      `HEIC support requires the optional dependency 'libheif-js'. Install it with: ${HEIC_INSTALL_HINT}`,
+      { cause }
+    );
+  }
+  const w = new import_node_worker_threads.Worker(WORKER_SOURCE, { eval: true, workerData: { libPath } });
+  stats.workersStarted += 1;
+  w.on(
+    "message",
+    (m) => {
+      const p = pending.get(m.id);
+      if (!p) return;
+      pending.delete(m.id);
+      releaseIfIdle(w);
+      if (m.ok) p.resolve(m);
+      else p.reject(new Error(m.error));
+    }
+  );
+  const fail = (err) => {
+    if (worker === w) worker = void 0;
+    for (const [id, p] of pending) {
+      if (p.owner !== w) continue;
+      pending.delete(id);
+      p.reject(err);
+    }
+  };
+  w.on(
+    "error",
+    (e) => fail(
+      new HeicWorkerError(
+        `HEIC decoder worker crashed: ${e instanceof Error ? e.message : String(e)}`,
+        { cause: e }
+      )
+    )
+  );
+  w.on(
+    "exit",
+    (code) => fail(
+      new HeicWorkerError(
+        `HEIC decoder worker exited unexpectedly (code ${code})`
+      )
+    )
+  );
+  w.unref();
+  return w;
+}
+function call(op, data) {
+  stats[op] += 1;
+  const w = worker ??= startWorker();
+  const id = nextId++;
+  const copy = new Uint8Array(data).buffer;
+  return new Promise((resolve, reject) => {
+    pending.set(id, { owner: w, resolve, reject });
+    w.ref();
+    w.postMessage({ id, op, data: copy }, [copy]);
+  });
+}
+async function heicInfo(data) {
+  const r = await call("info", data);
+  return { width: r.width, height: r.height };
+}
+async function heicValidate(data) {
+  const r = await call("validate", data);
+  return { width: r.width, height: r.height };
+}
+async function decodeHeic(data) {
+  const r = await call("decode", data);
+  return {
+    width: r.width,
+    height: r.height,
+    rgba: Buffer.from(r.rgba)
+  };
+}
+
 // src/validation.ts
 var FORMAT_TO_MIME = {
   jpeg: "image/jpeg",
   png: "image/png",
-  webp: "image/webp"
+  webp: "image/webp",
+  heic: "image/heic"
+  // optional: needs `npm install libheif-js` (see heic.ts)
 };
 var SUPPORTED_FORMATS = Object.keys(FORMAT_TO_MIME);
 var DEFAULT_MAX_PIXELS = 89e6;
@@ -1821,6 +2196,8 @@ async function validateImage(data, opts) {
   const sizeBytes = data.length;
   if (sizeBytes > maxBytes) throw new ImageTooLargeError(sizeBytes, maxBytes);
   if (sizeBytes === 0) throw new CorruptedImageError("Image data is empty");
+  if (looksLikeHeic(data))
+    return validateHeic(data, { maxBytes, maxPixels, allowedFormats });
   const sharp = await loadSharp();
   const make = () => sharp(data, { limitInputPixels: maxPixels, failOn: "error" });
   let format;
@@ -1856,6 +2233,45 @@ async function validateImage(data, opts) {
   }
   return { mimeType: FORMAT_TO_MIME[format], width, height, sizeBytes };
 }
+async function validateHeic(data, {
+  maxBytes,
+  maxPixels,
+  allowedFormats
+}) {
+  const sizeBytes = data.length;
+  if (!heicAvailable()) {
+    throw new ImageValidationError(
+      `This looks like a HEIC/HEIF image, which requires an optional dependency. Install it with: ${HEIC_INSTALL_HINT}`
+    );
+  }
+  if (!allowedFormats.includes("heic"))
+    throw new UnsupportedFormatError("heic", allowedFormats);
+  let width;
+  let height;
+  try {
+    ({ width, height } = await heicInfo(data));
+  } catch (exc) {
+    if (exc instanceof StorageError) throw exc;
+    throw new CorruptedImageError(
+      `Could not decode image: ${errMessage(exc)}`,
+      { cause: exc }
+    );
+  }
+  if (!width || !height)
+    throw new CorruptedImageError("Image has no readable dimensions");
+  if (width * height > maxPixels)
+    throw new ImageTooLargeError(sizeBytes, maxBytes);
+  try {
+    await heicValidate(data);
+  } catch (exc) {
+    if (exc instanceof StorageError) throw exc;
+    throw new CorruptedImageError(
+      `Image data is truncated or corrupted: ${errMessage(exc)}`,
+      { cause: exc }
+    );
+  }
+  return { mimeType: FORMAT_TO_MIME.heic, width, height, sizeBytes };
+}
 function isPixelLimit(exc) {
   return /pixel limit/i.test(errMessage(exc));
 }
@@ -1864,28 +2280,53 @@ function errMessage(exc) {
 }
 
 // src/optimize.ts
+var ENCODABLE_FORMATS = ["jpeg", "png", "webp"];
 var DEFAULT_JPEG_QUALITY = 90;
 var DEFAULT_WEBP_QUALITY = 88;
-var FORMAT_ALIASES = { jpg: "jpeg" };
 async function optimizeImage(data, opts) {
   const { maxWidth, quality, maxBytes, maxPixels = DEFAULT_MAX_PIXELS } = opts;
   const sharp = await loadSharp();
   const originalSizeBytes = data.length;
-  const base = sharp(data, { limitInputPixels: maxPixels, failOn: "error" });
-  const meta = await base.metadata();
-  const requested = (opts.format ?? meta.format ?? "jpeg").toLowerCase();
-  const target = FORMAT_ALIASES[requested] ?? requested;
-  if (!SUPPORTED_FORMATS.includes(target)) {
+  const requestedRaw = opts.format?.toLowerCase();
+  const requested = requestedRaw === "jpg" ? "jpeg" : requestedRaw;
+  if (requested === "heic" || requested === "heif") {
     throw new ImageValidationError(
-      `Cannot re-encode to unsupported format ${JSON.stringify(target)}. Supported: ${JSON.stringify(
-        [...SUPPORTED_FORMATS].sort()
-      )}`
+      "Encoding to HEIC is not available in the Node package (there is no HEVC encoder for Node). Optimize to jpeg, webp or png instead, or store the original HEIC without `optimize`."
     );
   }
-  let pipeline = sharp(data, {
+  let input = data;
+  let inputOptions = {
     limitInputPixels: maxPixels,
     failOn: "error"
-  }).rotate();
+  };
+  let sourceFormat;
+  if (looksLikeHeic(data)) {
+    if (!heicAvailable()) {
+      throw new ImageValidationError(
+        `This looks like a HEIC/HEIF image, which requires an optional dependency. Install it with: ${HEIC_INSTALL_HINT}`
+      );
+    }
+    if (!requested) {
+      throw new ImageValidationError(
+        "Optimizing a HEIC image needs an explicit output format, because re-encoding to HEIC is not available in the Node package. Pass optimize: { format: 'jpeg' | 'webp' | 'png' } (for example { format: 'webp', maxWidth: 1600 })."
+      );
+    }
+    const decoded = await decodeHeic(data);
+    input = decoded.rgba;
+    inputOptions = {
+      raw: { width: decoded.width, height: decoded.height, channels: 4 }
+    };
+    sourceFormat = "heic";
+  } else {
+    sourceFormat = (await sharp(data, inputOptions).metadata()).format;
+  }
+  const target = requested ?? sourceFormat ?? "jpeg";
+  if (!ENCODABLE_FORMATS.includes(target)) {
+    throw new ImageValidationError(
+      `Cannot re-encode to unsupported format ${JSON.stringify(target)}. Supported: ${JSON.stringify([...ENCODABLE_FORMATS].sort())}`
+    );
+  }
+  let pipeline = sharp(input, inputOptions).rotate();
   if (maxWidth !== void 0) {
     pipeline = pipeline.resize({
       width: maxWidth,
@@ -2328,7 +2769,7 @@ async function readInput(image, filename, maxBytes) {
     if (size > maxBytes) throw new ImageTooLargeError(size, maxBytes);
     return {
       data: await (0, import_promises.readFile)(image),
-      filename: filename ?? (0, import_node_path.basename)(image)
+      filename: filename ?? (0, import_node_path2.basename)(image)
     };
   }
   if (Buffer.isBuffer(image)) return { data: image, filename };
@@ -2363,7 +2804,7 @@ async function readInput(image, filename, maxBytes) {
     const p = image.path;
     return {
       data: Buffer.concat(chunks, total),
-      filename: filename ?? (typeof p === "string" ? (0, import_node_path.basename)(p) : void 0)
+      filename: filename ?? (typeof p === "string" ? (0, import_node_path2.basename)(p) : void 0)
     };
   }
   throw new TypeError(