zerobucket 0.4.0 → 0.5.0

package/dist/index.d.cts

@@ -1,5 +1,5 @@
-import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as ContentValidator, V as ValidatedContent } from './client-WZ4VRCPi.cjs';
-export { B as BatchDeleteResult, f as BatchGetResult, g as BatchPutResult, h as BeforeGetHook, i as BeforePutHook, j as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, I as Image, k as ImageInput, l as ImageMetadata, m as ObjectStorage, n as ObjectStorageOptions, o as OptimizationResult, p as OptimizeOptions, q as PutManyOptions, r as PutOptions, Q as Queryable, s as SqliteDatabaseLike, t as StreamOptions, W as WritableLike, Z as ZeroBucket, u as ZeroBucketOptions, v as optimizeImage, w as readInput } from './client-WZ4VRCPi.cjs';
+import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, g as ContentValidator, V as ValidatedContent } from './client-C7xs1dlr.cjs';
+export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as ObjectStorage, q as ObjectStorageOptions, r as OptimizationResult, s as OptimizeOptions, t as PutManyOptions, u as PutOptions, Q as Queryable, v as SqliteDatabaseLike, w as StreamOptions, W as WritableLike, Z as ZeroBucket, x as ZeroBucketOptions, y as optimizeImage, z as readInput } from './client-C7xs1dlr.cjs';
 import { Pool } from 'pg';
 
 /**
@@ -149,6 +149,8 @@ declare class PostgresBackend implements StorageBackend {
     delete(id: string, { connection }?: CallOptions): Promise<boolean>;
     deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
     exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    getInfo(): Promise<StorageInfo | null>;
+    listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
     private deleteDedup;
     private deleteManyDedup;
     /**
@@ -253,6 +255,8 @@ declare class SQLiteBackend implements StorageBackend {
     delete(id: string, { connection }?: CallOptions): Promise<boolean>;
     deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
     exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    getInfo(): Promise<StorageInfo | null>;
+    listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
     /**
      * Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
      * reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as ContentValidator, V as ValidatedContent } from './client-WZ4VRCPi.js';
-export { B as BatchDeleteResult, f as BatchGetResult, g as BatchPutResult, h as BeforeGetHook, i as BeforePutHook, j as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, I as Image, k as ImageInput, l as ImageMetadata, m as ObjectStorage, n as ObjectStorageOptions, o as OptimizationResult, p as OptimizeOptions, q as PutManyOptions, r as PutOptions, Q as Queryable, s as SqliteDatabaseLike, t as StreamOptions, W as WritableLike, Z as ZeroBucket, u as ZeroBucketOptions, v as optimizeImage, w as readInput } from './client-WZ4VRCPi.js';
+import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, g as ContentValidator, V as ValidatedContent } from './client-C7xs1dlr.js';
+export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as ObjectStorage, q as ObjectStorageOptions, r as OptimizationResult, s as OptimizeOptions, t as PutManyOptions, u as PutOptions, Q as Queryable, v as SqliteDatabaseLike, w as StreamOptions, W as WritableLike, Z as ZeroBucket, x as ZeroBucketOptions, y as optimizeImage, z as readInput } from './client-C7xs1dlr.js';
 import { Pool } from 'pg';
 
 /**
@@ -149,6 +149,8 @@ declare class PostgresBackend implements StorageBackend {
     delete(id: string, { connection }?: CallOptions): Promise<boolean>;
     deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
     exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    getInfo(): Promise<StorageInfo | null>;
+    listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
     private deleteDedup;
     private deleteManyDedup;
     /**
@@ -253,6 +255,8 @@ declare class SQLiteBackend implements StorageBackend {
     delete(id: string, { connection }?: CallOptions): Promise<boolean>;
     deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
     exists(id: string, { connection }?: CallOptions): Promise<boolean>;
+    getInfo(): Promise<StorageInfo | null>;
+    listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
     /**
      * Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
      * reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the

package/dist/index.js

@@ -87,6 +87,17 @@ async function* streamFromObjectStorage(os, id, key, start, end, chunkSize) {
     offset = last + 1;
   }
 }
+function prettySize(bytes) {
+  if (bytes < 1024) return `${bytes} bytes`;
+  const units = ["kB", "MB", "GB", "TB"];
+  let v = bytes / 1024;
+  let i = 0;
+  while (v >= 1024 && i < units.length - 1) {
+    v /= 1024;
+    i += 1;
+  }
+  return `${v.toFixed(1)} ${units[i]}`;
+}
 
 // src/adapters/postgres.ts
 var SCHEMA = `
@@ -721,6 +732,95 @@ var PostgresBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- maintenance (CLI) -----------------------------------------------
+  async getInfo() {
+    try {
+      return await this.run("info", void 0, async (q) => {
+        const imgTable = this.dedup ? "zerobucket_image_refs" : "zerobucket_images";
+        const present = await q.query(
+          "SELECT to_regclass($1) AS a, to_regclass('zerobucket_blobs') AS b",
+          [imgTable]
+        );
+        if (!present.rows[0].a || this.dedup && !present.rows[0].b)
+          return null;
+        const j = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
+        const size = this.dedup ? "b.size_bytes" : "size_bytes";
+        const created = this.dedup ? "r.created_at" : "created_at";
+        const mime = this.dedup ? "b.mime_type" : "mime_type";
+        const tot = (await q.query(
+          `SELECT count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes, min(${created})::text AS oldest, max(${created})::text AS newest ${j}`
+        )).rows[0];
+        const disk = (await q.query(
+          this.dedup ? "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_blobs') + pg_total_relation_size('zerobucket_image_refs')) AS s" : "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_images')) AS s"
+        )).rows[0].s;
+        const fmt = (await q.query(
+          `SELECT ${mime} AS m, count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
+        )).rows;
+        const info = {
+          mode: this.dedup ? "dedup" : "classic",
+          count: tot.n,
+          totalBytes: Number(tot.bytes),
+          // Postgres prints the offset as "+00"; Python's CLI (and ISO-8601) print "+00:00". Normalise so output is identical.
+          oldest: tot.oldest && tot.oldest.replace(/([+-]\d\d)$/, "$1:00"),
+          newest: tot.newest && tot.newest.replace(/([+-]\d\d)$/, "$1:00"),
+          onDisk: disk,
+          byFormat: fmt.map((r) => ({
+            mimeType: r.m,
+            count: r.n,
+            bytes: Number(r.bytes)
+          }))
+        };
+        if (this.dedup) {
+          const b = (await q.query(
+            "SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_blobs"
+          )).rows[0];
+          info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
+        } else {
+          const t = (await q.query(
+            "SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
+          )).rows[0];
+          info.tiered = { count: t.n, bytes: Number(t.bytes) };
+        }
+        return info;
+      });
+    } catch (exc) {
+      throw wrap("Failed to read storage info", exc);
+    }
+  }
+  async listImages(o = {}) {
+    const where = [];
+    const params = [];
+    const add = (sql2, v) => (params.push(v), where.push(sql2.replace("$?", `$${params.length}`)));
+    if (o.minSize !== void 0)
+      add(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= $?`, o.minSize);
+    if (o.olderThanDays !== void 0)
+      add(
+        `${this.dedup ? "r.created_at" : "created_at"} <= now() - make_interval(days => $?::int)`,
+        o.olderThanDays
+      );
+    if (o.onlyUntiered && !this.dedup)
+      where.push("storage_backend = 'postgres'");
+    const from = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
+    const idCol = this.dedup ? "r.id" : "id";
+    const tierCol = this.dedup ? "false AS tiered" : "(storage_backend = 'object_storage') AS tiered";
+    const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
+    const lim = o.sample ?? o.limit;
+    let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
+    if (lim !== void 0) {
+      params.push(lim);
+      sql += ` LIMIT $${params.length}`;
+    }
+    try {
+      const rows = await this.run(
+        "list_images",
+        void 0,
+        async (q) => (await q.query(sql, params)).rows
+      );
+      return rows.map((r) => ({ id: String(r.id), tiered: Boolean(r.tiered) }));
+    } catch (exc) {
+      throw wrap("Failed to list images", exc);
+    }
+  }
   // ---- dedup deletes / migration ---------------------------------------
   async deleteDedup(id, connection) {
     try {
@@ -1002,6 +1102,7 @@ function isQueryable(c) {
 
 // src/adapters/sqlite.ts
 import { randomUUID } from "crypto";
+import { existsSync, statSync } from "fs";
 var SCHEMA2 = `
 CREATE TABLE IF NOT EXISTS zerobucket_images (
     id                  TEXT PRIMARY KEY,
@@ -1588,6 +1689,96 @@ var SQLiteBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- maintenance (CLI) -----------------------------------------------
+  async getInfo() {
+    try {
+      return await this.execute("info", void 0, "read", (db) => {
+        const has = (t) => db.prepare(
+          "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?"
+        ).get(t) !== void 0;
+        if (!has(this.dedup ? "zerobucket_image_refs" : "zerobucket_images") || this.dedup && !has("zerobucket_blobs"))
+          return null;
+        const j = this.dedup ? D_FROM2 : "FROM zerobucket_images";
+        const size = this.dedup ? "b.size_bytes" : "size_bytes";
+        const created = this.dedup ? "r.created_at" : "created_at";
+        const mime = this.dedup ? "b.mime_type" : "mime_type";
+        const tot = db.prepare(
+          `SELECT count(*) AS n, coalesce(sum(${size}), 0) AS bytes, min(${created}) AS oldest, max(${created}) AS newest ${j}`
+        ).get();
+        const fmt = db.prepare(
+          `SELECT ${mime} AS m, count(*) AS n, coalesce(sum(${size}), 0) AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
+        ).all();
+        let onDisk = null;
+        if (this.opts.path !== ":memory:") {
+          let total = statSync(this.opts.path).size;
+          for (const suffix of ["-wal", "-shm"])
+            if (existsSync(this.opts.path + suffix))
+              total += statSync(this.opts.path + suffix).size;
+          onDisk = prettySize(total);
+        }
+        const info = {
+          mode: this.dedup ? "dedup" : "classic",
+          count: tot.n,
+          totalBytes: Number(tot.bytes),
+          oldest: tot.oldest,
+          newest: tot.newest,
+          onDisk,
+          byFormat: fmt.map((r) => ({
+            mimeType: r.m,
+            count: r.n,
+            bytes: Number(r.bytes)
+          }))
+        };
+        if (this.dedup) {
+          const b = db.prepare(
+            "SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_blobs"
+          ).get();
+          info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
+        } else {
+          const t = db.prepare(
+            "SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
+          ).get();
+          info.tiered = { count: t.n, bytes: Number(t.bytes) };
+        }
+        return info;
+      });
+    } catch (exc) {
+      throw wrap("Failed to read storage info", exc);
+    }
+  }
+  async listImages(o = {}) {
+    const where = [];
+    const params = [];
+    if (o.minSize !== void 0)
+      where.push(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= ?`), params.push(o.minSize);
+    if (o.olderThanDays !== void 0) {
+      const cutoff = new Date(Date.now() - o.olderThanDays * 864e5).toISOString().replace("Z", "000+00:00");
+      where.push(`${this.dedup ? "r.created_at" : "created_at"} <= ?`);
+      params.push(cutoff);
+    }
+    if (o.onlyUntiered && !this.dedup) where.push("storage_backend = 'sqlite'");
+    const from = this.dedup ? D_FROM2 : "FROM zerobucket_images";
+    const idCol = this.dedup ? "r.id" : "id";
+    const tierCol = this.dedup ? "0 AS tiered" : "(storage_backend = 'object_storage') AS tiered";
+    const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
+    const lim = o.sample ?? o.limit;
+    let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
+    if (lim !== void 0) sql += " LIMIT ?", params.push(lim);
+    try {
+      const rows = await this.execute(
+        "list_images",
+        void 0,
+        "read",
+        (db) => db.prepare(sql).all(...params)
+      );
+      return rows.map((r) => ({
+        id: String(r.id),
+        tiered: Boolean(r.tiered)
+      }));
+    } catch (exc) {
+      throw wrap("Failed to list images", exc);
+    }
+  }
   /**
    * Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
    * reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the