zerobucket 0.4.0 → 0.5.0

package/dist/{client-WZ4VRCPi.d.cts → client-C7xs1dlr.d.cts}

@@ -125,6 +125,50 @@ interface StreamOptions$1 extends CallOptions {
         end: number;
     };
 }
+/** Filters for maintenance listings (CLI `verify` / `tier`). All optional; combined with AND. */
+interface ImageListOptions {
+    /** Return a random sample of this many images instead of all of them. */
+    sample?: number;
+    limit?: number;
+    /** Only images with size_bytes >= this. */
+    minSize?: number;
+    /** Only images created at least this many days ago. */
+    olderThanDays?: number;
+    /** Only images whose bytes are still in the database (not tiered). */
+    onlyUntiered?: boolean;
+}
+interface ImageListEntry {
+    id: string;
+    /** Bytes live in object storage, not in the database. */
+    tiered: boolean;
+}
+/** Storage statistics for `zerobucket info`. */
+interface StorageInfo {
+    mode: "classic" | "dedup";
+    /** Number of images (references, in dedup mode). */
+    count: number;
+    /** Sum of every image's size_bytes as recorded by the application (logical size). */
+    totalBytes: number;
+    oldest: string | null;
+    newest: string | null;
+    /** Human-readable on-disk size, if the database can tell. */
+    onDisk: string | null;
+    byFormat: {
+        mimeType: string;
+        count: number;
+        bytes: number;
+    }[];
+    /** Classic mode: images currently living in object storage. */
+    tiered?: {
+        count: number;
+        bytes: number;
+    };
+    /** Dedup mode: distinct stored blobs and the bytes they physically occupy. */
+    dedup?: {
+        blobs: number;
+        storedBytes: number;
+    };
+}
 interface StorageBackend {
     /** Resolve once the backend is connected and migrated. Safe to call repeatedly. */
     ready(): Promise<void>;
@@ -153,6 +197,10 @@ interface StorageBackend {
         duplicateReferencesFound: number;
         alreadyMigrated: number;
     }>;
+    /** Maintenance: storage statistics, or null if the tables do not exist yet. */
+    getInfo?(): Promise<StorageInfo | null>;
+    /** Maintenance: list ids matching the filters (oldest first, or random when sampling). */
+    listImages?(opts?: ImageListOptions): Promise<ImageListEntry[]>;
     close(): Promise<void>;
 }
 
@@ -422,4 +470,4 @@ declare function readInput(image: ImageInput, filename: string | undefined, maxB
     filename: string | undefined;
 }>;
 
-export { type BatchDeleteResult as B, type CallOptions as C, DEFAULT_MAX_BYTES as D, type GetOptions as G, type Image as I, type OperationEvent as O, type PreparedRow as P, type Queryable as Q, type StorageBackend as S, type ValidatedContent as V, type WritableLike as W, ZeroBucket as Z, type ObjectStorageLike as a, type StoredRecord as b, type StoredRecordMetadata as c, type StreamOptions$1 as d, type ContentValidator as e, type BatchGetResult as f, type BatchPutResult as g, type BeforeGetHook as h, type BeforePutHook as i, type Connection as j, type ImageInput as k, type ImageMetadata as l, ObjectStorage as m, type ObjectStorageOptions as n, type OptimizationResult as o, type OptimizeOptions as p, type PutManyOptions as q, type PutOptions as r, type SqliteDatabaseLike as s, type StreamOptions as t, type ZeroBucketOptions as u, optimizeImage as v, readInput as w };
+export { type BatchDeleteResult as B, type CallOptions as C, DEFAULT_MAX_BYTES as D, type GetOptions as G, type ImageListOptions as I, type OperationEvent as O, type PreparedRow as P, type Queryable as Q, type StorageBackend as S, type ValidatedContent as V, type WritableLike as W, ZeroBucket as Z, type ObjectStorageLike as a, type StoredRecord as b, type StoredRecordMetadata as c, type StreamOptions$1 as d, type StorageInfo as e, type ImageListEntry as f, type ContentValidator as g, type BatchGetResult as h, type BatchPutResult as i, type BeforeGetHook as j, type BeforePutHook as k, type Connection as l, type Image as m, type ImageInput as n, type ImageMetadata as o, ObjectStorage as p, type ObjectStorageOptions as q, type OptimizationResult as r, type OptimizeOptions as s, type PutManyOptions as t, type PutOptions as u, type SqliteDatabaseLike as v, type StreamOptions as w, type ZeroBucketOptions as x, optimizeImage as y, readInput as z };

package/dist/{client-WZ4VRCPi.d.ts → client-C7xs1dlr.d.ts}

@@ -125,6 +125,50 @@ interface StreamOptions$1 extends CallOptions {
         end: number;
     };
 }
+/** Filters for maintenance listings (CLI `verify` / `tier`). All optional; combined with AND. */
+interface ImageListOptions {
+    /** Return a random sample of this many images instead of all of them. */
+    sample?: number;
+    limit?: number;
+    /** Only images with size_bytes >= this. */
+    minSize?: number;
+    /** Only images created at least this many days ago. */
+    olderThanDays?: number;
+    /** Only images whose bytes are still in the database (not tiered). */
+    onlyUntiered?: boolean;
+}
+interface ImageListEntry {
+    id: string;
+    /** Bytes live in object storage, not in the database. */
+    tiered: boolean;
+}
+/** Storage statistics for `zerobucket info`. */
+interface StorageInfo {
+    mode: "classic" | "dedup";
+    /** Number of images (references, in dedup mode). */
+    count: number;
+    /** Sum of every image's size_bytes as recorded by the application (logical size). */
+    totalBytes: number;
+    oldest: string | null;
+    newest: string | null;
+    /** Human-readable on-disk size, if the database can tell. */
+    onDisk: string | null;
+    byFormat: {
+        mimeType: string;
+        count: number;
+        bytes: number;
+    }[];
+    /** Classic mode: images currently living in object storage. */
+    tiered?: {
+        count: number;
+        bytes: number;
+    };
+    /** Dedup mode: distinct stored blobs and the bytes they physically occupy. */
+    dedup?: {
+        blobs: number;
+        storedBytes: number;
+    };
+}
 interface StorageBackend {
     /** Resolve once the backend is connected and migrated. Safe to call repeatedly. */
     ready(): Promise<void>;
@@ -153,6 +197,10 @@ interface StorageBackend {
         duplicateReferencesFound: number;
         alreadyMigrated: number;
     }>;
+    /** Maintenance: storage statistics, or null if the tables do not exist yet. */
+    getInfo?(): Promise<StorageInfo | null>;
+    /** Maintenance: list ids matching the filters (oldest first, or random when sampling). */
+    listImages?(opts?: ImageListOptions): Promise<ImageListEntry[]>;
     close(): Promise<void>;
 }
 
@@ -422,4 +470,4 @@ declare function readInput(image: ImageInput, filename: string | undefined, maxB
     filename: string | undefined;
 }>;
 
-export { type BatchDeleteResult as B, type CallOptions as C, DEFAULT_MAX_BYTES as D, type GetOptions as G, type Image as I, type OperationEvent as O, type PreparedRow as P, type Queryable as Q, type StorageBackend as S, type ValidatedContent as V, type WritableLike as W, ZeroBucket as Z, type ObjectStorageLike as a, type StoredRecord as b, type StoredRecordMetadata as c, type StreamOptions$1 as d, type ContentValidator as e, type BatchGetResult as f, type BatchPutResult as g, type BeforeGetHook as h, type BeforePutHook as i, type Connection as j, type ImageInput as k, type ImageMetadata as l, ObjectStorage as m, type ObjectStorageOptions as n, type OptimizationResult as o, type OptimizeOptions as p, type PutManyOptions as q, type PutOptions as r, type SqliteDatabaseLike as s, type StreamOptions as t, type ZeroBucketOptions as u, optimizeImage as v, readInput as w };
+export { type BatchDeleteResult as B, type CallOptions as C, DEFAULT_MAX_BYTES as D, type GetOptions as G, type ImageListOptions as I, type OperationEvent as O, type PreparedRow as P, type Queryable as Q, type StorageBackend as S, type ValidatedContent as V, type WritableLike as W, ZeroBucket as Z, type ObjectStorageLike as a, type StoredRecord as b, type StoredRecordMetadata as c, type StreamOptions$1 as d, type StorageInfo as e, type ImageListEntry as f, type ContentValidator as g, type BatchGetResult as h, type BatchPutResult as i, type BeforeGetHook as j, type BeforePutHook as k, type Connection as l, type Image as m, type ImageInput as n, type ImageMetadata as o, ObjectStorage as p, type ObjectStorageOptions as q, type OptimizationResult as r, type OptimizeOptions as s, type PutManyOptions as t, type PutOptions as u, type SqliteDatabaseLike as v, type StreamOptions as w, type ZeroBucketOptions as x, optimizeImage as y, readInput as z };

package/dist/http.d.cts

@@ -1,5 +1,5 @@
 import { IncomingMessage, ServerResponse } from 'node:http';
-import { Z as ZeroBucket } from './client-WZ4VRCPi.cjs';
+import { Z as ZeroBucket } from './client-C7xs1dlr.cjs';
 import 'pg';
 
 /**

package/dist/http.d.ts

@@ -1,5 +1,5 @@
 import { IncomingMessage, ServerResponse } from 'node:http';
-import { Z as ZeroBucket } from './client-WZ4VRCPi.js';
+import { Z as ZeroBucket } from './client-C7xs1dlr.js';
 import 'pg';
 
 /**

package/dist/index.cjs

@@ -187,6 +187,17 @@ async function* streamFromObjectStorage(os, id, key, start, end, chunkSize) {
     offset = last + 1;
   }
 }
+function prettySize(bytes) {
+  if (bytes < 1024) return `${bytes} bytes`;
+  const units = ["kB", "MB", "GB", "TB"];
+  let v = bytes / 1024;
+  let i = 0;
+  while (v >= 1024 && i < units.length - 1) {
+    v /= 1024;
+    i += 1;
+  }
+  return `${v.toFixed(1)} ${units[i]}`;
+}
 
 // src/adapters/postgres.ts
 var SCHEMA = `
@@ -821,6 +832,95 @@ var PostgresBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- maintenance (CLI) -----------------------------------------------
+  async getInfo() {
+    try {
+      return await this.run("info", void 0, async (q) => {
+        const imgTable = this.dedup ? "zerobucket_image_refs" : "zerobucket_images";
+        const present = await q.query(
+          "SELECT to_regclass($1) AS a, to_regclass('zerobucket_blobs') AS b",
+          [imgTable]
+        );
+        if (!present.rows[0].a || this.dedup && !present.rows[0].b)
+          return null;
+        const j = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
+        const size = this.dedup ? "b.size_bytes" : "size_bytes";
+        const created = this.dedup ? "r.created_at" : "created_at";
+        const mime = this.dedup ? "b.mime_type" : "mime_type";
+        const tot = (await q.query(
+          `SELECT count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes, min(${created})::text AS oldest, max(${created})::text AS newest ${j}`
+        )).rows[0];
+        const disk = (await q.query(
+          this.dedup ? "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_blobs') + pg_total_relation_size('zerobucket_image_refs')) AS s" : "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_images')) AS s"
+        )).rows[0].s;
+        const fmt = (await q.query(
+          `SELECT ${mime} AS m, count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
+        )).rows;
+        const info = {
+          mode: this.dedup ? "dedup" : "classic",
+          count: tot.n,
+          totalBytes: Number(tot.bytes),
+          // Postgres prints the offset as "+00"; Python's CLI (and ISO-8601) print "+00:00". Normalise so output is identical.
+          oldest: tot.oldest && tot.oldest.replace(/([+-]\d\d)$/, "$1:00"),
+          newest: tot.newest && tot.newest.replace(/([+-]\d\d)$/, "$1:00"),
+          onDisk: disk,
+          byFormat: fmt.map((r) => ({
+            mimeType: r.m,
+            count: r.n,
+            bytes: Number(r.bytes)
+          }))
+        };
+        if (this.dedup) {
+          const b = (await q.query(
+            "SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_blobs"
+          )).rows[0];
+          info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
+        } else {
+          const t = (await q.query(
+            "SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
+          )).rows[0];
+          info.tiered = { count: t.n, bytes: Number(t.bytes) };
+        }
+        return info;
+      });
+    } catch (exc) {
+      throw wrap("Failed to read storage info", exc);
+    }
+  }
+  async listImages(o = {}) {
+    const where = [];
+    const params = [];
+    const add = (sql2, v) => (params.push(v), where.push(sql2.replace("$?", `$${params.length}`)));
+    if (o.minSize !== void 0)
+      add(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= $?`, o.minSize);
+    if (o.olderThanDays !== void 0)
+      add(
+        `${this.dedup ? "r.created_at" : "created_at"} <= now() - make_interval(days => $?::int)`,
+        o.olderThanDays
+      );
+    if (o.onlyUntiered && !this.dedup)
+      where.push("storage_backend = 'postgres'");
+    const from = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
+    const idCol = this.dedup ? "r.id" : "id";
+    const tierCol = this.dedup ? "false AS tiered" : "(storage_backend = 'object_storage') AS tiered";
+    const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
+    const lim = o.sample ?? o.limit;
+    let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
+    if (lim !== void 0) {
+      params.push(lim);
+      sql += ` LIMIT $${params.length}`;
+    }
+    try {
+      const rows = await this.run(
+        "list_images",
+        void 0,
+        async (q) => (await q.query(sql, params)).rows
+      );
+      return rows.map((r) => ({ id: String(r.id), tiered: Boolean(r.tiered) }));
+    } catch (exc) {
+      throw wrap("Failed to list images", exc);
+    }
+  }
   // ---- dedup deletes / migration ---------------------------------------
   async deleteDedup(id, connection) {
     try {
@@ -1102,6 +1202,7 @@ function isQueryable(c) {
 
 // src/adapters/sqlite.ts
 var import_node_crypto = require("crypto");
+var import_node_fs = require("fs");
 var SCHEMA2 = `
 CREATE TABLE IF NOT EXISTS zerobucket_images (
     id                  TEXT PRIMARY KEY,
@@ -1688,6 +1789,96 @@ var SQLiteBackend = class {
       throw wrap("Failed to check image existence", exc);
     }
   }
+  // ---- maintenance (CLI) -----------------------------------------------
+  async getInfo() {
+    try {
+      return await this.execute("info", void 0, "read", (db) => {
+        const has = (t) => db.prepare(
+          "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?"
+        ).get(t) !== void 0;
+        if (!has(this.dedup ? "zerobucket_image_refs" : "zerobucket_images") || this.dedup && !has("zerobucket_blobs"))
+          return null;
+        const j = this.dedup ? D_FROM2 : "FROM zerobucket_images";
+        const size = this.dedup ? "b.size_bytes" : "size_bytes";
+        const created = this.dedup ? "r.created_at" : "created_at";
+        const mime = this.dedup ? "b.mime_type" : "mime_type";
+        const tot = db.prepare(
+          `SELECT count(*) AS n, coalesce(sum(${size}), 0) AS bytes, min(${created}) AS oldest, max(${created}) AS newest ${j}`
+        ).get();
+        const fmt = db.prepare(
+          `SELECT ${mime} AS m, count(*) AS n, coalesce(sum(${size}), 0) AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
+        ).all();
+        let onDisk = null;
+        if (this.opts.path !== ":memory:") {
+          let total = (0, import_node_fs.statSync)(this.opts.path).size;
+          for (const suffix of ["-wal", "-shm"])
+            if ((0, import_node_fs.existsSync)(this.opts.path + suffix))
+              total += (0, import_node_fs.statSync)(this.opts.path + suffix).size;
+          onDisk = prettySize(total);
+        }
+        const info = {
+          mode: this.dedup ? "dedup" : "classic",
+          count: tot.n,
+          totalBytes: Number(tot.bytes),
+          oldest: tot.oldest,
+          newest: tot.newest,
+          onDisk,
+          byFormat: fmt.map((r) => ({
+            mimeType: r.m,
+            count: r.n,
+            bytes: Number(r.bytes)
+          }))
+        };
+        if (this.dedup) {
+          const b = db.prepare(
+            "SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_blobs"
+          ).get();
+          info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
+        } else {
+          const t = db.prepare(
+            "SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
+          ).get();
+          info.tiered = { count: t.n, bytes: Number(t.bytes) };
+        }
+        return info;
+      });
+    } catch (exc) {
+      throw wrap("Failed to read storage info", exc);
+    }
+  }
+  async listImages(o = {}) {
+    const where = [];
+    const params = [];
+    if (o.minSize !== void 0)
+      where.push(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= ?`), params.push(o.minSize);
+    if (o.olderThanDays !== void 0) {
+      const cutoff = new Date(Date.now() - o.olderThanDays * 864e5).toISOString().replace("Z", "000+00:00");
+      where.push(`${this.dedup ? "r.created_at" : "created_at"} <= ?`);
+      params.push(cutoff);
+    }
+    if (o.onlyUntiered && !this.dedup) where.push("storage_backend = 'sqlite'");
+    const from = this.dedup ? D_FROM2 : "FROM zerobucket_images";
+    const idCol = this.dedup ? "r.id" : "id";
+    const tierCol = this.dedup ? "0 AS tiered" : "(storage_backend = 'object_storage') AS tiered";
+    const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
+    const lim = o.sample ?? o.limit;
+    let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
+    if (lim !== void 0) sql += " LIMIT ?", params.push(lim);
+    try {
+      const rows = await this.execute(
+        "list_images",
+        void 0,
+        "read",
+        (db) => db.prepare(sql).all(...params)
+      );
+      return rows.map((r) => ({
+        id: String(r.id),
+        tiered: Boolean(r.tiered)
+      }));
+    } catch (exc) {
+      throw wrap("Failed to list images", exc);
+    }
+  }
   /**
    * Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
    * reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the