zerobucket 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -20
- package/dist/bin.js +2506 -0
- package/dist/{chunk-W2C37XYV.js → chunk-SCPC3QDR.js} +1 -1
- package/dist/{client-WZ4VRCPi.d.cts → client-C7xs1dlr.d.cts} +49 -1
- package/dist/{client-WZ4VRCPi.d.ts → client-C7xs1dlr.d.ts} +49 -1
- package/dist/http.d.cts +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/http.js +1 -1
- package/dist/index.cjs +456 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +6 -2
- package/dist/index.d.ts +6 -2
- package/dist/index.js +450 -13
- package/dist/index.js.map +1 -1
- package/package.json +16 -4
- /package/dist/{chunk-W2C37XYV.js.map → chunk-SCPC3QDR.js.map} +0 -0
package/dist/index.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as ContentValidator, V as ValidatedContent } from './client-
|
|
2
|
-
export { B as BatchDeleteResult,
|
|
1
|
+
import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, g as ContentValidator, V as ValidatedContent } from './client-C7xs1dlr.cjs';
|
|
2
|
+
export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as ObjectStorage, q as ObjectStorageOptions, r as OptimizationResult, s as OptimizeOptions, t as PutManyOptions, u as PutOptions, Q as Queryable, v as SqliteDatabaseLike, w as StreamOptions, W as WritableLike, Z as ZeroBucket, x as ZeroBucketOptions, y as optimizeImage, z as readInput } from './client-C7xs1dlr.cjs';
|
|
3
3
|
import { Pool } from 'pg';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -149,6 +149,8 @@ declare class PostgresBackend implements StorageBackend {
|
|
|
149
149
|
delete(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
150
150
|
deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
|
|
151
151
|
exists(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
152
|
+
getInfo(): Promise<StorageInfo | null>;
|
|
153
|
+
listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
|
|
152
154
|
private deleteDedup;
|
|
153
155
|
private deleteManyDedup;
|
|
154
156
|
/**
|
|
@@ -253,6 +255,8 @@ declare class SQLiteBackend implements StorageBackend {
|
|
|
253
255
|
delete(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
254
256
|
deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
|
|
255
257
|
exists(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
258
|
+
getInfo(): Promise<StorageInfo | null>;
|
|
259
|
+
listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
|
|
256
260
|
/**
|
|
257
261
|
* Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
|
|
258
262
|
* reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as ContentValidator, V as ValidatedContent } from './client-
|
|
2
|
-
export { B as BatchDeleteResult,
|
|
1
|
+
import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, g as ContentValidator, V as ValidatedContent } from './client-C7xs1dlr.js';
|
|
2
|
+
export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as ObjectStorage, q as ObjectStorageOptions, r as OptimizationResult, s as OptimizeOptions, t as PutManyOptions, u as PutOptions, Q as Queryable, v as SqliteDatabaseLike, w as StreamOptions, W as WritableLike, Z as ZeroBucket, x as ZeroBucketOptions, y as optimizeImage, z as readInput } from './client-C7xs1dlr.js';
|
|
3
3
|
import { Pool } from 'pg';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -149,6 +149,8 @@ declare class PostgresBackend implements StorageBackend {
|
|
|
149
149
|
delete(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
150
150
|
deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
|
|
151
151
|
exists(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
152
|
+
getInfo(): Promise<StorageInfo | null>;
|
|
153
|
+
listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
|
|
152
154
|
private deleteDedup;
|
|
153
155
|
private deleteManyDedup;
|
|
154
156
|
/**
|
|
@@ -253,6 +255,8 @@ declare class SQLiteBackend implements StorageBackend {
|
|
|
253
255
|
delete(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
254
256
|
deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
|
|
255
257
|
exists(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
258
|
+
getInfo(): Promise<StorageInfo | null>;
|
|
259
|
+
listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
|
|
256
260
|
/**
|
|
257
261
|
* Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
|
|
258
262
|
* reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the
|
package/dist/index.js
CHANGED
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
StorageError,
|
|
9
9
|
UnsupportedFormatError,
|
|
10
10
|
ZeroBucketError
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-SCPC3QDR.js";
|
|
12
12
|
|
|
13
13
|
// src/client.ts
|
|
14
14
|
import { createHash } from "crypto";
|
|
@@ -87,6 +87,17 @@ async function* streamFromObjectStorage(os, id, key, start, end, chunkSize) {
|
|
|
87
87
|
offset = last + 1;
|
|
88
88
|
}
|
|
89
89
|
}
|
|
90
|
+
function prettySize(bytes) {
|
|
91
|
+
if (bytes < 1024) return `${bytes} bytes`;
|
|
92
|
+
const units = ["kB", "MB", "GB", "TB"];
|
|
93
|
+
let v = bytes / 1024;
|
|
94
|
+
let i = 0;
|
|
95
|
+
while (v >= 1024 && i < units.length - 1) {
|
|
96
|
+
v /= 1024;
|
|
97
|
+
i += 1;
|
|
98
|
+
}
|
|
99
|
+
return `${v.toFixed(1)} ${units[i]}`;
|
|
100
|
+
}
|
|
90
101
|
|
|
91
102
|
// src/adapters/postgres.ts
|
|
92
103
|
var SCHEMA = `
|
|
@@ -721,6 +732,95 @@ var PostgresBackend = class {
|
|
|
721
732
|
throw wrap("Failed to check image existence", exc);
|
|
722
733
|
}
|
|
723
734
|
}
|
|
735
|
+
// ---- maintenance (CLI) -----------------------------------------------
|
|
736
|
+
async getInfo() {
|
|
737
|
+
try {
|
|
738
|
+
return await this.run("info", void 0, async (q) => {
|
|
739
|
+
const imgTable = this.dedup ? "zerobucket_image_refs" : "zerobucket_images";
|
|
740
|
+
const present = await q.query(
|
|
741
|
+
"SELECT to_regclass($1) AS a, to_regclass('zerobucket_blobs') AS b",
|
|
742
|
+
[imgTable]
|
|
743
|
+
);
|
|
744
|
+
if (!present.rows[0].a || this.dedup && !present.rows[0].b)
|
|
745
|
+
return null;
|
|
746
|
+
const j = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
|
|
747
|
+
const size = this.dedup ? "b.size_bytes" : "size_bytes";
|
|
748
|
+
const created = this.dedup ? "r.created_at" : "created_at";
|
|
749
|
+
const mime = this.dedup ? "b.mime_type" : "mime_type";
|
|
750
|
+
const tot = (await q.query(
|
|
751
|
+
`SELECT count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes, min(${created})::text AS oldest, max(${created})::text AS newest ${j}`
|
|
752
|
+
)).rows[0];
|
|
753
|
+
const disk = (await q.query(
|
|
754
|
+
this.dedup ? "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_blobs') + pg_total_relation_size('zerobucket_image_refs')) AS s" : "SELECT pg_size_pretty(pg_total_relation_size('zerobucket_images')) AS s"
|
|
755
|
+
)).rows[0].s;
|
|
756
|
+
const fmt = (await q.query(
|
|
757
|
+
`SELECT ${mime} AS m, count(*)::int AS n, coalesce(sum(${size}), 0)::text AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
|
|
758
|
+
)).rows;
|
|
759
|
+
const info = {
|
|
760
|
+
mode: this.dedup ? "dedup" : "classic",
|
|
761
|
+
count: tot.n,
|
|
762
|
+
totalBytes: Number(tot.bytes),
|
|
763
|
+
// Postgres prints the offset as "+00"; Python's CLI (and ISO-8601) print "+00:00". Normalise so output is identical.
|
|
764
|
+
oldest: tot.oldest && tot.oldest.replace(/([+-]\d\d)$/, "$1:00"),
|
|
765
|
+
newest: tot.newest && tot.newest.replace(/([+-]\d\d)$/, "$1:00"),
|
|
766
|
+
onDisk: disk,
|
|
767
|
+
byFormat: fmt.map((r) => ({
|
|
768
|
+
mimeType: r.m,
|
|
769
|
+
count: r.n,
|
|
770
|
+
bytes: Number(r.bytes)
|
|
771
|
+
}))
|
|
772
|
+
};
|
|
773
|
+
if (this.dedup) {
|
|
774
|
+
const b = (await q.query(
|
|
775
|
+
"SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_blobs"
|
|
776
|
+
)).rows[0];
|
|
777
|
+
info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
|
|
778
|
+
} else {
|
|
779
|
+
const t = (await q.query(
|
|
780
|
+
"SELECT count(*)::int AS n, coalesce(sum(size_bytes), 0)::text AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
|
|
781
|
+
)).rows[0];
|
|
782
|
+
info.tiered = { count: t.n, bytes: Number(t.bytes) };
|
|
783
|
+
}
|
|
784
|
+
return info;
|
|
785
|
+
});
|
|
786
|
+
} catch (exc) {
|
|
787
|
+
throw wrap("Failed to read storage info", exc);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
async listImages(o = {}) {
|
|
791
|
+
const where = [];
|
|
792
|
+
const params = [];
|
|
793
|
+
const add = (sql2, v) => (params.push(v), where.push(sql2.replace("$?", `$${params.length}`)));
|
|
794
|
+
if (o.minSize !== void 0)
|
|
795
|
+
add(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= $?`, o.minSize);
|
|
796
|
+
if (o.olderThanDays !== void 0)
|
|
797
|
+
add(
|
|
798
|
+
`${this.dedup ? "r.created_at" : "created_at"} <= now() - make_interval(days => $?::int)`,
|
|
799
|
+
o.olderThanDays
|
|
800
|
+
);
|
|
801
|
+
if (o.onlyUntiered && !this.dedup)
|
|
802
|
+
where.push("storage_backend = 'postgres'");
|
|
803
|
+
const from = this.dedup ? "FROM zerobucket_image_refs r JOIN zerobucket_blobs b ON r.checksum_sha256 = b.checksum_sha256" : "FROM zerobucket_images";
|
|
804
|
+
const idCol = this.dedup ? "r.id" : "id";
|
|
805
|
+
const tierCol = this.dedup ? "false AS tiered" : "(storage_backend = 'object_storage') AS tiered";
|
|
806
|
+
const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
|
|
807
|
+
const lim = o.sample ?? o.limit;
|
|
808
|
+
let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
|
|
809
|
+
if (lim !== void 0) {
|
|
810
|
+
params.push(lim);
|
|
811
|
+
sql += ` LIMIT $${params.length}`;
|
|
812
|
+
}
|
|
813
|
+
try {
|
|
814
|
+
const rows = await this.run(
|
|
815
|
+
"list_images",
|
|
816
|
+
void 0,
|
|
817
|
+
async (q) => (await q.query(sql, params)).rows
|
|
818
|
+
);
|
|
819
|
+
return rows.map((r) => ({ id: String(r.id), tiered: Boolean(r.tiered) }));
|
|
820
|
+
} catch (exc) {
|
|
821
|
+
throw wrap("Failed to list images", exc);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
724
824
|
// ---- dedup deletes / migration ---------------------------------------
|
|
725
825
|
async deleteDedup(id, connection) {
|
|
726
826
|
try {
|
|
@@ -1002,6 +1102,7 @@ function isQueryable(c) {
|
|
|
1002
1102
|
|
|
1003
1103
|
// src/adapters/sqlite.ts
|
|
1004
1104
|
import { randomUUID } from "crypto";
|
|
1105
|
+
import { existsSync, statSync } from "fs";
|
|
1005
1106
|
var SCHEMA2 = `
|
|
1006
1107
|
CREATE TABLE IF NOT EXISTS zerobucket_images (
|
|
1007
1108
|
id TEXT PRIMARY KEY,
|
|
@@ -1588,6 +1689,96 @@ var SQLiteBackend = class {
|
|
|
1588
1689
|
throw wrap("Failed to check image existence", exc);
|
|
1589
1690
|
}
|
|
1590
1691
|
}
|
|
1692
|
+
// ---- maintenance (CLI) -----------------------------------------------
|
|
1693
|
+
async getInfo() {
|
|
1694
|
+
try {
|
|
1695
|
+
return await this.execute("info", void 0, "read", (db) => {
|
|
1696
|
+
const has = (t) => db.prepare(
|
|
1697
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?"
|
|
1698
|
+
).get(t) !== void 0;
|
|
1699
|
+
if (!has(this.dedup ? "zerobucket_image_refs" : "zerobucket_images") || this.dedup && !has("zerobucket_blobs"))
|
|
1700
|
+
return null;
|
|
1701
|
+
const j = this.dedup ? D_FROM2 : "FROM zerobucket_images";
|
|
1702
|
+
const size = this.dedup ? "b.size_bytes" : "size_bytes";
|
|
1703
|
+
const created = this.dedup ? "r.created_at" : "created_at";
|
|
1704
|
+
const mime = this.dedup ? "b.mime_type" : "mime_type";
|
|
1705
|
+
const tot = db.prepare(
|
|
1706
|
+
`SELECT count(*) AS n, coalesce(sum(${size}), 0) AS bytes, min(${created}) AS oldest, max(${created}) AS newest ${j}`
|
|
1707
|
+
).get();
|
|
1708
|
+
const fmt = db.prepare(
|
|
1709
|
+
`SELECT ${mime} AS m, count(*) AS n, coalesce(sum(${size}), 0) AS bytes ${j} GROUP BY ${mime} ORDER BY count(*) DESC, ${mime}`
|
|
1710
|
+
).all();
|
|
1711
|
+
let onDisk = null;
|
|
1712
|
+
if (this.opts.path !== ":memory:") {
|
|
1713
|
+
let total = statSync(this.opts.path).size;
|
|
1714
|
+
for (const suffix of ["-wal", "-shm"])
|
|
1715
|
+
if (existsSync(this.opts.path + suffix))
|
|
1716
|
+
total += statSync(this.opts.path + suffix).size;
|
|
1717
|
+
onDisk = prettySize(total);
|
|
1718
|
+
}
|
|
1719
|
+
const info = {
|
|
1720
|
+
mode: this.dedup ? "dedup" : "classic",
|
|
1721
|
+
count: tot.n,
|
|
1722
|
+
totalBytes: Number(tot.bytes),
|
|
1723
|
+
oldest: tot.oldest,
|
|
1724
|
+
newest: tot.newest,
|
|
1725
|
+
onDisk,
|
|
1726
|
+
byFormat: fmt.map((r) => ({
|
|
1727
|
+
mimeType: r.m,
|
|
1728
|
+
count: r.n,
|
|
1729
|
+
bytes: Number(r.bytes)
|
|
1730
|
+
}))
|
|
1731
|
+
};
|
|
1732
|
+
if (this.dedup) {
|
|
1733
|
+
const b = db.prepare(
|
|
1734
|
+
"SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_blobs"
|
|
1735
|
+
).get();
|
|
1736
|
+
info.dedup = { blobs: b.n, storedBytes: Number(b.bytes) };
|
|
1737
|
+
} else {
|
|
1738
|
+
const t = db.prepare(
|
|
1739
|
+
"SELECT count(*) AS n, coalesce(sum(size_bytes), 0) AS bytes FROM zerobucket_images WHERE storage_backend = 'object_storage'"
|
|
1740
|
+
).get();
|
|
1741
|
+
info.tiered = { count: t.n, bytes: Number(t.bytes) };
|
|
1742
|
+
}
|
|
1743
|
+
return info;
|
|
1744
|
+
});
|
|
1745
|
+
} catch (exc) {
|
|
1746
|
+
throw wrap("Failed to read storage info", exc);
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
async listImages(o = {}) {
|
|
1750
|
+
const where = [];
|
|
1751
|
+
const params = [];
|
|
1752
|
+
if (o.minSize !== void 0)
|
|
1753
|
+
where.push(`${this.dedup ? "b.size_bytes" : "size_bytes"} >= ?`), params.push(o.minSize);
|
|
1754
|
+
if (o.olderThanDays !== void 0) {
|
|
1755
|
+
const cutoff = new Date(Date.now() - o.olderThanDays * 864e5).toISOString().replace("Z", "000+00:00");
|
|
1756
|
+
where.push(`${this.dedup ? "r.created_at" : "created_at"} <= ?`);
|
|
1757
|
+
params.push(cutoff);
|
|
1758
|
+
}
|
|
1759
|
+
if (o.onlyUntiered && !this.dedup) where.push("storage_backend = 'sqlite'");
|
|
1760
|
+
const from = this.dedup ? D_FROM2 : "FROM zerobucket_images";
|
|
1761
|
+
const idCol = this.dedup ? "r.id" : "id";
|
|
1762
|
+
const tierCol = this.dedup ? "0 AS tiered" : "(storage_backend = 'object_storage') AS tiered";
|
|
1763
|
+
const order = o.sample !== void 0 ? "ORDER BY random()" : `ORDER BY ${this.dedup ? "r.created_at" : "created_at"}, ${idCol}`;
|
|
1764
|
+
const lim = o.sample ?? o.limit;
|
|
1765
|
+
let sql = `SELECT ${idCol} AS id, ${tierCol} ${from}${where.length ? " WHERE " + where.join(" AND ") : ""} ${order}`;
|
|
1766
|
+
if (lim !== void 0) sql += " LIMIT ?", params.push(lim);
|
|
1767
|
+
try {
|
|
1768
|
+
const rows = await this.execute(
|
|
1769
|
+
"list_images",
|
|
1770
|
+
void 0,
|
|
1771
|
+
"read",
|
|
1772
|
+
(db) => db.prepare(sql).all(...params)
|
|
1773
|
+
);
|
|
1774
|
+
return rows.map((r) => ({
|
|
1775
|
+
id: String(r.id),
|
|
1776
|
+
tiered: Boolean(r.tiered)
|
|
1777
|
+
}));
|
|
1778
|
+
} catch (exc) {
|
|
1779
|
+
throw wrap("Failed to list images", exc);
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1591
1782
|
/**
|
|
1592
1783
|
* Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
|
|
1593
1784
|
* reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the
|
|
@@ -1689,11 +1880,191 @@ var SQLiteBackend = class {
|
|
|
1689
1880
|
}
|
|
1690
1881
|
};
|
|
1691
1882
|
|
|
1883
|
+
// src/heic.ts
|
|
1884
|
+
import { createRequire } from "module";
|
|
1885
|
+
import { join } from "path";
|
|
1886
|
+
import { Worker } from "worker_threads";
|
|
1887
|
+
var HEIC_BRANDS = /* @__PURE__ */ new Set([
|
|
1888
|
+
"heic",
|
|
1889
|
+
"heix",
|
|
1890
|
+
"hevc",
|
|
1891
|
+
"heim",
|
|
1892
|
+
"heis",
|
|
1893
|
+
"mif1",
|
|
1894
|
+
"msf1"
|
|
1895
|
+
]);
|
|
1896
|
+
function looksLikeHeic(data) {
|
|
1897
|
+
if (data.length < 12) return false;
|
|
1898
|
+
if (String.fromCharCode(data[4], data[5], data[6], data[7]) !== "ftyp")
|
|
1899
|
+
return false;
|
|
1900
|
+
return HEIC_BRANDS.has(
|
|
1901
|
+
String.fromCharCode(data[8], data[9], data[10], data[11])
|
|
1902
|
+
);
|
|
1903
|
+
}
|
|
1904
|
+
var HEIC_INSTALL_HINT = "npm install libheif-js";
|
|
1905
|
+
var WORKER_SOURCE = `
|
|
1906
|
+
const { parentPort, workerData } = require("node:worker_threads");
|
|
1907
|
+
const libheif = require(workerData.libPath);
|
|
1908
|
+
// libheif-js reports decode problems by dumping a raw error object to the console. Capture that text instead:
|
|
1909
|
+
// it becomes part of the error we return (the useful diagnosis), and nothing is printed to the host's stderr.
|
|
1910
|
+
let diag = "";
|
|
1911
|
+
for (const level of ["log", "info", "warn", "error"]) {
|
|
1912
|
+
console[level] = (...args) => {
|
|
1913
|
+
diag = args.map((a) => (a && a.message) || (typeof a === "string" ? a : "")).filter(Boolean).join(" ").trim();
|
|
1914
|
+
};
|
|
1915
|
+
}
|
|
1916
|
+
const describe = (e) => {
|
|
1917
|
+
const base = String((e && e.message) || e);
|
|
1918
|
+
return diag && !base.includes(diag) ? base + ": " + diag : base;
|
|
1919
|
+
};
|
|
1920
|
+
function open(bytes) {
|
|
1921
|
+
const images = new libheif.HeifDecoder().decode(bytes);
|
|
1922
|
+
if (!images || images.length === 0) throw new Error("no images found in HEIF container");
|
|
1923
|
+
return images[0];
|
|
1924
|
+
}
|
|
1925
|
+
parentPort.on("message", async ({ id, op, data }) => {
|
|
1926
|
+
diag = "";
|
|
1927
|
+
try {
|
|
1928
|
+
const img = open(new Uint8Array(data));
|
|
1929
|
+
const width = img.get_width();
|
|
1930
|
+
const height = img.get_height();
|
|
1931
|
+
if (op === "info") return parentPort.postMessage({ id, ok: true, width, height });
|
|
1932
|
+
const out = await new Promise((resolve, reject) =>
|
|
1933
|
+
img.display({ data: new Uint8ClampedArray(width * height * 4), width, height }, (d) =>
|
|
1934
|
+
d ? resolve(d) : reject(new Error("HEIF processing error")),
|
|
1935
|
+
),
|
|
1936
|
+
);
|
|
1937
|
+
if (op === "validate") return parentPort.postMessage({ id, ok: true, width, height });
|
|
1938
|
+
const rgba = out.data.buffer;
|
|
1939
|
+
parentPort.postMessage({ id, ok: true, width, height, rgba }, [rgba]);
|
|
1940
|
+
} catch (e) {
|
|
1941
|
+
parentPort.postMessage({ id, ok: false, error: describe(e) });
|
|
1942
|
+
}
|
|
1943
|
+
});
|
|
1944
|
+
`;
|
|
1945
|
+
var HeicWorkerError = class extends StorageError {
|
|
1946
|
+
};
|
|
1947
|
+
var worker;
|
|
1948
|
+
var nextId = 1;
|
|
1949
|
+
var pending = /* @__PURE__ */ new Map();
|
|
1950
|
+
var stats = { info: 0, validate: 0, decode: 0, workersStarted: 0 };
|
|
1951
|
+
var resolveLib = () => {
|
|
1952
|
+
const attempts = [
|
|
1953
|
+
() => createRequire(import.meta.url).resolve("libheif-js/wasm-bundle"),
|
|
1954
|
+
// next to / above this package
|
|
1955
|
+
() => createRequire(join(process.cwd(), "noop.js")).resolve(
|
|
1956
|
+
"libheif-js/wasm-bundle"
|
|
1957
|
+
)
|
|
1958
|
+
// the app's own node_modules
|
|
1959
|
+
];
|
|
1960
|
+
let last;
|
|
1961
|
+
for (const attempt of attempts) {
|
|
1962
|
+
try {
|
|
1963
|
+
return attempt();
|
|
1964
|
+
} catch (e) {
|
|
1965
|
+
last = e;
|
|
1966
|
+
}
|
|
1967
|
+
}
|
|
1968
|
+
throw last;
|
|
1969
|
+
};
|
|
1970
|
+
function heicAvailable() {
|
|
1971
|
+
try {
|
|
1972
|
+
resolveLib();
|
|
1973
|
+
return true;
|
|
1974
|
+
} catch {
|
|
1975
|
+
return false;
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
function releaseIfIdle(w) {
|
|
1979
|
+
for (const p of pending.values()) if (p.owner === w) return;
|
|
1980
|
+
w.unref();
|
|
1981
|
+
}
|
|
1982
|
+
function startWorker() {
|
|
1983
|
+
let libPath;
|
|
1984
|
+
try {
|
|
1985
|
+
libPath = resolveLib();
|
|
1986
|
+
} catch (cause) {
|
|
1987
|
+
throw new StorageError(
|
|
1988
|
+
`HEIC support requires the optional dependency 'libheif-js'. Install it with: ${HEIC_INSTALL_HINT}`,
|
|
1989
|
+
{ cause }
|
|
1990
|
+
);
|
|
1991
|
+
}
|
|
1992
|
+
const w = new Worker(WORKER_SOURCE, { eval: true, workerData: { libPath } });
|
|
1993
|
+
stats.workersStarted += 1;
|
|
1994
|
+
w.on(
|
|
1995
|
+
"message",
|
|
1996
|
+
(m) => {
|
|
1997
|
+
const p = pending.get(m.id);
|
|
1998
|
+
if (!p) return;
|
|
1999
|
+
pending.delete(m.id);
|
|
2000
|
+
releaseIfIdle(w);
|
|
2001
|
+
if (m.ok) p.resolve(m);
|
|
2002
|
+
else p.reject(new Error(m.error));
|
|
2003
|
+
}
|
|
2004
|
+
);
|
|
2005
|
+
const fail = (err) => {
|
|
2006
|
+
if (worker === w) worker = void 0;
|
|
2007
|
+
for (const [id, p] of pending) {
|
|
2008
|
+
if (p.owner !== w) continue;
|
|
2009
|
+
pending.delete(id);
|
|
2010
|
+
p.reject(err);
|
|
2011
|
+
}
|
|
2012
|
+
};
|
|
2013
|
+
w.on(
|
|
2014
|
+
"error",
|
|
2015
|
+
(e) => fail(
|
|
2016
|
+
new HeicWorkerError(
|
|
2017
|
+
`HEIC decoder worker crashed: ${e instanceof Error ? e.message : String(e)}`,
|
|
2018
|
+
{ cause: e }
|
|
2019
|
+
)
|
|
2020
|
+
)
|
|
2021
|
+
);
|
|
2022
|
+
w.on(
|
|
2023
|
+
"exit",
|
|
2024
|
+
(code) => fail(
|
|
2025
|
+
new HeicWorkerError(
|
|
2026
|
+
`HEIC decoder worker exited unexpectedly (code ${code})`
|
|
2027
|
+
)
|
|
2028
|
+
)
|
|
2029
|
+
);
|
|
2030
|
+
w.unref();
|
|
2031
|
+
return w;
|
|
2032
|
+
}
|
|
2033
|
+
function call(op, data) {
|
|
2034
|
+
stats[op] += 1;
|
|
2035
|
+
const w = worker ??= startWorker();
|
|
2036
|
+
const id = nextId++;
|
|
2037
|
+
const copy = new Uint8Array(data).buffer;
|
|
2038
|
+
return new Promise((resolve, reject) => {
|
|
2039
|
+
pending.set(id, { owner: w, resolve, reject });
|
|
2040
|
+
w.ref();
|
|
2041
|
+
w.postMessage({ id, op, data: copy }, [copy]);
|
|
2042
|
+
});
|
|
2043
|
+
}
|
|
2044
|
+
async function heicInfo(data) {
|
|
2045
|
+
const r = await call("info", data);
|
|
2046
|
+
return { width: r.width, height: r.height };
|
|
2047
|
+
}
|
|
2048
|
+
async function heicValidate(data) {
|
|
2049
|
+
const r = await call("validate", data);
|
|
2050
|
+
return { width: r.width, height: r.height };
|
|
2051
|
+
}
|
|
2052
|
+
async function decodeHeic(data) {
|
|
2053
|
+
const r = await call("decode", data);
|
|
2054
|
+
return {
|
|
2055
|
+
width: r.width,
|
|
2056
|
+
height: r.height,
|
|
2057
|
+
rgba: Buffer.from(r.rgba)
|
|
2058
|
+
};
|
|
2059
|
+
}
|
|
2060
|
+
|
|
1692
2061
|
// src/validation.ts
|
|
1693
2062
|
var FORMAT_TO_MIME = {
|
|
1694
2063
|
jpeg: "image/jpeg",
|
|
1695
2064
|
png: "image/png",
|
|
1696
|
-
webp: "image/webp"
|
|
2065
|
+
webp: "image/webp",
|
|
2066
|
+
heic: "image/heic"
|
|
2067
|
+
// optional: needs `npm install libheif-js` (see heic.ts)
|
|
1697
2068
|
};
|
|
1698
2069
|
var SUPPORTED_FORMATS = Object.keys(FORMAT_TO_MIME);
|
|
1699
2070
|
var DEFAULT_MAX_PIXELS = 89e6;
|
|
@@ -1721,6 +2092,8 @@ async function validateImage(data, opts) {
|
|
|
1721
2092
|
const sizeBytes = data.length;
|
|
1722
2093
|
if (sizeBytes > maxBytes) throw new ImageTooLargeError(sizeBytes, maxBytes);
|
|
1723
2094
|
if (sizeBytes === 0) throw new CorruptedImageError("Image data is empty");
|
|
2095
|
+
if (looksLikeHeic(data))
|
|
2096
|
+
return validateHeic(data, { maxBytes, maxPixels, allowedFormats });
|
|
1724
2097
|
const sharp = await loadSharp();
|
|
1725
2098
|
const make = () => sharp(data, { limitInputPixels: maxPixels, failOn: "error" });
|
|
1726
2099
|
let format;
|
|
@@ -1756,6 +2129,45 @@ async function validateImage(data, opts) {
|
|
|
1756
2129
|
}
|
|
1757
2130
|
return { mimeType: FORMAT_TO_MIME[format], width, height, sizeBytes };
|
|
1758
2131
|
}
|
|
2132
|
+
async function validateHeic(data, {
|
|
2133
|
+
maxBytes,
|
|
2134
|
+
maxPixels,
|
|
2135
|
+
allowedFormats
|
|
2136
|
+
}) {
|
|
2137
|
+
const sizeBytes = data.length;
|
|
2138
|
+
if (!heicAvailable()) {
|
|
2139
|
+
throw new ImageValidationError(
|
|
2140
|
+
`This looks like a HEIC/HEIF image, which requires an optional dependency. Install it with: ${HEIC_INSTALL_HINT}`
|
|
2141
|
+
);
|
|
2142
|
+
}
|
|
2143
|
+
if (!allowedFormats.includes("heic"))
|
|
2144
|
+
throw new UnsupportedFormatError("heic", allowedFormats);
|
|
2145
|
+
let width;
|
|
2146
|
+
let height;
|
|
2147
|
+
try {
|
|
2148
|
+
({ width, height } = await heicInfo(data));
|
|
2149
|
+
} catch (exc) {
|
|
2150
|
+
if (exc instanceof StorageError) throw exc;
|
|
2151
|
+
throw new CorruptedImageError(
|
|
2152
|
+
`Could not decode image: ${errMessage(exc)}`,
|
|
2153
|
+
{ cause: exc }
|
|
2154
|
+
);
|
|
2155
|
+
}
|
|
2156
|
+
if (!width || !height)
|
|
2157
|
+
throw new CorruptedImageError("Image has no readable dimensions");
|
|
2158
|
+
if (width * height > maxPixels)
|
|
2159
|
+
throw new ImageTooLargeError(sizeBytes, maxBytes);
|
|
2160
|
+
try {
|
|
2161
|
+
await heicValidate(data);
|
|
2162
|
+
} catch (exc) {
|
|
2163
|
+
if (exc instanceof StorageError) throw exc;
|
|
2164
|
+
throw new CorruptedImageError(
|
|
2165
|
+
`Image data is truncated or corrupted: ${errMessage(exc)}`,
|
|
2166
|
+
{ cause: exc }
|
|
2167
|
+
);
|
|
2168
|
+
}
|
|
2169
|
+
return { mimeType: FORMAT_TO_MIME.heic, width, height, sizeBytes };
|
|
2170
|
+
}
|
|
1759
2171
|
function isPixelLimit(exc) {
|
|
1760
2172
|
return /pixel limit/i.test(errMessage(exc));
|
|
1761
2173
|
}
|
|
@@ -1764,28 +2176,53 @@ function errMessage(exc) {
|
|
|
1764
2176
|
}
|
|
1765
2177
|
|
|
1766
2178
|
// src/optimize.ts
|
|
2179
|
+
var ENCODABLE_FORMATS = ["jpeg", "png", "webp"];
|
|
1767
2180
|
var DEFAULT_JPEG_QUALITY = 90;
|
|
1768
2181
|
var DEFAULT_WEBP_QUALITY = 88;
|
|
1769
|
-
var FORMAT_ALIASES = { jpg: "jpeg" };
|
|
1770
2182
|
async function optimizeImage(data, opts) {
|
|
1771
2183
|
const { maxWidth, quality, maxBytes, maxPixels = DEFAULT_MAX_PIXELS } = opts;
|
|
1772
2184
|
const sharp = await loadSharp();
|
|
1773
2185
|
const originalSizeBytes = data.length;
|
|
1774
|
-
const
|
|
1775
|
-
const
|
|
1776
|
-
|
|
1777
|
-
const target = FORMAT_ALIASES[requested] ?? requested;
|
|
1778
|
-
if (!SUPPORTED_FORMATS.includes(target)) {
|
|
2186
|
+
const requestedRaw = opts.format?.toLowerCase();
|
|
2187
|
+
const requested = requestedRaw === "jpg" ? "jpeg" : requestedRaw;
|
|
2188
|
+
if (requested === "heic" || requested === "heif") {
|
|
1779
2189
|
throw new ImageValidationError(
|
|
1780
|
-
|
|
1781
|
-
[...SUPPORTED_FORMATS].sort()
|
|
1782
|
-
)}`
|
|
2190
|
+
"Encoding to HEIC is not available in the Node package (there is no HEVC encoder for Node). Optimize to jpeg, webp or png instead, or store the original HEIC without `optimize`."
|
|
1783
2191
|
);
|
|
1784
2192
|
}
|
|
1785
|
-
let
|
|
2193
|
+
let input = data;
|
|
2194
|
+
let inputOptions = {
|
|
1786
2195
|
limitInputPixels: maxPixels,
|
|
1787
2196
|
failOn: "error"
|
|
1788
|
-
}
|
|
2197
|
+
};
|
|
2198
|
+
let sourceFormat;
|
|
2199
|
+
if (looksLikeHeic(data)) {
|
|
2200
|
+
if (!heicAvailable()) {
|
|
2201
|
+
throw new ImageValidationError(
|
|
2202
|
+
`This looks like a HEIC/HEIF image, which requires an optional dependency. Install it with: ${HEIC_INSTALL_HINT}`
|
|
2203
|
+
);
|
|
2204
|
+
}
|
|
2205
|
+
if (!requested) {
|
|
2206
|
+
throw new ImageValidationError(
|
|
2207
|
+
"Optimizing a HEIC image needs an explicit output format, because re-encoding to HEIC is not available in the Node package. Pass optimize: { format: 'jpeg' | 'webp' | 'png' } (for example { format: 'webp', maxWidth: 1600 })."
|
|
2208
|
+
);
|
|
2209
|
+
}
|
|
2210
|
+
const decoded = await decodeHeic(data);
|
|
2211
|
+
input = decoded.rgba;
|
|
2212
|
+
inputOptions = {
|
|
2213
|
+
raw: { width: decoded.width, height: decoded.height, channels: 4 }
|
|
2214
|
+
};
|
|
2215
|
+
sourceFormat = "heic";
|
|
2216
|
+
} else {
|
|
2217
|
+
sourceFormat = (await sharp(data, inputOptions).metadata()).format;
|
|
2218
|
+
}
|
|
2219
|
+
const target = requested ?? sourceFormat ?? "jpeg";
|
|
2220
|
+
if (!ENCODABLE_FORMATS.includes(target)) {
|
|
2221
|
+
throw new ImageValidationError(
|
|
2222
|
+
`Cannot re-encode to unsupported format ${JSON.stringify(target)}. Supported: ${JSON.stringify([...ENCODABLE_FORMATS].sort())}`
|
|
2223
|
+
);
|
|
2224
|
+
}
|
|
2225
|
+
let pipeline = sharp(input, inputOptions).rotate();
|
|
1789
2226
|
if (maxWidth !== void 0) {
|
|
1790
2227
|
pipeline = pipeline.resize({
|
|
1791
2228
|
width: maxWidth,
|