zerobucket 0.1.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -20
- package/dist/bin.js +2506 -0
- package/dist/{client-C_W0t8My.d.cts → client-C7xs1dlr.d.cts} +159 -5
- package/dist/{client-C_W0t8My.d.ts → client-C7xs1dlr.d.ts} +159 -5
- package/dist/http.d.cts +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.cjs +1639 -134
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +203 -56
- package/dist/index.d.ts +203 -56
- package/dist/index.js +1635 -132
- package/dist/index.js.map +1 -1
- package/package.json +26 -4
package/dist/index.d.cts
CHANGED
|
@@ -1,7 +1,64 @@
|
|
|
1
|
-
import { S as StorageBackend, O as OperationEvent, P as PreparedRow, C as CallOptions,
|
|
2
|
-
export { B as BatchDeleteResult,
|
|
1
|
+
import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, g as ContentValidator, V as ValidatedContent } from './client-C7xs1dlr.cjs';
|
|
2
|
+
export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as ObjectStorage, q as ObjectStorageOptions, r as OptimizationResult, s as OptimizeOptions, t as PutManyOptions, u as PutOptions, Q as Queryable, v as SqliteDatabaseLike, w as StreamOptions, W as WritableLike, Z as ZeroBucket, x as ZeroBucketOptions, y as optimizeImage, z as readInput } from './client-C7xs1dlr.cjs';
|
|
3
3
|
import { Pool } from 'pg';
|
|
4
4
|
|
|
5
|
+
/**
|
|
6
|
+
* Error hierarchy for ZeroBucket. Mirrors the Python package's exceptions
|
|
7
|
+
* one-to-one so behaviour is portable across languages.
|
|
8
|
+
*
|
|
9
|
+
* Catch broadly with `instanceof ZeroBucketError`, or narrowly with a
|
|
10
|
+
* subclass such as `ImageNotFoundError`.
|
|
11
|
+
*/
|
|
12
|
+
declare class ZeroBucketError extends Error {
|
|
13
|
+
constructor(message: string, options?: {
|
|
14
|
+
cause?: unknown;
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
/** Any content (image or custom-validated) failed validation. */
|
|
18
|
+
declare class ContentValidationError extends ZeroBucketError {
|
|
19
|
+
}
|
|
20
|
+
/** An image failed validation (bad format, too large, corrupted, ...). */
|
|
21
|
+
declare class ImageValidationError extends ContentValidationError {
|
|
22
|
+
}
|
|
23
|
+
declare class ImageTooLargeError extends ImageValidationError {
|
|
24
|
+
readonly sizeBytes: number;
|
|
25
|
+
readonly maxBytes: number;
|
|
26
|
+
constructor(sizeBytes: number, maxBytes: number);
|
|
27
|
+
}
|
|
28
|
+
declare class UnsupportedFormatError extends ImageValidationError {
|
|
29
|
+
readonly detectedFormat: string | null;
|
|
30
|
+
readonly allowed: readonly string[];
|
|
31
|
+
constructor(detectedFormat: string | null, allowed: readonly string[]);
|
|
32
|
+
}
|
|
33
|
+
/** Image bytes could not be decoded despite a recognizable header. */
|
|
34
|
+
declare class CorruptedImageError extends ImageValidationError {
|
|
35
|
+
}
|
|
36
|
+
/** `get()`, `getStream()` or `metadata()` was called with an id that does not exist. */
|
|
37
|
+
declare class ImageNotFoundError extends ZeroBucketError {
|
|
38
|
+
readonly imageId: string;
|
|
39
|
+
constructor(imageId: string);
|
|
40
|
+
}
|
|
41
|
+
/** Underlying storage / database failure not covered above. */
|
|
42
|
+
declare class StorageError extends ZeroBucketError {
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* A `beforeGet` / `beforePut` hook denied the operation by returning false.
|
|
46
|
+
*
|
|
47
|
+
* If the hook itself throws, that exception propagates unchanged: it is NOT
|
|
48
|
+
* wrapped in AccessDeniedError and NOT treated as an implicit allow. A broken
|
|
49
|
+
* authorization check must fail closed, never open.
|
|
50
|
+
*/
|
|
51
|
+
declare class AccessDeniedError extends ZeroBucketError {
|
|
52
|
+
readonly operation: string;
|
|
53
|
+
readonly imageId?: string | undefined;
|
|
54
|
+
constructor(operation: string, imageId?: string | undefined);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Helpers shared by every storage adapter (kept here so Postgres and SQLite cannot drift apart). */
|
|
58
|
+
|
|
59
|
+
/** 1 MiB. Default chunk size for getStream(). */
|
|
60
|
+
declare const DEFAULT_STREAM_CHUNK_SIZE: number;
|
|
61
|
+
|
|
5
62
|
/**
|
|
6
63
|
* PostgreSQL storage adapter (classic mode).
|
|
7
64
|
*
|
|
@@ -20,8 +77,6 @@ import { Pool } from 'pg';
|
|
|
20
77
|
* race on CREATE TABLE.
|
|
21
78
|
*/
|
|
22
79
|
|
|
23
|
-
/** 1 MiB. Default chunk size for getStream(). */
|
|
24
|
-
declare const DEFAULT_STREAM_CHUNK_SIZE: number;
|
|
25
80
|
interface PostgresBackendOptions {
|
|
26
81
|
/** postgresql:// connection string. Required unless `pool` is given. */
|
|
27
82
|
connectionString?: string;
|
|
@@ -38,6 +93,18 @@ interface PostgresBackendOptions {
|
|
|
38
93
|
/** Max wait for a pooled connection. Default 10000. */
|
|
39
94
|
poolTimeoutMs?: number;
|
|
40
95
|
onOperation?: (event: OperationEvent) => void;
|
|
96
|
+
/**
|
|
97
|
+
* Enables transparent reads of tiered rows and `tierToObjectStorage()`.
|
|
98
|
+
* Configure it with the same bucket/credentials used to tier the images.
|
|
99
|
+
*/
|
|
100
|
+
objectStorage?: ObjectStorageLike;
|
|
101
|
+
/**
|
|
102
|
+
* Content-addressed mode: identical bytes are stored once (zerobucket_blobs) and each
|
|
103
|
+
* put() adds a lightweight reference (zerobucket_image_refs) with its own id and filename.
|
|
104
|
+
* Separate tables from classic mode, so the two can never be confused. Cannot be combined
|
|
105
|
+
* with `objectStorage`. Same schema as the Python package's dedup mode.
|
|
106
|
+
*/
|
|
107
|
+
dedup?: boolean;
|
|
41
108
|
}
|
|
42
109
|
declare class PostgresBackend implements StorageBackend {
|
|
43
110
|
private readonly opts;
|
|
@@ -48,6 +115,8 @@ declare class PostgresBackend implements StorageBackend {
|
|
|
48
115
|
private readonly maxRetries;
|
|
49
116
|
private readonly retryBaseDelayMs;
|
|
50
117
|
private readonly autoMigrate;
|
|
118
|
+
private readonly dedup;
|
|
119
|
+
private readonly sql;
|
|
51
120
|
constructor(opts: PostgresBackendOptions);
|
|
52
121
|
ready(): Promise<void>;
|
|
53
122
|
private init;
|
|
@@ -63,6 +132,14 @@ declare class PostgresBackend implements StorageBackend {
|
|
|
63
132
|
* failures retry the WHOLE closure on a fresh connection.
|
|
64
133
|
*/
|
|
65
134
|
private run;
|
|
135
|
+
/**
|
|
136
|
+
* Make multi-statement work all-or-nothing on a CALLER-supplied connection.
|
|
137
|
+
* - Caller has an open transaction -> SAVEPOINT: a failure rolls back only our work and the
|
|
138
|
+
* caller's transaction stays usable; success joins their transaction.
|
|
139
|
+
* - No open transaction (an autocommit pg.Client) -> open and commit our own, so a failure
|
|
140
|
+
* can never leave half-applied state (e.g. an inflated dedup ref_count).
|
|
141
|
+
*/
|
|
142
|
+
private atomic;
|
|
66
143
|
put(row: PreparedRow, { connection }?: CallOptions): Promise<string>;
|
|
67
144
|
putMany(rows: PreparedRow[], { connection }?: CallOptions): Promise<string[]>;
|
|
68
145
|
get(id: string, { connection }?: CallOptions): Promise<StoredRecord | null>;
|
|
@@ -72,60 +149,130 @@ declare class PostgresBackend implements StorageBackend {
|
|
|
72
149
|
delete(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
73
150
|
deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
|
|
74
151
|
exists(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
75
|
-
|
|
76
|
-
|
|
152
|
+
getInfo(): Promise<StorageInfo | null>;
|
|
153
|
+
listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
|
|
154
|
+
private deleteDedup;
|
|
155
|
+
private deleteManyDedup;
|
|
156
|
+
/**
|
|
157
|
+
* One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
|
|
158
|
+
* dedup instance's tables, preserving every id (external references keep working) and
|
|
159
|
+
* deduplicating identical content on the way. The classic table is never modified.
|
|
160
|
+
*
|
|
161
|
+
* Differences from the Python version, all deliberate: rows are copied in small batches (bounded
|
|
162
|
+
* memory instead of loading the whole table); re-running is safe (already-migrated ids are
|
|
163
|
+
* skipped and reported, not an error); and classic rows already tiered to object storage are
|
|
164
|
+
* refused up front with a clear message (their bytes are not in the database to copy).
|
|
165
|
+
* Still all-or-nothing: one transaction.
|
|
166
|
+
*/
|
|
167
|
+
migrateClassicToDedup(): Promise<{
|
|
168
|
+
imagesMigrated: number;
|
|
169
|
+
distinctBlobsCreated: number;
|
|
170
|
+
duplicateReferencesFound: number;
|
|
171
|
+
alreadyMigrated: number;
|
|
172
|
+
}>;
|
|
173
|
+
private hydrate;
|
|
174
|
+
/**
|
|
175
|
+
* Remove tiered objects AFTER their rows are gone. Ordering is deliberate:
|
|
176
|
+
* once the row is deleted the image is correctly "not found" to everyone, so
|
|
177
|
+
* a failed object delete only leaves a harmless orphan, whereas deleting the
|
|
178
|
+
* object first and then failing the row delete would leave a row pointing at nothing.
|
|
179
|
+
*
|
|
180
|
+
* Never runs inside a caller-owned transaction (`connection`): we cannot know
|
|
181
|
+
* whether it will commit, and destroying data before the commit outcome is known
|
|
182
|
+
* could leave a rolled-back row pointing at a deleted object. In that case the
|
|
183
|
+
* object stays until you remove it yourself after committing (objectStorage.delete(id)).
|
|
184
|
+
* Failures are reported through onOperation and never fail the delete itself.
|
|
185
|
+
*/
|
|
186
|
+
private cleanupObject;
|
|
187
|
+
/**
|
|
188
|
+
* Move an image's bytes out of Postgres into object storage, replacing the
|
|
189
|
+
* row's `data` with a pointer. Returns:
|
|
190
|
+
* null - no such image
|
|
191
|
+
* false - already tiered (a safe no-op, so backfills can be re-run)
|
|
192
|
+
* true - tiered just now
|
|
193
|
+
*
|
|
194
|
+
* SAFETY: the upload happens INSIDE the same transaction as the row lock
|
|
195
|
+
* (SELECT ... FOR UPDATE) and the UPDATE that flips the row. If the upload
|
|
196
|
+
* fails, the transaction rolls back and the row is untouched: there is no
|
|
197
|
+
* moment when the bytes exist nowhere, or when a row claims to be tiered
|
|
198
|
+
* without a completed upload.
|
|
199
|
+
*
|
|
200
|
+
* TRADEOFF: the row lock is held for the whole upload. Concurrent writes
|
|
201
|
+
* to THIS image (delete, tier) wait; every other row is unaffected, and plain
|
|
202
|
+
* reads are never blocked (Postgres MVCC). Meant for explicit maintenance, not hot paths.
|
|
203
|
+
*
|
|
204
|
+
* Retry is safe: a transient error replays the whole closure, and the
|
|
205
|
+
* deterministic key (the image id) makes the re-upload an idempotent overwrite.
|
|
206
|
+
*/
|
|
207
|
+
tierToObjectStorage(id: string, { connection }?: CallOptions): Promise<boolean | null>;
|
|
77
208
|
}
|
|
78
209
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
/** Any content (image or custom-validated) failed validation. */
|
|
92
|
-
declare class ContentValidationError extends ZeroBucketError {
|
|
93
|
-
}
|
|
94
|
-
/** An image failed validation (bad format, too large, corrupted, ...). */
|
|
95
|
-
declare class ImageValidationError extends ContentValidationError {
|
|
96
|
-
}
|
|
97
|
-
declare class ImageTooLargeError extends ImageValidationError {
|
|
98
|
-
readonly sizeBytes: number;
|
|
99
|
-
readonly maxBytes: number;
|
|
100
|
-
constructor(sizeBytes: number, maxBytes: number);
|
|
101
|
-
}
|
|
102
|
-
declare class UnsupportedFormatError extends ImageValidationError {
|
|
103
|
-
readonly detectedFormat: string | null;
|
|
104
|
-
readonly allowed: readonly string[];
|
|
105
|
-
constructor(detectedFormat: string | null, allowed: readonly string[]);
|
|
106
|
-
}
|
|
107
|
-
/** Image bytes could not be decoded despite a recognizable header. */
|
|
108
|
-
declare class CorruptedImageError extends ImageValidationError {
|
|
109
|
-
}
|
|
110
|
-
/** `get()`, `getStream()` or `metadata()` was called with an id that does not exist. */
|
|
111
|
-
declare class ImageNotFoundError extends ZeroBucketError {
|
|
112
|
-
readonly imageId: string;
|
|
113
|
-
constructor(imageId: string);
|
|
114
|
-
}
|
|
115
|
-
/** Underlying storage / database failure not covered above. */
|
|
116
|
-
declare class StorageError extends ZeroBucketError {
|
|
210
|
+
interface SQLiteBackendOptions {
|
|
211
|
+
/** Database file path, or ":memory:" for a throwaway in-process database (tiering is unavailable for ":memory:"). */
|
|
212
|
+
path: string;
|
|
213
|
+
/** Create the schema on first use. Default true. */
|
|
214
|
+
autoMigrate?: boolean;
|
|
215
|
+
/** Content-addressed mode (separate tables; same schema as the Python package). Cannot be combined with objectStorage. */
|
|
216
|
+
dedup?: boolean;
|
|
217
|
+
/** Enables tiering and transparent reads of tiered rows. */
|
|
218
|
+
objectStorage?: ObjectStorageLike;
|
|
219
|
+
/** Total time to keep retrying when another connection/process holds the write lock. Default 10000. */
|
|
220
|
+
busyTimeoutMs?: number;
|
|
221
|
+
onOperation?: (event: OperationEvent) => void;
|
|
117
222
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
readonly
|
|
127
|
-
|
|
128
|
-
|
|
223
|
+
declare class SQLiteBackend implements StorageBackend {
|
|
224
|
+
private readonly opts;
|
|
225
|
+
private db;
|
|
226
|
+
private Database;
|
|
227
|
+
private readyPromise;
|
|
228
|
+
private closed;
|
|
229
|
+
private readonly dedup;
|
|
230
|
+
private readonly busyTimeoutMs;
|
|
231
|
+
private readonly writeLock;
|
|
232
|
+
constructor(opts: SQLiteBackendOptions);
|
|
233
|
+
ready(): Promise<void>;
|
|
234
|
+
private init;
|
|
235
|
+
close(): Promise<void>;
|
|
236
|
+
private emit;
|
|
237
|
+
/** Re-run `fn` while another connection holds a lock, sleeping asynchronously between attempts. `fn` must be atomic. */
|
|
238
|
+
private retrying;
|
|
239
|
+
/** BEGIN IMMEDIATE takes the write lock up front, so a BUSY comes before any work and the whole unit is safely re-runnable. */
|
|
240
|
+
private tx;
|
|
241
|
+
/**
|
|
242
|
+
* All-or-nothing work on a CALLER-supplied handle. SAVEPOINT works whether or not the caller has an open
|
|
243
|
+
* transaction: inside one it nests (a failure rolls back only our work); outside one it begins and, on
|
|
244
|
+
* RELEASE, commits.
|
|
245
|
+
*/
|
|
246
|
+
private atomic;
|
|
247
|
+
private execute;
|
|
248
|
+
put(row: PreparedRow, { connection }?: CallOptions): Promise<string>;
|
|
249
|
+
private insertOne;
|
|
250
|
+
putMany(rows: PreparedRow[], { connection }?: CallOptions): Promise<string[]>;
|
|
251
|
+
get(id: string, { connection }?: CallOptions): Promise<StoredRecord | null>;
|
|
252
|
+
getMany(ids: string[], { connection }?: CallOptions): Promise<StoredRecord[]>;
|
|
253
|
+
getMetadata(id: string, { connection }?: CallOptions): Promise<StoredRecordMetadata | null>;
|
|
254
|
+
getStream(id: string, opts: StreamOptions): Promise<AsyncIterable<Buffer> | null>;
|
|
255
|
+
delete(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
256
|
+
deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
|
|
257
|
+
exists(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
258
|
+
getInfo(): Promise<StorageInfo | null>;
|
|
259
|
+
listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
|
|
260
|
+
/**
|
|
261
|
+
* Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
|
|
262
|
+
* reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the
|
|
263
|
+
* commit outcome is unknown. Failures never fail the delete; they are reported through onOperation.
|
|
264
|
+
*/
|
|
265
|
+
private cleanupObjects;
|
|
266
|
+
/**
|
|
267
|
+
* Move an image's bytes into object storage. null = no such image, false = already tiered, true = tiered now.
|
|
268
|
+
*
|
|
269
|
+
* SAFETY (same guarantee as Postgres): the upload happens INSIDE a BEGIN IMMEDIATE transaction; if it fails the
|
|
270
|
+
* transaction rolls back and the row is untouched.
|
|
271
|
+
*
|
|
272
|
+
* TRADEOFF (coarser than Postgres): SQLite has no row locks, so BEGIN IMMEDIATE locks the WHOLE DATABASE for the
|
|
273
|
+
* upload. Other processes' writes wait; this process's writes queue behind an async lock; reads are not blocked.
|
|
274
|
+
*/
|
|
275
|
+
tierToObjectStorage(id: string, { connection }?: CallOptions): Promise<boolean | null>;
|
|
129
276
|
}
|
|
130
277
|
|
|
131
278
|
declare const SUPPORTED_FORMATS: readonly string[];
|
|
@@ -161,4 +308,4 @@ declare class PDFValidator implements ContentValidator {
|
|
|
161
308
|
}): ValidatedContent;
|
|
162
309
|
}
|
|
163
310
|
|
|
164
|
-
export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };
|
|
311
|
+
export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,64 @@
|
|
|
1
|
-
import { S as StorageBackend, O as OperationEvent, P as PreparedRow, C as CallOptions,
|
|
2
|
-
export { B as BatchDeleteResult,
|
|
1
|
+
import { S as StorageBackend, O as OperationEvent, a as ObjectStorageLike, P as PreparedRow, C as CallOptions, b as StoredRecord, c as StoredRecordMetadata, d as StreamOptions, e as StorageInfo, I as ImageListOptions, f as ImageListEntry, g as ContentValidator, V as ValidatedContent } from './client-C7xs1dlr.js';
|
|
2
|
+
export { B as BatchDeleteResult, h as BatchGetResult, i as BatchPutResult, j as BeforeGetHook, k as BeforePutHook, l as Connection, D as DEFAULT_MAX_BYTES, G as GetOptions, m as Image, n as ImageInput, o as ImageMetadata, p as ObjectStorage, q as ObjectStorageOptions, r as OptimizationResult, s as OptimizeOptions, t as PutManyOptions, u as PutOptions, Q as Queryable, v as SqliteDatabaseLike, w as StreamOptions, W as WritableLike, Z as ZeroBucket, x as ZeroBucketOptions, y as optimizeImage, z as readInput } from './client-C7xs1dlr.js';
|
|
3
3
|
import { Pool } from 'pg';
|
|
4
4
|
|
|
5
|
+
/**
|
|
6
|
+
* Error hierarchy for ZeroBucket. Mirrors the Python package's exceptions
|
|
7
|
+
* one-to-one so behaviour is portable across languages.
|
|
8
|
+
*
|
|
9
|
+
* Catch broadly with `instanceof ZeroBucketError`, or narrowly with a
|
|
10
|
+
* subclass such as `ImageNotFoundError`.
|
|
11
|
+
*/
|
|
12
|
+
declare class ZeroBucketError extends Error {
|
|
13
|
+
constructor(message: string, options?: {
|
|
14
|
+
cause?: unknown;
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
/** Any content (image or custom-validated) failed validation. */
|
|
18
|
+
declare class ContentValidationError extends ZeroBucketError {
|
|
19
|
+
}
|
|
20
|
+
/** An image failed validation (bad format, too large, corrupted, ...). */
|
|
21
|
+
declare class ImageValidationError extends ContentValidationError {
|
|
22
|
+
}
|
|
23
|
+
declare class ImageTooLargeError extends ImageValidationError {
|
|
24
|
+
readonly sizeBytes: number;
|
|
25
|
+
readonly maxBytes: number;
|
|
26
|
+
constructor(sizeBytes: number, maxBytes: number);
|
|
27
|
+
}
|
|
28
|
+
declare class UnsupportedFormatError extends ImageValidationError {
|
|
29
|
+
readonly detectedFormat: string | null;
|
|
30
|
+
readonly allowed: readonly string[];
|
|
31
|
+
constructor(detectedFormat: string | null, allowed: readonly string[]);
|
|
32
|
+
}
|
|
33
|
+
/** Image bytes could not be decoded despite a recognizable header. */
|
|
34
|
+
declare class CorruptedImageError extends ImageValidationError {
|
|
35
|
+
}
|
|
36
|
+
/** `get()`, `getStream()` or `metadata()` was called with an id that does not exist. */
|
|
37
|
+
declare class ImageNotFoundError extends ZeroBucketError {
|
|
38
|
+
readonly imageId: string;
|
|
39
|
+
constructor(imageId: string);
|
|
40
|
+
}
|
|
41
|
+
/** Underlying storage / database failure not covered above. */
|
|
42
|
+
declare class StorageError extends ZeroBucketError {
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* A `beforeGet` / `beforePut` hook denied the operation by returning false.
|
|
46
|
+
*
|
|
47
|
+
* If the hook itself throws, that exception propagates unchanged: it is NOT
|
|
48
|
+
* wrapped in AccessDeniedError and NOT treated as an implicit allow. A broken
|
|
49
|
+
* authorization check must fail closed, never open.
|
|
50
|
+
*/
|
|
51
|
+
declare class AccessDeniedError extends ZeroBucketError {
|
|
52
|
+
readonly operation: string;
|
|
53
|
+
readonly imageId?: string | undefined;
|
|
54
|
+
constructor(operation: string, imageId?: string | undefined);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Helpers shared by every storage adapter (kept here so Postgres and SQLite cannot drift apart). */
|
|
58
|
+
|
|
59
|
+
/** 1 MiB. Default chunk size for getStream(). */
|
|
60
|
+
declare const DEFAULT_STREAM_CHUNK_SIZE: number;
|
|
61
|
+
|
|
5
62
|
/**
|
|
6
63
|
* PostgreSQL storage adapter (classic mode).
|
|
7
64
|
*
|
|
@@ -20,8 +77,6 @@ import { Pool } from 'pg';
|
|
|
20
77
|
* race on CREATE TABLE.
|
|
21
78
|
*/
|
|
22
79
|
|
|
23
|
-
/** 1 MiB. Default chunk size for getStream(). */
|
|
24
|
-
declare const DEFAULT_STREAM_CHUNK_SIZE: number;
|
|
25
80
|
interface PostgresBackendOptions {
|
|
26
81
|
/** postgresql:// connection string. Required unless `pool` is given. */
|
|
27
82
|
connectionString?: string;
|
|
@@ -38,6 +93,18 @@ interface PostgresBackendOptions {
|
|
|
38
93
|
/** Max wait for a pooled connection. Default 10000. */
|
|
39
94
|
poolTimeoutMs?: number;
|
|
40
95
|
onOperation?: (event: OperationEvent) => void;
|
|
96
|
+
/**
|
|
97
|
+
* Enables transparent reads of tiered rows and `tierToObjectStorage()`.
|
|
98
|
+
* Configure it with the same bucket/credentials used to tier the images.
|
|
99
|
+
*/
|
|
100
|
+
objectStorage?: ObjectStorageLike;
|
|
101
|
+
/**
|
|
102
|
+
* Content-addressed mode: identical bytes are stored once (zerobucket_blobs) and each
|
|
103
|
+
* put() adds a lightweight reference (zerobucket_image_refs) with its own id and filename.
|
|
104
|
+
* Separate tables from classic mode, so the two can never be confused. Cannot be combined
|
|
105
|
+
* with `objectStorage`. Same schema as the Python package's dedup mode.
|
|
106
|
+
*/
|
|
107
|
+
dedup?: boolean;
|
|
41
108
|
}
|
|
42
109
|
declare class PostgresBackend implements StorageBackend {
|
|
43
110
|
private readonly opts;
|
|
@@ -48,6 +115,8 @@ declare class PostgresBackend implements StorageBackend {
|
|
|
48
115
|
private readonly maxRetries;
|
|
49
116
|
private readonly retryBaseDelayMs;
|
|
50
117
|
private readonly autoMigrate;
|
|
118
|
+
private readonly dedup;
|
|
119
|
+
private readonly sql;
|
|
51
120
|
constructor(opts: PostgresBackendOptions);
|
|
52
121
|
ready(): Promise<void>;
|
|
53
122
|
private init;
|
|
@@ -63,6 +132,14 @@ declare class PostgresBackend implements StorageBackend {
|
|
|
63
132
|
* failures retry the WHOLE closure on a fresh connection.
|
|
64
133
|
*/
|
|
65
134
|
private run;
|
|
135
|
+
/**
|
|
136
|
+
* Make multi-statement work all-or-nothing on a CALLER-supplied connection.
|
|
137
|
+
* - Caller has an open transaction -> SAVEPOINT: a failure rolls back only our work and the
|
|
138
|
+
* caller's transaction stays usable; success joins their transaction.
|
|
139
|
+
* - No open transaction (an autocommit pg.Client) -> open and commit our own, so a failure
|
|
140
|
+
* can never leave half-applied state (e.g. an inflated dedup ref_count).
|
|
141
|
+
*/
|
|
142
|
+
private atomic;
|
|
66
143
|
put(row: PreparedRow, { connection }?: CallOptions): Promise<string>;
|
|
67
144
|
putMany(rows: PreparedRow[], { connection }?: CallOptions): Promise<string[]>;
|
|
68
145
|
get(id: string, { connection }?: CallOptions): Promise<StoredRecord | null>;
|
|
@@ -72,60 +149,130 @@ declare class PostgresBackend implements StorageBackend {
|
|
|
72
149
|
delete(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
73
150
|
deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
|
|
74
151
|
exists(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
75
|
-
|
|
76
|
-
|
|
152
|
+
getInfo(): Promise<StorageInfo | null>;
|
|
153
|
+
listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
|
|
154
|
+
private deleteDedup;
|
|
155
|
+
private deleteManyDedup;
|
|
156
|
+
/**
|
|
157
|
+
* One-time, NON-DESTRUCTIVE copy of every row in the classic `zerobucket_images` table into this
|
|
158
|
+
* dedup instance's tables, preserving every id (external references keep working) and
|
|
159
|
+
* deduplicating identical content on the way. The classic table is never modified.
|
|
160
|
+
*
|
|
161
|
+
* Differences from the Python version, all deliberate: rows are copied in small batches (bounded
|
|
162
|
+
* memory instead of loading the whole table); re-running is safe (already-migrated ids are
|
|
163
|
+
* skipped and reported, not an error); and classic rows already tiered to object storage are
|
|
164
|
+
* refused up front with a clear message (their bytes are not in the database to copy).
|
|
165
|
+
* Still all-or-nothing: one transaction.
|
|
166
|
+
*/
|
|
167
|
+
migrateClassicToDedup(): Promise<{
|
|
168
|
+
imagesMigrated: number;
|
|
169
|
+
distinctBlobsCreated: number;
|
|
170
|
+
duplicateReferencesFound: number;
|
|
171
|
+
alreadyMigrated: number;
|
|
172
|
+
}>;
|
|
173
|
+
private hydrate;
|
|
174
|
+
/**
|
|
175
|
+
* Remove tiered objects AFTER their rows are gone. Ordering is deliberate:
|
|
176
|
+
* once the row is deleted the image is correctly "not found" to everyone, so
|
|
177
|
+
* a failed object delete only leaves a harmless orphan, whereas deleting the
|
|
178
|
+
* object first and then failing the row delete would leave a row pointing at nothing.
|
|
179
|
+
*
|
|
180
|
+
* Never runs inside a caller-owned transaction (`connection`): we cannot know
|
|
181
|
+
* whether it will commit, and destroying data before the commit outcome is known
|
|
182
|
+
* could leave a rolled-back row pointing at a deleted object. In that case the
|
|
183
|
+
* object stays until you remove it yourself after committing (objectStorage.delete(id)).
|
|
184
|
+
* Failures are reported through onOperation and never fail the delete itself.
|
|
185
|
+
*/
|
|
186
|
+
private cleanupObject;
|
|
187
|
+
/**
|
|
188
|
+
* Move an image's bytes out of Postgres into object storage, replacing the
|
|
189
|
+
* row's `data` with a pointer. Returns:
|
|
190
|
+
* null - no such image
|
|
191
|
+
* false - already tiered (a safe no-op, so backfills can be re-run)
|
|
192
|
+
* true - tiered just now
|
|
193
|
+
*
|
|
194
|
+
* SAFETY: the upload happens INSIDE the same transaction as the row lock
|
|
195
|
+
* (SELECT ... FOR UPDATE) and the UPDATE that flips the row. If the upload
|
|
196
|
+
* fails, the transaction rolls back and the row is untouched: there is no
|
|
197
|
+
* moment when the bytes exist nowhere, or when a row claims to be tiered
|
|
198
|
+
* without a completed upload.
|
|
199
|
+
*
|
|
200
|
+
* TRADEOFF: the row lock is held for the whole upload. Concurrent writes
|
|
201
|
+
* to THIS image (delete, tier) wait; every other row is unaffected, and plain
|
|
202
|
+
* reads are never blocked (Postgres MVCC). Meant for explicit maintenance, not hot paths.
|
|
203
|
+
*
|
|
204
|
+
* Retry is safe: a transient error replays the whole closure, and the
|
|
205
|
+
* deterministic key (the image id) makes the re-upload an idempotent overwrite.
|
|
206
|
+
*/
|
|
207
|
+
tierToObjectStorage(id: string, { connection }?: CallOptions): Promise<boolean | null>;
|
|
77
208
|
}
|
|
78
209
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
/** Any content (image or custom-validated) failed validation. */
|
|
92
|
-
declare class ContentValidationError extends ZeroBucketError {
|
|
93
|
-
}
|
|
94
|
-
/** An image failed validation (bad format, too large, corrupted, ...). */
|
|
95
|
-
declare class ImageValidationError extends ContentValidationError {
|
|
96
|
-
}
|
|
97
|
-
declare class ImageTooLargeError extends ImageValidationError {
|
|
98
|
-
readonly sizeBytes: number;
|
|
99
|
-
readonly maxBytes: number;
|
|
100
|
-
constructor(sizeBytes: number, maxBytes: number);
|
|
101
|
-
}
|
|
102
|
-
declare class UnsupportedFormatError extends ImageValidationError {
|
|
103
|
-
readonly detectedFormat: string | null;
|
|
104
|
-
readonly allowed: readonly string[];
|
|
105
|
-
constructor(detectedFormat: string | null, allowed: readonly string[]);
|
|
106
|
-
}
|
|
107
|
-
/** Image bytes could not be decoded despite a recognizable header. */
|
|
108
|
-
declare class CorruptedImageError extends ImageValidationError {
|
|
109
|
-
}
|
|
110
|
-
/** `get()`, `getStream()` or `metadata()` was called with an id that does not exist. */
|
|
111
|
-
declare class ImageNotFoundError extends ZeroBucketError {
|
|
112
|
-
readonly imageId: string;
|
|
113
|
-
constructor(imageId: string);
|
|
114
|
-
}
|
|
115
|
-
/** Underlying storage / database failure not covered above. */
|
|
116
|
-
declare class StorageError extends ZeroBucketError {
|
|
210
|
+
interface SQLiteBackendOptions {
|
|
211
|
+
/** Database file path, or ":memory:" for a throwaway in-process database (tiering is unavailable for ":memory:"). */
|
|
212
|
+
path: string;
|
|
213
|
+
/** Create the schema on first use. Default true. */
|
|
214
|
+
autoMigrate?: boolean;
|
|
215
|
+
/** Content-addressed mode (separate tables; same schema as the Python package). Cannot be combined with objectStorage. */
|
|
216
|
+
dedup?: boolean;
|
|
217
|
+
/** Enables tiering and transparent reads of tiered rows. */
|
|
218
|
+
objectStorage?: ObjectStorageLike;
|
|
219
|
+
/** Total time to keep retrying when another connection/process holds the write lock. Default 10000. */
|
|
220
|
+
busyTimeoutMs?: number;
|
|
221
|
+
onOperation?: (event: OperationEvent) => void;
|
|
117
222
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
readonly
|
|
127
|
-
|
|
128
|
-
|
|
223
|
+
declare class SQLiteBackend implements StorageBackend {
|
|
224
|
+
private readonly opts;
|
|
225
|
+
private db;
|
|
226
|
+
private Database;
|
|
227
|
+
private readyPromise;
|
|
228
|
+
private closed;
|
|
229
|
+
private readonly dedup;
|
|
230
|
+
private readonly busyTimeoutMs;
|
|
231
|
+
private readonly writeLock;
|
|
232
|
+
constructor(opts: SQLiteBackendOptions);
|
|
233
|
+
ready(): Promise<void>;
|
|
234
|
+
private init;
|
|
235
|
+
close(): Promise<void>;
|
|
236
|
+
private emit;
|
|
237
|
+
/** Re-run `fn` while another connection holds a lock, sleeping asynchronously between attempts. `fn` must be atomic. */
|
|
238
|
+
private retrying;
|
|
239
|
+
/** BEGIN IMMEDIATE takes the write lock up front, so a BUSY comes before any work and the whole unit is safely re-runnable. */
|
|
240
|
+
private tx;
|
|
241
|
+
/**
|
|
242
|
+
* All-or-nothing work on a CALLER-supplied handle. SAVEPOINT works whether or not the caller has an open
|
|
243
|
+
* transaction: inside one it nests (a failure rolls back only our work); outside one it begins and, on
|
|
244
|
+
* RELEASE, commits.
|
|
245
|
+
*/
|
|
246
|
+
private atomic;
|
|
247
|
+
private execute;
|
|
248
|
+
put(row: PreparedRow, { connection }?: CallOptions): Promise<string>;
|
|
249
|
+
private insertOne;
|
|
250
|
+
putMany(rows: PreparedRow[], { connection }?: CallOptions): Promise<string[]>;
|
|
251
|
+
get(id: string, { connection }?: CallOptions): Promise<StoredRecord | null>;
|
|
252
|
+
getMany(ids: string[], { connection }?: CallOptions): Promise<StoredRecord[]>;
|
|
253
|
+
getMetadata(id: string, { connection }?: CallOptions): Promise<StoredRecordMetadata | null>;
|
|
254
|
+
getStream(id: string, opts: StreamOptions): Promise<AsyncIterable<Buffer> | null>;
|
|
255
|
+
delete(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
256
|
+
deleteMany(ids: string[], { connection }?: CallOptions): Promise<string[]>;
|
|
257
|
+
exists(id: string, { connection }?: CallOptions): Promise<boolean>;
|
|
258
|
+
getInfo(): Promise<StorageInfo | null>;
|
|
259
|
+
listImages(o?: ImageListOptions): Promise<ImageListEntry[]>;
|
|
260
|
+
/**
|
|
261
|
+
* Remove tiered objects AFTER their rows are gone (a failed object delete leaves a harmless orphan; the
|
|
262
|
+
* reverse order could leave a row pointing at nothing). Skipped inside a caller-owned transaction, since the
|
|
263
|
+
* commit outcome is unknown. Failures never fail the delete; they are reported through onOperation.
|
|
264
|
+
*/
|
|
265
|
+
private cleanupObjects;
|
|
266
|
+
/**
|
|
267
|
+
* Move an image's bytes into object storage. null = no such image, false = already tiered, true = tiered now.
|
|
268
|
+
*
|
|
269
|
+
* SAFETY (same guarantee as Postgres): the upload happens INSIDE a BEGIN IMMEDIATE transaction; if it fails the
|
|
270
|
+
* transaction rolls back and the row is untouched.
|
|
271
|
+
*
|
|
272
|
+
* TRADEOFF (coarser than Postgres): SQLite has no row locks, so BEGIN IMMEDIATE locks the WHOLE DATABASE for the
|
|
273
|
+
* upload. Other processes' writes wait; this process's writes queue behind an async lock; reads are not blocked.
|
|
274
|
+
*/
|
|
275
|
+
tierToObjectStorage(id: string, { connection }?: CallOptions): Promise<boolean | null>;
|
|
129
276
|
}
|
|
130
277
|
|
|
131
278
|
declare const SUPPORTED_FORMATS: readonly string[];
|
|
@@ -161,4 +308,4 @@ declare class PDFValidator implements ContentValidator {
|
|
|
161
308
|
}): ValidatedContent;
|
|
162
309
|
}
|
|
163
310
|
|
|
164
|
-
export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };
|
|
311
|
+
export { AccessDeniedError, CallOptions, ContentValidationError, ContentValidator, CorruptedImageError, DEFAULT_MAX_PIXELS, DEFAULT_STREAM_CHUNK_SIZE, ImageNotFoundError, ImageTooLargeError, ImageValidationError, ObjectStorageLike, OperationEvent, PDFValidator, PostgresBackend, type PostgresBackendOptions, PreparedRow, SQLiteBackend, type SQLiteBackendOptions, SUPPORTED_FORMATS, StorageBackend, StorageError, StoredRecord, StoredRecordMetadata, UnsupportedFormatError, ValidatedContent, type ValidatedImage, ZeroBucketError, validateImage };
|