partial-content 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/LICENSE +21 -0
- package/README.md +601 -0
- package/SECURITY.md +92 -0
- package/dist/azure.d.ts +79 -0
- package/dist/azure.d.ts.map +1 -0
- package/dist/azure.js +251 -0
- package/dist/azure.js.map +1 -0
- package/dist/content-disposition.d.ts +74 -0
- package/dist/content-disposition.d.ts.map +1 -0
- package/dist/content-disposition.js +253 -0
- package/dist/content-disposition.js.map +1 -0
- package/dist/fs.d.ts +86 -0
- package/dist/fs.d.ts.map +1 -0
- package/dist/fs.js +375 -0
- package/dist/fs.js.map +1 -0
- package/dist/gcs.d.ts +72 -0
- package/dist/gcs.d.ts.map +1 -0
- package/dist/gcs.js +202 -0
- package/dist/gcs.js.map +1 -0
- package/dist/hono.d.ts +92 -0
- package/dist/hono.d.ts.map +1 -0
- package/dist/hono.js +61 -0
- package/dist/hono.js.map +1 -0
- package/dist/http.d.ts +70 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +281 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +27 -0
- package/dist/index.js.map +1 -0
- package/dist/kernel.d.ts +541 -0
- package/dist/kernel.d.ts.map +1 -0
- package/dist/kernel.js +1218 -0
- package/dist/kernel.js.map +1 -0
- package/dist/memory.d.ts +55 -0
- package/dist/memory.d.ts.map +1 -0
- package/dist/memory.js +107 -0
- package/dist/memory.js.map +1 -0
- package/dist/mime.d.ts +49 -0
- package/dist/mime.d.ts.map +1 -0
- package/dist/mime.js +150 -0
- package/dist/mime.js.map +1 -0
- package/dist/node.d.ts +84 -0
- package/dist/node.d.ts.map +1 -0
- package/dist/node.js +215 -0
- package/dist/node.js.map +1 -0
- package/dist/object-store.d.ts +472 -0
- package/dist/object-store.d.ts.map +1 -0
- package/dist/object-store.js +335 -0
- package/dist/object-store.js.map +1 -0
- package/dist/r2.d.ts +94 -0
- package/dist/r2.d.ts.map +1 -0
- package/dist/r2.js +150 -0
- package/dist/r2.js.map +1 -0
- package/dist/s3.d.ts +49 -0
- package/dist/s3.d.ts.map +1 -0
- package/dist/s3.js +263 -0
- package/dist/s3.js.map +1 -0
- package/dist/web.d.ts +336 -0
- package/dist/web.d.ts.map +1 -0
- package/dist/web.js +1094 -0
- package/dist/web.js.map +1 -0
- package/docs/DESIGN.md +426 -0
- package/package.json +182 -0
- package/src/azure.ts +329 -0
- package/src/content-disposition.ts +300 -0
- package/src/fs.ts +469 -0
- package/src/gcs.ts +290 -0
- package/src/hono.ts +123 -0
- package/src/http.ts +351 -0
- package/src/index.ts +85 -0
- package/src/kernel.ts +1498 -0
- package/src/memory.ts +148 -0
- package/src/mime.ts +160 -0
- package/src/node.ts +261 -0
- package/src/object-store.ts +665 -0
- package/src/r2.ts +232 -0
- package/src/s3.ts +324 -0
- package/src/web.ts +1603 -0
package/dist/fs.js
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local filesystem ObjectStore adapter for partial-content.
|
|
3
|
+
*
|
|
4
|
+
* Implements the read-only {@link ObjectStore} interface using Node.js
|
|
5
|
+
* `fs.stat()` and `fs.createReadStream()`. Suitable for:
|
|
6
|
+
* - Development servers
|
|
7
|
+
* - Small/medium deployments
|
|
8
|
+
* - Hybrid architectures (local cache + cloud primary)
|
|
9
|
+
* - Testing and CI pipelines
|
|
10
|
+
*
|
|
11
|
+
* Security: all keys are resolved relative to a fixed root directory.
|
|
12
|
+
* Path traversal attempts (`..`), absolute paths, and null bytes are
|
|
13
|
+
* rejected with ObjectNotFoundError. Symbolic links inside the root ARE
|
|
14
|
+
* followed (matching nginx/caddy defaults); do not place untrusted
|
|
15
|
+
* symlinks under the root if it must be a strict sandbox.
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* ```typescript
|
|
19
|
+
* import { fsStore } from "partial-content/fs";
|
|
20
|
+
* import { serveObject } from "partial-content/web";
|
|
21
|
+
*
|
|
22
|
+
* const store = fsStore({ root: "/var/data/uploads" });
|
|
23
|
+
*
|
|
24
|
+
* // Use with the web adapter:
|
|
25
|
+
* const handler = serveObject(store, { disposition: "inline" });
|
|
26
|
+
* ```
|
|
27
|
+
*
|
|
28
|
+
* @packageDocumentation
|
|
29
|
+
*/
|
|
30
|
+
import { open, stat } from "node:fs/promises";
|
|
31
|
+
import { resolve, relative, sep, isAbsolute } from "node:path";
|
|
32
|
+
import { ObjectNotFoundError, nodeStreamToWeb, } from "./index.js";
|
|
33
|
+
// Re-export for convenience (consumers can catch without importing the kernel)
|
|
34
|
+
export { ObjectNotFoundError };
|
|
35
|
+
// ─── Factory ────────────────────────────────────────────────────────────────
|
|
36
|
+
/**
|
|
37
|
+
* Create an {@link ObjectStore} backed by the local filesystem.
|
|
38
|
+
*
|
|
39
|
+
* @example
|
|
40
|
+
* ```typescript
|
|
41
|
+
* import { fsStore } from "partial-content/fs";
|
|
42
|
+
*
|
|
43
|
+
* const store = fsStore({ root: "/var/data/uploads" });
|
|
44
|
+
* const meta = await store.headObject("reports/q4.pdf");
|
|
45
|
+
* ```
|
|
46
|
+
*/
|
|
47
|
+
export function fsStore(opts) {
|
|
48
|
+
const root = resolve(opts.root);
|
|
49
|
+
const cacheTtl = opts.cache?.ttlMs ?? 0;
|
|
50
|
+
const cacheMax = opts.cache?.maxEntries ?? 1024;
|
|
51
|
+
const cacheMaxBytes = opts.cache?.maxBytes ?? DEFAULT_CACHE_MAX_BYTES;
|
|
52
|
+
const cache = cacheTtl > 0 ? new Map() : null;
|
|
53
|
+
// Total bytes held by cached bodies, maintained by cacheDelete/cacheSet.
|
|
54
|
+
// Metadata-only entries contribute zero.
|
|
55
|
+
let cacheBytes = 0;
|
|
56
|
+
/** Remove an entry, keeping the body byte accounting exact. */
|
|
57
|
+
function cacheDelete(key) {
|
|
58
|
+
const prev = cache.get(key);
|
|
59
|
+
if (prev === undefined)
|
|
60
|
+
return false;
|
|
61
|
+
cacheBytes -= prev.bytes ? prev.bytes.byteLength : 0;
|
|
62
|
+
cache.delete(key);
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
/** Evict the least recently used entry (first in insertion order). */
|
|
66
|
+
function evictOldest() {
|
|
67
|
+
const oldest = cache.keys().next();
|
|
68
|
+
if (!oldest.done)
|
|
69
|
+
cacheDelete(oldest.value);
|
|
70
|
+
}
|
|
71
|
+
function cacheGet(key) {
|
|
72
|
+
if (!cache)
|
|
73
|
+
return null;
|
|
74
|
+
const entry = cache.get(key);
|
|
75
|
+
if (!entry)
|
|
76
|
+
return null;
|
|
77
|
+
if (entry.expiresAt < Date.now()) {
|
|
78
|
+
cacheDelete(key);
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
// LRU touch: Map preserves insertion order; re-insert marks as recent.
|
|
82
|
+
// Same entry out and in, so the byte accounting is untouched.
|
|
83
|
+
cache.delete(key);
|
|
84
|
+
cache.set(key, entry);
|
|
85
|
+
return entry;
|
|
86
|
+
}
|
|
87
|
+
function cacheSet(key, entry) {
|
|
88
|
+
if (!cache)
|
|
89
|
+
return;
|
|
90
|
+
const prev = cache.get(key);
|
|
91
|
+
// Preserve a cached body across a metadata-only write-back: a range GET
|
|
92
|
+
// refreshes metadata but carries no bytes, and blindly replacing the
|
|
93
|
+
// entry would evict a still-valid full body (same representation) and
|
|
94
|
+
// force the next full GET back to disk. Keep the bytes when the
|
|
95
|
+
// representation is unchanged (same weak/strong validator); drop them
|
|
96
|
+
// when the etag moved (the file changed -> old bytes are stale).
|
|
97
|
+
const carried = (entry.bytes === undefined && prev?.bytes && prev.etag === entry.etag)
|
|
98
|
+
? { ...entry, bytes: prev.bytes }
|
|
99
|
+
: entry;
|
|
100
|
+
// A body that alone exceeds the byte budget can never fit: store the
|
|
101
|
+
// entry metadata-only rather than evicting every other body and still
|
|
102
|
+
// failing. The object itself is unaffected (this GET already served it).
|
|
103
|
+
const stored = carried.bytes && carried.bytes.byteLength > cacheMaxBytes
|
|
104
|
+
? { totalSize: carried.totalSize, lastModified: carried.lastModified, etag: carried.etag }
|
|
105
|
+
: carried;
|
|
106
|
+
// Delete-before-set: Map.set on an existing key keeps its old insertion
|
|
107
|
+
// order, so a refresh must re-insert to reach the most-recent LRU slot.
|
|
108
|
+
// It also makes eviction conditional on genuinely growing the map --
|
|
109
|
+
// refreshing a hot key at capacity must not evict an unrelated entry.
|
|
110
|
+
const existed = cacheDelete(key);
|
|
111
|
+
if (!existed && cache.size >= cacheMax) {
|
|
112
|
+
evictOldest();
|
|
113
|
+
}
|
|
114
|
+
// Byte budget: evict least-recent entries until the new body fits.
|
|
115
|
+
const newBytes = stored.bytes ? stored.bytes.byteLength : 0;
|
|
116
|
+
while (newBytes > 0 && cacheBytes + newBytes > cacheMaxBytes && cache.size > 0) {
|
|
117
|
+
evictOldest();
|
|
118
|
+
}
|
|
119
|
+
cache.set(key, { ...stored, expiresAt: Date.now() + cacheTtl });
|
|
120
|
+
cacheBytes += newBytes;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Resolve a key to an absolute path within the root directory.
|
|
124
|
+
* Rejects path traversal attempts that escape the root.
|
|
125
|
+
*/
|
|
126
|
+
function safePath(key) {
|
|
127
|
+
// Null bytes are invalid in every filesystem and are a classic probe for
|
|
128
|
+
// C-string truncation bugs in lower layers. Treat as "no such object"
|
|
129
|
+
// rather than letting fs throw ERR_INVALID_ARG_VALUE (which would map
|
|
130
|
+
// to a misleading 502 upstream).
|
|
131
|
+
if (key.includes("\0")) {
|
|
132
|
+
throw new ObjectNotFoundError(key);
|
|
133
|
+
}
|
|
134
|
+
// Windows-only hardening (sep === "\\"). ':' is illegal in a normal
|
|
135
|
+
// Windows filename -- it only appears in a drive designator ("D:\x")
|
|
136
|
+
// or an NTFS alternate-data-stream name ("secret.txt::$DATA"). Rejecting
|
|
137
|
+
// it early stops a cross-volume escape (resolve() would turn "D:\x" into
|
|
138
|
+
// an absolute path on another drive) and hidden-stream access. Reserved
|
|
139
|
+
// device names (NUL, CON, COM1...) map to hardware from ANY directory,
|
|
140
|
+
// so open() on them never touches a file under the root.
|
|
141
|
+
if (sep === "\\") {
|
|
142
|
+
if (key.includes(":")) {
|
|
143
|
+
throw new ObjectNotFoundError(key);
|
|
144
|
+
}
|
|
145
|
+
const base = key.slice(Math.max(key.lastIndexOf("/"), key.lastIndexOf("\\")) + 1);
|
|
146
|
+
if (/^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\.|$)/i.test(base)) {
|
|
147
|
+
throw new ObjectNotFoundError(key);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
const resolved = resolve(root, key);
|
|
151
|
+
const rel = relative(root, resolved);
|
|
152
|
+
// rel escapes the root when it climbs out ("..") or is itself absolute.
|
|
153
|
+
// A cross-drive key resolves onto another volume and relative() then
|
|
154
|
+
// returns that absolute path unchanged, which a bare startsWith("..")
|
|
155
|
+
// check would wave through.
|
|
156
|
+
if (rel === ".." || rel.startsWith(".." + sep) || isAbsolute(rel)) {
|
|
157
|
+
throw new ObjectNotFoundError(key);
|
|
158
|
+
}
|
|
159
|
+
return resolved;
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
162
|
+
supportsRange: true,
|
|
163
|
+
async headObject(key, opts) {
|
|
164
|
+
opts?.signal?.throwIfAborted();
|
|
165
|
+
const cached = cacheGet(key);
|
|
166
|
+
if (cached) {
|
|
167
|
+
return {
|
|
168
|
+
contentLength: cached.totalSize,
|
|
169
|
+
lastModified: cached.lastModified,
|
|
170
|
+
etag: cached.etag,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
const filePath = safePath(key);
|
|
174
|
+
let stats;
|
|
175
|
+
try {
|
|
176
|
+
stats = await stat(filePath, { bigint: true });
|
|
177
|
+
}
|
|
178
|
+
catch (err) {
|
|
179
|
+
if (isFileNotFound(err))
|
|
180
|
+
throw new ObjectNotFoundError(key, err);
|
|
181
|
+
throw err;
|
|
182
|
+
}
|
|
183
|
+
if (!stats.isFile()) {
|
|
184
|
+
throw new ObjectNotFoundError(key);
|
|
185
|
+
}
|
|
186
|
+
const meta = {
|
|
187
|
+
totalSize: Number(stats.size),
|
|
188
|
+
lastModified: stats.mtime.toUTCString(),
|
|
189
|
+
// Derive the weak validator once per stat instead of letting the
|
|
190
|
+
// orchestrator re-parse the Last-Modified string on every request
|
|
191
|
+
// (Date.parse on the serve hot path is measurable at 10k req/s).
|
|
192
|
+
etag: weakFsETag(stats.size, stats.mtimeNs),
|
|
193
|
+
};
|
|
194
|
+
cacheSet(key, meta);
|
|
195
|
+
return {
|
|
196
|
+
contentLength: meta.totalSize,
|
|
197
|
+
lastModified: meta.lastModified,
|
|
198
|
+
etag: meta.etag,
|
|
199
|
+
};
|
|
200
|
+
},
|
|
201
|
+
async getObject(key, opts) {
|
|
202
|
+
const { range, signal } = opts ?? {};
|
|
203
|
+
signal?.throwIfAborted();
|
|
204
|
+
// Hot path: a cached small body serves ranges and full reads as
|
|
205
|
+
// zero-copy subarray views -- no syscalls at all. Metadata-only
|
|
206
|
+
// entries (from headObject) cannot serve bytes and fall through.
|
|
207
|
+
const cached = cacheGet(key);
|
|
208
|
+
if (cached?.bytes) {
|
|
209
|
+
if (range && range.start >= cached.totalSize) {
|
|
210
|
+
// Unsatisfiable ranges are the orchestrator's job to reject
|
|
211
|
+
// (parseRangeHeader 416s them first); a direct caller gets a loud
|
|
212
|
+
// error instead of an empty body with lying bounds.
|
|
213
|
+
throw new RangeError(`fsStore ${key}: range start ${range.start} is beyond object size ${cached.totalSize}`);
|
|
214
|
+
}
|
|
215
|
+
// Clamp so ObjectStream.range always reports the bounds actually
|
|
216
|
+
// served, even for a direct caller passing an unclamped range.
|
|
217
|
+
const bytes = range
|
|
218
|
+
? cached.bytes.subarray(range.start, Math.min(range.end + 1, cached.totalSize))
|
|
219
|
+
: cached.bytes;
|
|
220
|
+
return {
|
|
221
|
+
// Node fs read buffers are ArrayBuffer-backed; narrow so the body is
|
|
222
|
+
// `new Response(...)`-assignable under DOM lib (F5).
|
|
223
|
+
body: bytes,
|
|
224
|
+
contentLength: bytes.length,
|
|
225
|
+
totalSize: cached.totalSize,
|
|
226
|
+
range: range ? { start: range.start, end: range.start + bytes.length - 1 } : undefined,
|
|
227
|
+
lastModified: cached.lastModified,
|
|
228
|
+
etag: cached.etag,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
const filePath = safePath(key);
|
|
232
|
+
// Open once and both stat and stream from the same file handle. A
|
|
233
|
+
// stat-then-reopen sequence races against file replacement: the stream
|
|
234
|
+
// could read a different (shorter) file than the one measured, producing
|
|
235
|
+
// a Content-Length that doesn't match the bytes sent.
|
|
236
|
+
let handle;
|
|
237
|
+
try {
|
|
238
|
+
handle = await open(filePath, "r");
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
if (isFileNotFound(err))
|
|
242
|
+
throw new ObjectNotFoundError(key, err);
|
|
243
|
+
throw err;
|
|
244
|
+
}
|
|
245
|
+
let stats;
|
|
246
|
+
try {
|
|
247
|
+
stats = await handle.stat({ bigint: true });
|
|
248
|
+
if (!stats.isFile())
|
|
249
|
+
throw new ObjectNotFoundError(key);
|
|
250
|
+
}
|
|
251
|
+
catch (err) {
|
|
252
|
+
await handle.close().catch(() => {
|
|
253
|
+
// Best-effort close; the stat/isFile error is the one that matters
|
|
254
|
+
});
|
|
255
|
+
throw err;
|
|
256
|
+
}
|
|
257
|
+
const totalSize = Number(stats.size);
|
|
258
|
+
const lastModified = stats.mtime.toUTCString();
|
|
259
|
+
const etag = weakFsETag(stats.size, stats.mtimeNs);
|
|
260
|
+
if (range && range.start >= totalSize) {
|
|
261
|
+
await handle.close().catch(() => {
|
|
262
|
+
// Best-effort close; the range error is the one that matters
|
|
263
|
+
});
|
|
264
|
+
throw new RangeError(`fsStore ${key}: range start ${range.start} is beyond object size ${totalSize}`);
|
|
265
|
+
}
|
|
266
|
+
// Clamp the end so served bounds always reflect the file's actual
|
|
267
|
+
// size (the orchestrator pre-clamps; direct callers may not).
|
|
268
|
+
const end = range ? Math.min(range.end, totalSize - 1) : totalSize - 1;
|
|
269
|
+
const contentLength = range ? end - range.start + 1 : totalSize;
|
|
270
|
+
// Small transfers: one exact-length positional read, close, and a
|
|
271
|
+
// single-chunk stream. Skips the ReadStream + async-iterator bridge
|
|
272
|
+
// machinery entirely, which dominates per-request cost for small
|
|
273
|
+
// files (the memory bound is SMALL_READ_LIMIT per in-flight request).
|
|
274
|
+
if (contentLength <= SMALL_READ_LIMIT) {
|
|
275
|
+
// Cache-destined buffers are allocated off-pool: for small sizes
|
|
276
|
+
// allocUnsafe returns a view into Node's shared 8 KiB slab, and a
|
|
277
|
+
// long-lived cache entry would pin the whole slab and expose
|
|
278
|
+
// adjacent slab bytes to any consumer that reads `.buffer` without
|
|
279
|
+
// honoring byteOffset. Only full-object reads populate bytes: a
|
|
280
|
+
// range slice is not the whole body, and mixing partial bytes into
|
|
281
|
+
// the cache would serve wrong content for other ranges.
|
|
282
|
+
const willCache = cache !== null && !range && contentLength === totalSize;
|
|
283
|
+
const buffer = willCache
|
|
284
|
+
? Buffer.allocUnsafeSlow(contentLength)
|
|
285
|
+
: Buffer.allocUnsafe(contentLength);
|
|
286
|
+
try {
|
|
287
|
+
const { bytesRead } = await handle.read(buffer, 0, contentLength, range?.start ?? 0);
|
|
288
|
+
// The handle's own stat promised these bytes; a short read means
|
|
289
|
+
// the file was truncated mid-request. Failing here (502, before
|
|
290
|
+
// headers) beats the streaming path's only option of a torn body.
|
|
291
|
+
if (bytesRead < contentLength) {
|
|
292
|
+
throw new Error(`fsStore ${key}: file shrank during read (expected ${contentLength} bytes, got ${bytesRead})`);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
finally {
|
|
296
|
+
await handle.close().catch(() => {
|
|
297
|
+
// Best-effort close; the bytes (or the read error) already decide the outcome
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
if (willCache) {
|
|
301
|
+
cacheSet(key, { totalSize, lastModified, etag, bytes: buffer });
|
|
302
|
+
}
|
|
303
|
+
else {
|
|
304
|
+
// Metadata write-back: every disk read refreshes the entry so
|
|
305
|
+
// HEAD/conditional traffic converges on the representation this
|
|
306
|
+
// GET actually served instead of a stale headObject snapshot.
|
|
307
|
+
cacheSet(key, { totalSize, lastModified, etag });
|
|
308
|
+
}
|
|
309
|
+
return {
|
|
310
|
+
body: buffer,
|
|
311
|
+
contentLength,
|
|
312
|
+
totalSize,
|
|
313
|
+
range: range ? { start: range.start, end } : undefined,
|
|
314
|
+
lastModified,
|
|
315
|
+
etag,
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
// autoClose (default true) closes the handle when the stream ends or
|
|
319
|
+
// is destroyed, covering completion, cancel, and abort paths.
|
|
320
|
+
const nodeStream = handle.createReadStream(range ? { start: range.start, end } : {});
|
|
321
|
+
// expectedBytes mirrors the small-read path's short-read guard onto the
|
|
322
|
+
// streaming path: an in-place truncation mid-serve ends the ReadStream
|
|
323
|
+
// early, and without this the body would under-run the committed
|
|
324
|
+
// Content-Length silently. The atomic write-temp+rename overwrite pattern
|
|
325
|
+
// does not trip it (the open handle keeps reading the original inode).
|
|
326
|
+
const webStream = nodeStreamToWeb(nodeStream, { signal, expectedBytes: contentLength });
|
|
327
|
+
// Metadata write-back (bytes stay uncached above the single-read
|
|
328
|
+
// limit): keeps HEAD/conditional responses coherent with the
|
|
329
|
+
// representation this GET is serving.
|
|
330
|
+
cacheSet(key, { totalSize, lastModified, etag });
|
|
331
|
+
return {
|
|
332
|
+
body: webStream,
|
|
333
|
+
contentLength,
|
|
334
|
+
totalSize,
|
|
335
|
+
range: range ? { start: range.start, end } : undefined,
|
|
336
|
+
lastModified,
|
|
337
|
+
etag,
|
|
338
|
+
};
|
|
339
|
+
},
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
// ─── Internal Helpers ───────────────────────────────────────────────────────
|
|
343
|
+
/**
|
|
344
|
+
* Transfers at or below this size are served by a single positional read
|
|
345
|
+
* instead of a ReadStream. 128 KiB covers typical documents, thumbnails,
|
|
346
|
+
* and range chunks while bounding per-request buffering.
|
|
347
|
+
*/
|
|
348
|
+
const SMALL_READ_LIMIT = 128 * 1024;
|
|
349
|
+
/**
|
|
350
|
+
* Default total byte budget for cached bodies (64 MiB). Together with the
|
|
351
|
+
* entry cap this bounds the cache's worst-case footprint explicitly, instead
|
|
352
|
+
* of leaving it implied by `maxEntries * SMALL_READ_LIMIT`.
|
|
353
|
+
*/
|
|
354
|
+
const DEFAULT_CACHE_MAX_BYTES = 64 * 1024 * 1024;
|
|
355
|
+
/**
|
|
356
|
+
* Weak validator from size + NANOSECOND mtime (bigint stat).
|
|
357
|
+
*
|
|
358
|
+
* The kernel's generic formatter floors mtime to whole seconds; on a
|
|
359
|
+
* filesystem store that recreates the classic weak-ETag hazard: two
|
|
360
|
+
* same-length writes within one second are indistinguishable, and a
|
|
361
|
+
* revalidating client gets a false-fresh 304 for changed bytes. Nanosecond
|
|
362
|
+
* resolution closes that window on ext4/APFS/NTFS; filesystems with coarse
|
|
363
|
+
* timestamps (FAT: 2 s) keep a residual window, documented in DESIGN.md.
|
|
364
|
+
*/
|
|
365
|
+
function weakFsETag(size, mtimeNs) {
|
|
366
|
+
return `W/"${size.toString(16)}-${mtimeNs.toString(16)}"`;
|
|
367
|
+
}
|
|
368
|
+
function isFileNotFound(err) {
|
|
369
|
+
if (err instanceof Error && "code" in err) {
|
|
370
|
+
const code = err.code;
|
|
371
|
+
return code === "ENOENT" || code === "ENOTDIR";
|
|
372
|
+
}
|
|
373
|
+
return false;
|
|
374
|
+
}
|
|
375
|
+
//# sourceMappingURL=fs.js.map
|
package/dist/fs.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fs.js","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC/D,OAAO,EACL,mBAAmB,EACnB,eAAe,GAKhB,MAAM,YAAY,CAAC;AAEpB,+EAA+E;AAC/E,OAAO,EAAE,mBAAmB,EAAE,CAAC;AA0D/B,+EAA+E;AAE/E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,OAAO,CAAC,IAAoB;IAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,UAAU,IAAI,IAAI,CAAC;IAChD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,EAAE,QAAQ,IAAI,uBAAuB,CAAC;IACtE,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAsB,CAAC,CAAC,CAAC,IAAI,CAAC;IAClE,yEAAyE;IACzE,yCAAyC;IACzC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,+DAA+D;IAC/D,SAAS,WAAW,CAAC,GAAW;QAC9B,MAAM,IAAI,GAAG,KAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACrC,UAAU,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,KAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sEAAsE;IACtE,SAAS,WAAW;QAClB,MAAM,MAAM,GAAG,KAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI;YAAE,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,SAAS,QAAQ,CAAC,GAAW;QAC3B,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACjC,WAAW,CAAC,GAAG,CAAC,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,uEAAuE;QACvE,8DAA8D;QAC9D,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,QAAQ,CAAC,GAAW,EAAE,KAAoC;QACjE,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,wEAAwE;QACxE,qEAAqE;QACrE,sEAAsE;QACtE,gEAAgE;QAChE,sEAAsE;QACtE,iEAAiE;QACjE,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC;YACpF,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;YACjC,CAAC,CAAC,KAAK,CAAC;QACV,qEAAqE;QACrE,sEAAsE;QACtE,yEAAyE;QACzE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,aAAa;YACtE,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;YAC1F,CAAC,CAAC,OAAO,CAAC;QACZ,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,sEAAsE;QACtE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;YACvC,WAAW,EAAE,CAAC;QAChB,CAAC;QACD,mEAAmE;QACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,OAAO,QAAQ,GAAG,CAAC,IAAI,UAAU,GAAG,QAAQ,GAAG,aAAa,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/E,WAAW,EAAE,CAAC;QAChB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC;QAChE,UAAU,IAAI,QAAQ,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,SAAS,QAAQ,CAAC,GAAW;QAC3B,yEAAyE;QACzE,sEAAsE;QACtE,sEAAsE;QACtE,iCAAiC;QACjC,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QAED,oEAAoE;QACpE,qEAAqE;QACrE,yEAAyE;QACzE,yEAAyE;QACzE,wEAAwE;QACxE,uEAAuE;QACvE,yDAAyD;QACzD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;YACrC,CAAC;YACD,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CACpB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAC1D,CAAC;YACF,IAAI,6CAA6C,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7D,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAErC,wEAAwE;QACxE,qEAAqE;QACrE,sEAAsE;QACtE,4BAA4B;QAC5B,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO;QACL,aAAa,EAAE,IAAI;QAEnB,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,IAA+B;YAC3D,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO;oBACL,aAAa,EAAE,MAAM,CAAC,SAAS;oBAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,IAAI,EAAE,MAAM,CAAC,IAAI;iBAClB,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YAE/B,IAAI,KAAK,CAAC;YACV,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,cAAc,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,mBAAmB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACjE,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpB,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;YACrC,CAAC;YAED,MAAM,IAAI,GAAG;gBACX,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;gBAC7B,YAAY,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;gBACvC,iEAAiE;gBACjE,kEAAkE;gBAClE,iEAAiE;gBACjE,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;aAC5C,CAAC;YACF,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACpB,OAAO;gBACL,aAAa,EAAE,IAAI,CAAC,SAAS;gBAC7B,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,IAAoD;YAC/E,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,EAAE,cAAc,EAAE,CAAC;YAEzB,gEAAgE;YAChE,gEAAgE;YAChE,iEAAiE;YACjE,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,MAAM,EAAE,KAAK,EAAE,CAAC;gBAClB,IAAI,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;oBAC7C,4DAA4D;oBAC5D,kEAAkE;oBAClE,oDAAoD;oBACpD,MAAM,IAAI,UAAU,CAClB,WAAW,GAAG,iBAAiB,KAAK,CAAC,KAAK,0BAA0B,MAAM,CAAC,SAAS,EAAE,CACvF,CAAC;gBACJ,CAAC;gBACD,iEAAiE;gBACjE,+DAA+D;gBAC/D,MAAM,KAAK,GAAG,KAAK;oBACjB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;oBAC/E,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBACjB,OAAO;oBACL,qEAAqE;oBACrE,qDAAqD;oBACrD,IAAI,EAAE,KAAgC;oBACtC,aAAa,EAAE,KAAK,CAAC,MAAM;oBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;oBACtF,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,IAAI,EAAE,MAAM,CAAC,IAAI;iBAClB,CAAC;YACJ,CAAC;YAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YAE/B,kEAAkE;YAClE,uEAAuE;YACvE,yEAAyE;YACzE,sDAAsD;YACtD,IAAI,MAAM,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YACrC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,cAAc,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,mBAAmB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBACjE,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,IAAI,KAAK,CAAC;YACV,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5C,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;oBAAE,MAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,CAAC;YAC1D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC9B,mEAAmE;gBACrE,CAAC,CAAC,CAAC;gBACH,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAEnD,IAAI,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;gBACtC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC9B,6DAA6D;gBAC/D,CAAC,CAAC,CAAC;gBACH,MAAM,IAAI,UAAU,CAClB,WAAW,GAAG,iBAAiB,KAAK,CAAC,KAAK,0BAA0B,SAAS,EAAE,CAChF,CAAC;YACJ,CAAC;YACD,kEAAkE;YAClE,8DAA8D;YAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;YACvE,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAEhE,kEAAkE;YAClE,oEAAoE;YACpE,iEAAiE;YACjE,sEAAsE;YACtE,IAAI,aAAa,IAAI,gBAAgB,EAAE,CAAC;gBACtC,iEAAiE;gBACjE,kEAAkE;gBAClE,6DAA6D;gBAC7D,mEAAmE;gBACnE,gEAAgE;gBAChE,mEAAmE;gBACnE,wDAAwD;gBACxD,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,aAAa,KAAK,SAAS,CAAC;gBAC1E,MAAM,MAAM,GAAG,SAAS;oBACtB,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,aAAa,CAAC;oBACvC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;gBACtC,IAAI,CAAC;oBACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;oBACrF,iEAAiE;oBACjE,gEAAgE;oBAChE,kEAAkE;oBAClE,IAAI,SAAS,GAAG,aAAa,EAAE,CAAC;wBAC9B,MAAM,IAAI,KAAK,CACb,WAAW,GAAG,uCAAuC,aAAa,eAAe,SAAS,GAAG,CAC9F,CAAC;oBACJ,CAAC;gBACH,CAAC;wBAAS,CAAC;oBACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;wBAC9B,8EAA8E;oBAChF,CAAC,CAAC,CAAC;gBACL,CAAC;gBACD,IAAI,SAAS,EAAE,CAAC;oBACd,QAAQ,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClE,CAAC;qBAAM,CAAC;oBACN,8DAA8D;oBAC9D,gEAAgE;oBAChE,8DAA8D;oBAC9D,QAAQ,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;gBACnD,CAAC;gBACD,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,aAAa;oBACb,SAAS;oBACT,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS;oBACtD,YAAY;oBACZ,IAAI;iBACL,CAAC;YACJ,CAAC;YAED,qEAAqE;YACrE,8DAA8D;YAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CACxC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CACzC,CAAC;YAEF,wEAAwE;YACxE,uEAAuE;YACvE,iEAAiE;YACjE,0EAA0E;YAC1E,uEAAuE;YACvE,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC,CAAC;YAExF,iEAAiE;YACjE,6DAA6D;YAC7D,sCAAsC;YACtC,QAAQ,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YAEjD,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,aAAa;gBACb,SAAS;gBACT,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS;gBACtD,YAAY;gBACZ,IAAI;aACL,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,GAAG,GAAG,IAAI,CAAC;AAEpC;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEjD;;;;;;;;;GASG;AACH,SAAS,UAAU,CAAC,IAAY,EAAE,OAAe;IAC/C,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC;AAC5D,CAAC;AAED,SAAS,cAAc,CAAC,GAAY;IAClC,IAAI,GAAG,YAAY,KAAK,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAI,GAAwB,CAAC,IAAI,CAAC;QAC5C,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,SAAS,CAAC;IACjD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/dist/gcs.d.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Google Cloud Storage ObjectStore adapter for partial-content.
|
|
3
|
+
*
|
|
4
|
+
* Wraps `@google-cloud/storage` to implement the {@link ObjectStore} interface.
|
|
5
|
+
*
|
|
6
|
+
* @example
|
|
7
|
+
* ```typescript
|
|
8
|
+
* import { Storage } from "@google-cloud/storage";
|
|
9
|
+
* import { gcsStore } from "partial-content/gcs";
|
|
10
|
+
*
|
|
11
|
+
* const storage = new Storage();
|
|
12
|
+
* const store = gcsStore({ storage, bucket: "my-bucket" });
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* @packageDocumentation
|
|
16
|
+
*/
|
|
17
|
+
import { ObjectNotFoundError, ObjectChangedError, StoreUnavailableError, type ObjectStore } from "./index.js";
|
|
18
|
+
export { ObjectNotFoundError, ObjectChangedError, StoreUnavailableError };
|
|
19
|
+
/**
|
|
20
|
+
* Minimal GCS Storage interface.
|
|
21
|
+
*
|
|
22
|
+
* Declared locally to avoid a hard dependency on `@google-cloud/storage`.
|
|
23
|
+
* Users who import `partial-content/gcs` will have it installed as an
|
|
24
|
+
* optional peer dependency.
|
|
25
|
+
*/
|
|
26
|
+
interface GcsStorage {
|
|
27
|
+
bucket(name: string): GcsBucket;
|
|
28
|
+
}
|
|
29
|
+
interface GcsBucket {
|
|
30
|
+
file(name: string, opts?: {
|
|
31
|
+
generation?: string | number;
|
|
32
|
+
}): GcsFile;
|
|
33
|
+
}
|
|
34
|
+
interface GcsFile {
|
|
35
|
+
getMetadata(): Promise<[GcsFileMetadata]>;
|
|
36
|
+
createReadStream(opts?: {
|
|
37
|
+
start?: number;
|
|
38
|
+
end?: number;
|
|
39
|
+
}): NodeJS.ReadableStream & AsyncIterable<Buffer>;
|
|
40
|
+
}
|
|
41
|
+
interface GcsFileMetadata {
|
|
42
|
+
size: string;
|
|
43
|
+
etag?: string;
|
|
44
|
+
/** Object generation: changes on every overwrite. Used to pin reads. */
|
|
45
|
+
generation?: string | number;
|
|
46
|
+
updated?: string;
|
|
47
|
+
md5Hash?: string;
|
|
48
|
+
crc32c?: string;
|
|
49
|
+
metadata?: Record<string, string>;
|
|
50
|
+
}
|
|
51
|
+
export interface GcsStoreOptions {
|
|
52
|
+
/** Pre-configured @google-cloud/storage instance. */
|
|
53
|
+
storage: GcsStorage;
|
|
54
|
+
/** GCS bucket name. */
|
|
55
|
+
bucket: string;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Create an {@link ObjectStore} backed by a Google Cloud Storage bucket.
|
|
59
|
+
*
|
|
60
|
+
* @example
|
|
61
|
+
* ```typescript
|
|
62
|
+
* import { Storage } from "@google-cloud/storage";
|
|
63
|
+
* import { gcsStore } from "partial-content/gcs";
|
|
64
|
+
*
|
|
65
|
+
* const storage = new Storage();
|
|
66
|
+
* const store = gcsStore({ storage, bucket: "my-bucket" });
|
|
67
|
+
*
|
|
68
|
+
* const meta = await store.headObject("reports/q4.pdf");
|
|
69
|
+
* ```
|
|
70
|
+
*/
|
|
71
|
+
export declare function gcsStore(opts: GcsStoreOptions): ObjectStore;
|
|
72
|
+
//# sourceMappingURL=gcs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gcs.d.ts","sourceRoot":"","sources":["../src/gcs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,EAGrB,KAAK,WAAW,EAKjB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,CAAC;AAI1E;;;;;;GAMG;AACH,UAAU,UAAU;IAClB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CACjC;AAED,UAAU,SAAS;IACjB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;CACtE;AAED,UAAU,OAAO;IACf,WAAW,IAAI,OAAO,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;IAC1C,gBAAgB,CAAC,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;CAC1G;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,wEAAwE;IACxE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAID,MAAM,WAAW,eAAe;IAC9B,qDAAqD;IACrD,OAAO,EAAE,UAAU,CAAC;IACpB,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;CAChB;AAID;;;;;;;;;;;;;GAaG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,eAAe,GAAG,WAAW,CAsH3D"}
|
package/dist/gcs.js
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Google Cloud Storage ObjectStore adapter for partial-content.
|
|
3
|
+
*
|
|
4
|
+
* Wraps `@google-cloud/storage` to implement the {@link ObjectStore} interface.
|
|
5
|
+
*
|
|
6
|
+
* @example
|
|
7
|
+
* ```typescript
|
|
8
|
+
* import { Storage } from "@google-cloud/storage";
|
|
9
|
+
* import { gcsStore } from "partial-content/gcs";
|
|
10
|
+
*
|
|
11
|
+
* const storage = new Storage();
|
|
12
|
+
* const store = gcsStore({ storage, bucket: "my-bucket" });
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* @packageDocumentation
|
|
16
|
+
*/
|
|
17
|
+
import { ObjectNotFoundError, ObjectChangedError, StoreUnavailableError, classifyStoreRead, nodeStreamToWeb, } from "./index.js";
|
|
18
|
+
// Re-export for convenience
|
|
19
|
+
export { ObjectNotFoundError, ObjectChangedError, StoreUnavailableError };
|
|
20
|
+
// ─── Factory ────────────────────────────────────────────────────────────────
|
|
21
|
+
/**
|
|
22
|
+
* Create an {@link ObjectStore} backed by a Google Cloud Storage bucket.
|
|
23
|
+
*
|
|
24
|
+
* @example
|
|
25
|
+
* ```typescript
|
|
26
|
+
* import { Storage } from "@google-cloud/storage";
|
|
27
|
+
* import { gcsStore } from "partial-content/gcs";
|
|
28
|
+
*
|
|
29
|
+
* const storage = new Storage();
|
|
30
|
+
* const store = gcsStore({ storage, bucket: "my-bucket" });
|
|
31
|
+
*
|
|
32
|
+
* const meta = await store.headObject("reports/q4.pdf");
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
export function gcsStore(opts) {
|
|
36
|
+
const bucket = opts.storage.bucket(opts.bucket);
|
|
37
|
+
return {
|
|
38
|
+
supportsRange: true,
|
|
39
|
+
async headObject(key, opts) {
|
|
40
|
+
opts?.signal?.throwIfAborted();
|
|
41
|
+
const [metadata] = await classifyStoreRead(key, () => bucket.file(key).getMetadata(), gcsClassifiers);
|
|
42
|
+
return {
|
|
43
|
+
contentLength: parseInt(metadata.size, 10),
|
|
44
|
+
etag: metadata.etag,
|
|
45
|
+
lastModified: metadata.updated
|
|
46
|
+
? new Date(metadata.updated).toUTCString()
|
|
47
|
+
: undefined,
|
|
48
|
+
// A generation names an immutable object version, so it can carry
|
|
49
|
+
// everything getObject would otherwise re-fetch. No generation
|
|
50
|
+
// (emulators, mocks) means no pin: getObject falls back to its own
|
|
51
|
+
// metadata read.
|
|
52
|
+
pin: metadata.generation !== undefined
|
|
53
|
+
? encodeGcsPin(metadata)
|
|
54
|
+
: undefined,
|
|
55
|
+
};
|
|
56
|
+
},
|
|
57
|
+
async getObject(key, opts) {
|
|
58
|
+
const { range, signal, ifMatch, pin } = opts ?? {};
|
|
59
|
+
signal?.throwIfAborted();
|
|
60
|
+
// A pin from our own headObject carries the generation, size, and
|
|
61
|
+
// validators of the exact representation the caller validated, so the
|
|
62
|
+
// metadata re-fetch is skipped entirely: one backend read per GET.
|
|
63
|
+
// A pin whose etag disagrees with ifMatch (or that fails to decode)
|
|
64
|
+
// is ignored and the metadata path revalidates from scratch. When an
|
|
65
|
+
// ifMatch is present the pin must carry a matching etag to be trusted:
|
|
66
|
+
// a pin lacking the validator cannot silently satisfy the precondition,
|
|
67
|
+
// it falls through to the metadata path that actually enforces ifMatch.
|
|
68
|
+
const pinned = pin ? decodeGcsPin(pin) : null;
|
|
69
|
+
const usablePin = pinned && (!ifMatch || pinned.etag === ifMatch)
|
|
70
|
+
? pinned
|
|
71
|
+
: null;
|
|
72
|
+
let generation;
|
|
73
|
+
let totalSize;
|
|
74
|
+
let etag;
|
|
75
|
+
let lastModified;
|
|
76
|
+
if (usablePin) {
|
|
77
|
+
generation = usablePin.generation;
|
|
78
|
+
totalSize = usablePin.size;
|
|
79
|
+
etag = usablePin.etag;
|
|
80
|
+
lastModified = usablePin.lastModified;
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
// GCS streams carry no total size, the ifMatch check needs the
|
|
84
|
+
// current etag, and generation pinning needs the current generation.
|
|
85
|
+
const [metadata] = await classifyStoreRead(key, () => bucket.file(key).getMetadata(), gcsClassifiers);
|
|
86
|
+
// Caller pin: the object must still match the validator captured at
|
|
87
|
+
// HEAD time. GCS etags change on every overwrite, so a mismatch means
|
|
88
|
+
// the representation the caller validated no longer exists.
|
|
89
|
+
if (ifMatch && metadata.etag && metadata.etag !== ifMatch) {
|
|
90
|
+
throw new ObjectChangedError(key);
|
|
91
|
+
}
|
|
92
|
+
generation = metadata.generation;
|
|
93
|
+
totalSize = parseInt(metadata.size, 10);
|
|
94
|
+
etag = metadata.etag;
|
|
95
|
+
lastModified = metadata.updated
|
|
96
|
+
? new Date(metadata.updated).toUTCString()
|
|
97
|
+
: undefined;
|
|
98
|
+
}
|
|
99
|
+
if (range && range.start >= totalSize) {
|
|
100
|
+
// Unsatisfiable ranges are the orchestrator's job to reject
|
|
101
|
+
// (parseRangeHeader 416s them first); a direct caller gets a loud
|
|
102
|
+
// error instead of a truncated body under an inflated Content-Length.
|
|
103
|
+
throw new RangeError(`gcsStore ${key}: range start ${range.start} is beyond object size ${totalSize}`);
|
|
104
|
+
}
|
|
105
|
+
// Clamp the end so contentLength and the reported range match what
|
|
106
|
+
// the backend stream will actually deliver (createReadStream clamps
|
|
107
|
+
// to EOF silently; the reported bounds must not diverge from it).
|
|
108
|
+
const end = range ? Math.min(range.end, totalSize - 1) : totalSize - 1;
|
|
109
|
+
const streamOpts = range
|
|
110
|
+
? { start: range.start, end }
|
|
111
|
+
: undefined;
|
|
112
|
+
// Pin the stream to the generation just measured (or the caller's
|
|
113
|
+
// pinned generation): metadata and bytes come from the SAME object
|
|
114
|
+
// version even if it is overwritten between the two calls (reading a
|
|
115
|
+
// specific generation is immutable in GCS).
|
|
116
|
+
const pinnedFile = generation !== undefined
|
|
117
|
+
? bucket.file(key, { generation })
|
|
118
|
+
: bucket.file(key);
|
|
119
|
+
const nodeStream = pinnedFile.createReadStream(streamOpts);
|
|
120
|
+
const contentLength = range
|
|
121
|
+
? (end - range.start + 1)
|
|
122
|
+
: totalSize;
|
|
123
|
+
// nodeStreamToWeb auto-detects the stream's destroy() capability.
|
|
124
|
+
// expectedBytes guards a graceful short-read: a truncated GCS download
|
|
125
|
+
// that ends cleanly below the committed length errors the body instead
|
|
126
|
+
// of under-running the Content-Length undetected.
|
|
127
|
+
const webStream = nodeStreamToWeb(nodeStream, { signal, expectedBytes: contentLength });
|
|
128
|
+
return {
|
|
129
|
+
body: webStream,
|
|
130
|
+
contentLength,
|
|
131
|
+
totalSize,
|
|
132
|
+
range: range ? { start: range.start, end } : undefined,
|
|
133
|
+
etag,
|
|
134
|
+
lastModified,
|
|
135
|
+
};
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
function encodeGcsPin(metadata) {
|
|
140
|
+
const pin = {
|
|
141
|
+
generation: metadata.generation,
|
|
142
|
+
size: parseInt(metadata.size, 10),
|
|
143
|
+
etag: metadata.etag,
|
|
144
|
+
lastModified: metadata.updated
|
|
145
|
+
? new Date(metadata.updated).toUTCString()
|
|
146
|
+
: undefined,
|
|
147
|
+
};
|
|
148
|
+
return JSON.stringify(pin);
|
|
149
|
+
}
|
|
150
|
+
/** Decode a pin token; null for foreign/corrupt tokens (fall back to metadata). */
|
|
151
|
+
function decodeGcsPin(pin) {
|
|
152
|
+
try {
|
|
153
|
+
const parsed = JSON.parse(pin);
|
|
154
|
+
if (typeof parsed === "object" && parsed !== null &&
|
|
155
|
+
"generation" in parsed && "size" in parsed &&
|
|
156
|
+
typeof parsed.size === "number" &&
|
|
157
|
+
Number.isFinite(parsed.size) &&
|
|
158
|
+
// `generation` flows straight into bucket.file(key, { generation }) and
|
|
159
|
+
// selects the immutable version served. Only a non-empty string or a
|
|
160
|
+
// finite number is a real generation; anything else (object, array,
|
|
161
|
+
// null) is a corrupt/hostile token that must revalidate from scratch,
|
|
162
|
+
// never index an arbitrary version.
|
|
163
|
+
isValidGeneration(parsed.generation)) {
|
|
164
|
+
return parsed;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
catch { /* not our token: revalidate via metadata instead */ }
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
/** A GCS generation is a non-empty string or a finite number; nothing else. */
|
|
171
|
+
function isValidGeneration(gen) {
|
|
172
|
+
return (typeof gen === "string" && gen.length > 0) ||
|
|
173
|
+
(typeof gen === "number" && Number.isFinite(gen));
|
|
174
|
+
}
|
|
175
|
+
function isGcsNotFound(err) {
|
|
176
|
+
if (typeof err === "object" && err !== null && "code" in err) {
|
|
177
|
+
return err.code === 404;
|
|
178
|
+
}
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Whether a GCS error is a transient throttle/overload the client should retry
|
|
183
|
+
* (mapped to a 503, not a 502). The Cloud Storage client surfaces the HTTP
|
|
184
|
+
* status on `err.code`: 429 (rate limit) or 503 (backend unavailable).
|
|
185
|
+
*/
|
|
186
|
+
function isGcsThrottled(err) {
|
|
187
|
+
if (typeof err === "object" && err !== null && "code" in err) {
|
|
188
|
+
const code = err.code;
|
|
189
|
+
return code === 429 || code === 503;
|
|
190
|
+
}
|
|
191
|
+
return false;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* The error-classification set shared by both metadata reads. GCS has no
|
|
195
|
+
* `changed` predicate: its pin is an etag comparison (handled inline, throwing
|
|
196
|
+
* {@link ObjectChangedError} directly), not a native conditional error.
|
|
197
|
+
*/
|
|
198
|
+
const gcsClassifiers = {
|
|
199
|
+
notFound: isGcsNotFound,
|
|
200
|
+
throttled: isGcsThrottled,
|
|
201
|
+
};
|
|
202
|
+
//# sourceMappingURL=gcs.js.map
|