pyric-admin 0.1.0-alpha.21 → 0.1.0-alpha.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -4
- package/dist/app-check/index.d.ts +3 -0
- package/dist/app-check/index.d.ts.map +1 -0
- package/dist/app-check/index.js +11 -0
- package/dist/app-check/index.js.map +1 -0
- package/dist/data-connect/index.d.ts +3 -0
- package/dist/data-connect/index.d.ts.map +1 -0
- package/dist/data-connect/index.js +11 -0
- package/dist/data-connect/index.js.map +1 -0
- package/dist/deferred.d.ts +21 -0
- package/dist/deferred.d.ts.map +1 -0
- package/dist/deferred.js +29 -0
- package/dist/deferred.js.map +1 -0
- package/dist/eventarc/index.d.ts +3 -0
- package/dist/eventarc/index.d.ts.map +1 -0
- package/dist/eventarc/index.js +11 -0
- package/dist/eventarc/index.js.map +1 -0
- package/dist/extensions/index.d.ts +3 -0
- package/dist/extensions/index.d.ts.map +1 -0
- package/dist/extensions/index.js +11 -0
- package/dist/extensions/index.js.map +1 -0
- package/dist/functions/index.d.ts +3 -0
- package/dist/functions/index.d.ts.map +1 -0
- package/dist/functions/index.js +11 -0
- package/dist/functions/index.js.map +1 -0
- package/dist/installations/index.d.ts +3 -0
- package/dist/installations/index.d.ts.map +1 -0
- package/dist/installations/index.js +11 -0
- package/dist/installations/index.js.map +1 -0
- package/dist/instance-id/index.d.ts +3 -0
- package/dist/instance-id/index.d.ts.map +1 -0
- package/dist/instance-id/index.js +11 -0
- package/dist/instance-id/index.js.map +1 -0
- package/dist/machine-learning/index.d.ts +3 -0
- package/dist/machine-learning/index.d.ts.map +1 -0
- package/dist/machine-learning/index.js +11 -0
- package/dist/machine-learning/index.js.map +1 -0
- package/dist/messaging/index.d.ts +0 -1
- package/dist/messaging/index.d.ts.map +1 -1
- package/dist/messaging/index.js +7 -8
- package/dist/messaging/index.js.map +1 -1
- package/dist/phone-number-verification/index.d.ts +3 -0
- package/dist/phone-number-verification/index.d.ts.map +1 -0
- package/dist/phone-number-verification/index.js +11 -0
- package/dist/phone-number-verification/index.js.map +1 -0
- package/dist/project-management/index.d.ts +3 -0
- package/dist/project-management/index.d.ts.map +1 -0
- package/dist/project-management/index.js +11 -0
- package/dist/project-management/index.js.map +1 -0
- package/dist/remote-config/index.d.ts +3 -0
- package/dist/remote-config/index.d.ts.map +1 -0
- package/dist/remote-config/index.js +11 -0
- package/dist/remote-config/index.js.map +1 -0
- package/dist/security-rules/index.d.ts +3 -0
- package/dist/security-rules/index.d.ts.map +1 -0
- package/dist/security-rules/index.js +11 -0
- package/dist/security-rules/index.js.map +1 -0
- package/dist/storage/index.d.ts +88 -16
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +409 -69
- package/dist/storage/index.js.map +1 -1
- package/package.json +50 -2
- package/src/app-check/index.ts +14 -0
- package/src/data-connect/index.ts +14 -0
- package/src/deferred.ts +33 -0
- package/src/eventarc/index.ts +14 -0
- package/src/extensions/index.ts +14 -0
- package/src/functions/index.ts +14 -0
- package/src/installations/index.ts +15 -0
- package/src/instance-id/index.ts +14 -0
- package/src/machine-learning/index.ts +14 -0
- package/src/messaging/index.ts +2 -9
- package/src/phone-number-verification/index.ts +14 -0
- package/src/project-management/index.ts +15 -0
- package/src/remote-config/index.ts +15 -0
- package/src/security-rules/index.ts +14 -0
- package/src/storage/index.ts +520 -89
package/dist/storage/index.js
CHANGED
|
@@ -7,10 +7,10 @@
|
|
|
7
7
|
*
|
|
8
8
|
* - **Remote sandbox path** — a handle branded by `@pyric/cli`'
|
|
9
9
|
* `connectRemoteSandbox()`/`remoteSandbox()` relays every data
|
|
10
|
-
* operation
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* the remote arm section below.
|
|
10
|
+
* operation to the sandbox's host (admin lens pinned — rules bypass).
|
|
11
|
+
* Bytes move over the host's HTTP byte route when it has one, and as
|
|
12
|
+
* frames when it does not (a browser tab's SharedWorker). Single bucket;
|
|
13
|
+
* `getSignedUrl` stays the local stub. See the remote arm section below.
|
|
14
14
|
*
|
|
15
15
|
* - **Sandbox path** — returns an in-process {@link Storage} backed
|
|
16
16
|
* by an in-memory `Map<bucketName, Map<path, FileEntry>>`. State
|
|
@@ -23,19 +23,37 @@
|
|
|
23
23
|
* - `storage.bucket(name?)` → {@link Bucket}-shaped handle
|
|
24
24
|
* - `bucket.file(path)` → {@link File}-shaped handle
|
|
25
25
|
* - `file.save(data, options?)` — `Buffer | string | Uint8Array`
|
|
26
|
-
* - `file.download(options?)` → `[Buffer]`
|
|
26
|
+
* - `file.download(options?)` → `[Buffer]`, whole or `start..end`
|
|
27
|
+
* - `file.createReadStream(options?)`, whole or `start..end`
|
|
28
|
+
* - `file.createWriteStream(options?)`
|
|
29
|
+
* - `file.getMetadata()` / `file.setMetadata(metadata)`: settable fields
|
|
30
|
+
* and custom metadata; a custom key set to `null` is removed
|
|
31
|
+
* - `getDownloadURL(file)`: mints a token into
|
|
32
|
+
* `firebaseStorageDownloadTokens` when the file has none. The Node
|
|
33
|
+
* host returns its HTTP URL; elsewhere the URL is a `data:` URI
|
|
27
34
|
* - `file.delete()` — idempotent
|
|
28
35
|
* - `file.exists()` → `[boolean]`
|
|
29
36
|
* - `file.getSignedUrl(options)` → `['pyric-sandbox-storage://…']`
|
|
30
37
|
*
|
|
31
38
|
* **Deferred in the sandbox backend** (throws `"not implemented in
|
|
32
|
-
* pyric-admin/storage sandbox backend"`):
|
|
33
|
-
*
|
|
34
|
-
*
|
|
39
|
+
* pyric-admin/storage sandbox backend"`): resumable uploads, signed
|
|
40
|
+
* cookies, IAM policies, lifecycle rules, ACLs, copy/move,
|
|
41
|
+
* notifications.
|
|
35
42
|
*/
|
|
43
|
+
import { createWriteStream as createFileWriteStream, openAsBlob } from 'node:fs';
|
|
44
|
+
import { randomUUID } from 'node:crypto';
|
|
45
|
+
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
|
46
|
+
import { tmpdir } from 'node:os';
|
|
47
|
+
import { join } from 'node:path';
|
|
48
|
+
import { Readable, Writable } from 'node:stream';
|
|
36
49
|
import { isRemoteSandbox, } from 'pyric/sandbox';
|
|
50
|
+
import { fetchFromByteRoute, uploadOverByteRoute } from 'pyric/storage/internal';
|
|
37
51
|
import { ADMIN_APP_TARGET, getApp, } from '../app/index.js';
|
|
38
52
|
import { assertAdminAppActive } from '../app/lifecycle.js';
|
|
53
|
+
/** The custom metadata key holding a file's download tokens, comma-separated. */
|
|
54
|
+
const DOWNLOAD_TOKENS_KEY = 'firebaseStorageDownloadTokens';
|
|
55
|
+
/** Settable fields the sandbox keeps, besides custom metadata. */
|
|
56
|
+
const SETTABLE_FIELDS = ['contentType', 'cacheControl', 'contentDisposition', 'contentEncoding', 'contentLanguage'];
|
|
39
57
|
/**
|
|
40
58
|
* Get the {@link Storage} service for the given app.
|
|
41
59
|
*
|
|
@@ -64,13 +82,23 @@ export function getStorage(app) {
|
|
|
64
82
|
throw new TypeError('pyric-admin/storage: getStorage expected a PyricAdminApp from `initializeApp`; ' +
|
|
65
83
|
'received a value with no recognized ADMIN_APP_TARGET brand.');
|
|
66
84
|
}
|
|
85
|
+
/** Each arm's own download URL. */
|
|
86
|
+
const DOWNLOAD_URL = Symbol('pyric-admin/storage download URL');
|
|
67
87
|
/**
|
|
68
|
-
*
|
|
69
|
-
*
|
|
88
|
+
* Mirror of `firebase-admin/storage`'s `getDownloadURL(file)`. A file without
|
|
89
|
+
* a download token gets one in `firebaseStorageDownloadTokens`, as production
|
|
90
|
+
* mints it. On the Node host the URL is its HTTP byte route URL, which serves
|
|
91
|
+
* ranges and stops working when the token is removed. In process and on a
|
|
92
|
+
* SharedWorker host there is no HTTP origin, so the URL is a `data:` URI that
|
|
93
|
+
* carries the bytes.
|
|
70
94
|
*/
|
|
71
95
|
export async function getDownloadURL(file) {
|
|
72
|
-
const
|
|
73
|
-
|
|
96
|
+
const source = file;
|
|
97
|
+
const served = typeof source[DOWNLOAD_URL] === 'function';
|
|
98
|
+
if (!served) {
|
|
99
|
+
throw new Error('pyric-admin/storage: getDownloadURL expects a File from getStorage().bucket().file().');
|
|
100
|
+
}
|
|
101
|
+
return source[DOWNLOAD_URL]();
|
|
74
102
|
}
|
|
75
103
|
// ─── Sandbox path ───────────────────────────────────────────────────────
|
|
76
104
|
/**
|
|
@@ -79,6 +107,12 @@ export async function getDownloadURL(file) {
|
|
|
79
107
|
* surfaces don't see an unexpected bucket name change.
|
|
80
108
|
*/
|
|
81
109
|
const DEFAULT_SANDBOX_BUCKET = 'pyric-default';
|
|
110
|
+
let generationSequence = 0;
|
|
111
|
+
/** A generation in the form `@google-cloud/storage` reports: microseconds since the epoch. */
|
|
112
|
+
function nextGeneration() {
|
|
113
|
+
generationSequence = (generationSequence + 1) % 1000;
|
|
114
|
+
return `${Date.now()}${String(generationSequence).padStart(3, '0')}`;
|
|
115
|
+
}
|
|
82
116
|
/**
|
|
83
117
|
* State + reset-handler registry keyed on `Sandbox` so a single
|
|
84
118
|
* Sandbox shares its storage across every `getStorage` call. The
|
|
@@ -165,22 +199,97 @@ class SandboxFile {
|
|
|
165
199
|
throw new Error('not implemented in pyric-admin/storage sandbox backend: resumable uploads');
|
|
166
200
|
}
|
|
167
201
|
const bytes = toBytes(data);
|
|
168
|
-
const
|
|
169
|
-
const
|
|
202
|
+
const saved = patchOf(options.metadata ?? {}, { strict: false });
|
|
203
|
+
const contentType = options.contentType ?? saved.settable.contentType;
|
|
204
|
+
const now = new Date().toISOString();
|
|
205
|
+
const custom = {};
|
|
206
|
+
for (const [key, value] of Object.entries(saved.customMetadata))
|
|
207
|
+
if (value !== null)
|
|
208
|
+
custom[key] = value;
|
|
209
|
+
const hasTokens = typeof saved.downloadTokens === 'string';
|
|
210
|
+
if (hasTokens)
|
|
211
|
+
custom[DOWNLOAD_TOKENS_KEY] = saved.downloadTokens;
|
|
212
|
+
this.files.set(this.name, {
|
|
170
213
|
data: bytes,
|
|
171
|
-
|
|
172
|
-
|
|
214
|
+
settable: { ...saved.settable, ...(contentType !== undefined ? { contentType } : {}) },
|
|
215
|
+
custom,
|
|
216
|
+
generation: nextGeneration(),
|
|
217
|
+
metageneration: 1,
|
|
218
|
+
timeCreated: now,
|
|
219
|
+
updated: now,
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
async getMetadata() {
|
|
223
|
+
return [this.metadataOf(this.entry())];
|
|
224
|
+
}
|
|
225
|
+
async setMetadata(metadata) {
|
|
226
|
+
const entry = this.entry();
|
|
227
|
+
const patch = patchOf(metadata, { strict: true });
|
|
228
|
+
Object.assign(entry.settable, patch.settable);
|
|
229
|
+
for (const [key, value] of Object.entries(patch.customMetadata)) {
|
|
230
|
+
if (value === null)
|
|
231
|
+
delete entry.custom[key];
|
|
232
|
+
else
|
|
233
|
+
entry.custom[key] = value;
|
|
234
|
+
}
|
|
235
|
+
if (patch.downloadTokens === null)
|
|
236
|
+
delete entry.custom[DOWNLOAD_TOKENS_KEY];
|
|
237
|
+
else if (patch.downloadTokens !== undefined)
|
|
238
|
+
entry.custom[DOWNLOAD_TOKENS_KEY] = patch.downloadTokens;
|
|
239
|
+
this.touch(entry);
|
|
240
|
+
return [this.metadataOf(entry)];
|
|
241
|
+
}
|
|
242
|
+
async [DOWNLOAD_URL]() {
|
|
243
|
+
const entry = this.entry();
|
|
244
|
+
const tokenless = !entry.custom[DOWNLOAD_TOKENS_KEY];
|
|
245
|
+
if (tokenless) {
|
|
246
|
+
entry.custom[DOWNLOAD_TOKENS_KEY] = randomUUID();
|
|
247
|
+
this.touch(entry);
|
|
248
|
+
}
|
|
249
|
+
return dataUri(entry.data, entry.settable.contentType);
|
|
250
|
+
}
|
|
251
|
+
entry() {
|
|
252
|
+
const entry = this.files.get(this.name);
|
|
253
|
+
if (!entry)
|
|
254
|
+
throw new Error(`No such object: ${this.bucket.name}/${this.name}`);
|
|
255
|
+
return entry;
|
|
256
|
+
}
|
|
257
|
+
touch(entry) {
|
|
258
|
+
entry.metageneration += 1;
|
|
259
|
+
entry.updated = new Date().toISOString();
|
|
260
|
+
}
|
|
261
|
+
metadataOf(entry) {
|
|
262
|
+
const hasCustom = Object.keys(entry.custom).length > 0;
|
|
263
|
+
return {
|
|
264
|
+
name: this.name,
|
|
265
|
+
bucket: this.bucket.name,
|
|
266
|
+
generation: entry.generation,
|
|
267
|
+
metageneration: String(entry.metageneration),
|
|
268
|
+
size: String(entry.data.byteLength),
|
|
269
|
+
timeCreated: entry.timeCreated,
|
|
270
|
+
updated: entry.updated,
|
|
271
|
+
...entry.settable,
|
|
272
|
+
...(hasCustom ? { metadata: { ...entry.custom } } : {}),
|
|
173
273
|
};
|
|
174
|
-
this.files.set(this.name, entry);
|
|
175
274
|
}
|
|
176
|
-
async download(
|
|
275
|
+
async download(options = {}) {
|
|
177
276
|
const entry = this.files.get(this.name);
|
|
178
277
|
if (!entry) {
|
|
179
278
|
// Mirror the gcs/firebase-admin error message shape so consumer
|
|
180
279
|
// catch-blocks that string-match `No such object` keep working.
|
|
181
280
|
throw new Error(`No such object: ${this.bucket.name}/${this.name}`);
|
|
182
281
|
}
|
|
183
|
-
return [Buffer.from(entry.data)];
|
|
282
|
+
return [Buffer.from(byteRange(entry.data, options))];
|
|
283
|
+
}
|
|
284
|
+
createReadStream(options = {}) {
|
|
285
|
+
const read = async function* (file) {
|
|
286
|
+
const [bytes] = await file.download(options);
|
|
287
|
+
yield bytes;
|
|
288
|
+
};
|
|
289
|
+
return Readable.from(read(this), { objectMode: false });
|
|
290
|
+
}
|
|
291
|
+
createWriteStream(options = {}) {
|
|
292
|
+
return spooledWriteStream(async (spooled) => this.save(await readFile(spooled), options));
|
|
184
293
|
}
|
|
185
294
|
async delete() {
|
|
186
295
|
this.files.delete(this.name);
|
|
@@ -191,21 +300,12 @@ class SandboxFile {
|
|
|
191
300
|
async getSignedUrl(options) {
|
|
192
301
|
return [stubSignedUrl(this.bucket.name, this.name, options)];
|
|
193
302
|
}
|
|
194
|
-
// ─── Deferred surface (declared so TS callers see a clear error) ────
|
|
195
|
-
/** @deprecated Streaming writes are deferred — see module header. */
|
|
196
|
-
createWriteStream() {
|
|
197
|
-
throw new Error('not implemented in pyric-admin/storage sandbox backend: createWriteStream');
|
|
198
|
-
}
|
|
199
|
-
/** @deprecated Streaming reads are deferred — see module header. */
|
|
200
|
-
createReadStream() {
|
|
201
|
-
throw new Error('not implemented in pyric-admin/storage sandbox backend: createReadStream');
|
|
202
|
-
}
|
|
203
303
|
}
|
|
204
304
|
// ─── Remote sandbox arm (remote sandbox, slice 2) ───────────────────────
|
|
205
305
|
//
|
|
206
|
-
// The app's `Sandbox` is a Node-side handle onto
|
|
207
|
-
// SharedWorker sandbox. Every data operation relays over the
|
|
208
|
-
// worker channel with `actAs: { mode: 'admin' }` pinned — firebase-admin's
|
|
306
|
+
// The app's `Sandbox` is a Node-side handle onto a hosted sandbox or a
|
|
307
|
+
// browser tab's SharedWorker sandbox. Every data operation relays over the
|
|
308
|
+
// handle's worker channel with `actAs: { mode: 'admin' }` pinned — firebase-admin's
|
|
209
309
|
// rules-bypass semantics against the ONE object store the app + Studio +
|
|
210
310
|
// agents share (the host resolves the lens to `pyric/storage/internal`'s
|
|
211
311
|
// admin plane). There is deliberately NO local state here: a `WeakMap`
|
|
@@ -218,19 +318,21 @@ class SandboxFile {
|
|
|
218
318
|
// ("the data store is shared" — bucket names only round-trip in
|
|
219
319
|
// metadata), so `bucket('non-default')` throws instead of silently
|
|
220
320
|
// merging buckets. The default bucket name matches the local arm.
|
|
221
|
-
// -
|
|
222
|
-
//
|
|
321
|
+
// - a host with an HTTP byte route takes and serves bytes there; a
|
|
322
|
+
// SharedWorker host has none and takes frames, in 4 MiB parts past that.
|
|
223
323
|
// `getSignedUrl` does NOT relay: it stays the byte-identical local stub.
|
|
224
324
|
/** firebase-admin's rules-bypass lens, pinned on every relayed operation. */
|
|
225
325
|
const STORAGE_REMOTE_ADMIN_LENS = { mode: 'admin' };
|
|
226
326
|
/**
|
|
227
|
-
* Raw per-
|
|
228
|
-
* `@pyric/cli`' `
|
|
327
|
+
* Raw per-part byte cap for storage payloads sent as frames. MUST mirror
|
|
328
|
+
* `@pyric/cli`' `MAX_STORAGE_PART_BYTES` (serve/worker/protocol.ts) — the
|
|
229
329
|
* worker host enforces the same cap on its end. Inlined (like the RTDB
|
|
230
330
|
* push-id generator) because `pyric-admin` deliberately does not depend on
|
|
231
331
|
* `@pyric/cli`.
|
|
232
332
|
*/
|
|
233
|
-
const
|
|
333
|
+
const MAX_STORAGE_PART_BYTES = 4 * 1024 * 1024;
|
|
334
|
+
/** Maximum whole-object size supported by the sandbox backend (512 MiB). Matches MAX_STORAGE_OBJECT_BYTES. */
|
|
335
|
+
const MAX_STORAGE_OBJECT_BYTES = 512 * 1024 * 1024;
|
|
234
336
|
/** One remote `Storage` per remote handle (handles only — never data). */
|
|
235
337
|
const remoteStorageBySandbox = new WeakMap();
|
|
236
338
|
function getRemoteStorage(sandbox) {
|
|
@@ -283,38 +385,183 @@ class RemoteFile {
|
|
|
283
385
|
if (options.resumable === true) {
|
|
284
386
|
throw new Error('not implemented in pyric-admin/storage remote sandbox backend: resumable uploads');
|
|
285
387
|
}
|
|
286
|
-
const
|
|
287
|
-
if (
|
|
288
|
-
throw
|
|
388
|
+
const rawLength = typeof data === 'string' ? Buffer.byteLength(data, 'utf8') : data.byteLength;
|
|
389
|
+
if (rawLength > MAX_STORAGE_OBJECT_BYTES) {
|
|
390
|
+
throw quotaExceeded(rawLength, `save() payload for '${this.name}'`);
|
|
289
391
|
}
|
|
290
|
-
await this.
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
392
|
+
await this.upload(new Blob([toBytes(data)]), options);
|
|
393
|
+
}
|
|
394
|
+
async download(options = {}) {
|
|
395
|
+
const parts = [];
|
|
396
|
+
for await (const part of this.read(options))
|
|
397
|
+
parts.push(part);
|
|
398
|
+
return [Buffer.concat(parts)];
|
|
399
|
+
}
|
|
400
|
+
createReadStream(options = {}) {
|
|
401
|
+
return Readable.from(this.read(options), { objectMode: false });
|
|
402
|
+
}
|
|
403
|
+
createWriteStream(options = {}) {
|
|
404
|
+
return spooledWriteStream(async (spooled) => {
|
|
405
|
+
const data = await openAsBlob(spooled);
|
|
406
|
+
if (data.size > MAX_STORAGE_OBJECT_BYTES) {
|
|
407
|
+
throw quotaExceeded(data.size, `createWriteStream() payload for '${this.name}'`);
|
|
408
|
+
}
|
|
409
|
+
await this.upload(data, options);
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
/** Store `data` over the host's byte route, or as one frame when the host has none. */
|
|
413
|
+
async upload(data, options) {
|
|
414
|
+
const request = {
|
|
294
415
|
...(options.contentType !== undefined ? { contentType: options.contentType } : {}),
|
|
295
416
|
...(options.metadata !== undefined ? { metadata: options.metadata } : {}),
|
|
417
|
+
};
|
|
418
|
+
const route = await this.channel.byteRoute?.();
|
|
419
|
+
const routed = route !== undefined;
|
|
420
|
+
if (routed) {
|
|
421
|
+
await uploadOverByteRoute(this.channel, route, { path: this.name, data, ...request, actAs: STORAGE_REMOTE_ADMIN_LENS });
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
424
|
+
const bytes = new Uint8Array(await data.arrayBuffer());
|
|
425
|
+
if (bytes.byteLength <= MAX_STORAGE_PART_BYTES) {
|
|
426
|
+
await this.channel.op({
|
|
427
|
+
method: 'storage.putBytes',
|
|
428
|
+
path: this.name,
|
|
429
|
+
dataB64: base64Of(bytes),
|
|
430
|
+
...request,
|
|
431
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
432
|
+
});
|
|
433
|
+
return;
|
|
434
|
+
}
|
|
435
|
+
// A SharedWorker host takes a larger object in parts (ADR 0015).
|
|
436
|
+
const { uploadId } = (await this.channel.op({
|
|
437
|
+
method: 'storage.beginUpload',
|
|
438
|
+
path: this.name,
|
|
439
|
+
size: bytes.byteLength,
|
|
440
|
+
...request,
|
|
296
441
|
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
297
|
-
});
|
|
442
|
+
}));
|
|
443
|
+
try {
|
|
444
|
+
for (let offset = 0, index = 0; offset < bytes.byteLength; offset += MAX_STORAGE_PART_BYTES, index++) {
|
|
445
|
+
await this.channel.op({
|
|
446
|
+
method: 'storage.putPart',
|
|
447
|
+
uploadId,
|
|
448
|
+
partIndex: index,
|
|
449
|
+
dataB64: base64Of(bytes.subarray(offset, offset + MAX_STORAGE_PART_BYTES)),
|
|
450
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
await this.channel.op({ method: 'storage.finishUpload', uploadId, actAs: STORAGE_REMOTE_ADMIN_LENS });
|
|
454
|
+
}
|
|
455
|
+
catch (err) {
|
|
456
|
+
try {
|
|
457
|
+
await this.channel.op({ method: 'storage.abortUpload', uploadId, actAs: STORAGE_REMOTE_ADMIN_LENS });
|
|
458
|
+
}
|
|
459
|
+
catch {
|
|
460
|
+
// secondary abort best effort
|
|
461
|
+
}
|
|
462
|
+
throw err;
|
|
463
|
+
}
|
|
298
464
|
}
|
|
299
|
-
|
|
300
|
-
|
|
465
|
+
/** The object's bytes, or the inclusive range `start..end`, as they arrive. */
|
|
466
|
+
async *read(options) {
|
|
467
|
+
const metadata = await this.missingAsNoSuchObject(this.channel.op({
|
|
468
|
+
method: 'storage.getMetadata',
|
|
469
|
+
path: this.name,
|
|
470
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
471
|
+
}));
|
|
472
|
+
const route = await this.channel.byteRoute?.();
|
|
473
|
+
const routed = route !== undefined;
|
|
474
|
+
if (!routed) {
|
|
475
|
+
yield* this.readFrames(metadata, options);
|
|
476
|
+
return;
|
|
477
|
+
}
|
|
478
|
+
const response = await this.missingAsNoSuchObject(fetchFromByteRoute(route, {
|
|
479
|
+
bucket: metadata.bucket,
|
|
480
|
+
path: this.name,
|
|
481
|
+
start: options.start,
|
|
482
|
+
end: options.end,
|
|
483
|
+
}));
|
|
484
|
+
const body = response.body;
|
|
485
|
+
if (body === null)
|
|
486
|
+
return;
|
|
487
|
+
const reader = body.getReader();
|
|
301
488
|
try {
|
|
302
|
-
|
|
489
|
+
for (;;) {
|
|
490
|
+
const { done, value } = await reader.read();
|
|
491
|
+
if (done)
|
|
492
|
+
return;
|
|
493
|
+
yield value;
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
finally {
|
|
497
|
+
reader.releaseLock();
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
/** Read from a SharedWorker host, which sends bytes as frames: whole, or in ranged parts (ADR 0015). */
|
|
501
|
+
async *readFrames(metadata, options) {
|
|
502
|
+
if (metadata.size <= MAX_STORAGE_PART_BYTES) {
|
|
503
|
+
const wire = await this.missingAsNoSuchObject(this.channel.op({
|
|
504
|
+
method: 'storage.getBytes',
|
|
505
|
+
path: this.name,
|
|
506
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
507
|
+
}));
|
|
508
|
+
yield byteRange(Buffer.from(wire.dataB64, 'base64'), options);
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
const last = Math.min(metadata.size, options.end === undefined ? metadata.size : options.end + 1);
|
|
512
|
+
for (let offset = options.start ?? 0; offset < last; offset += MAX_STORAGE_PART_BYTES) {
|
|
513
|
+
const wire = await this.missingAsNoSuchObject(this.channel.op({
|
|
303
514
|
method: 'storage.getBytes',
|
|
304
515
|
path: this.name,
|
|
516
|
+
offset,
|
|
517
|
+
length: Math.min(MAX_STORAGE_PART_BYTES, last - offset),
|
|
518
|
+
expectedGeneration: metadata.generation,
|
|
305
519
|
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
306
520
|
}));
|
|
521
|
+
yield Buffer.from(wire.dataB64, 'base64');
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
async getMetadata() {
|
|
525
|
+
const full = await this.missingAsNoSuchObject(this.channel.op({
|
|
526
|
+
method: 'storage.getMetadata',
|
|
527
|
+
path: this.name,
|
|
528
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
529
|
+
}));
|
|
530
|
+
return [fileMetadataOf(full)];
|
|
531
|
+
}
|
|
532
|
+
async setMetadata(metadata) {
|
|
533
|
+
const full = await this.missingAsNoSuchObject(this.channel.op({
|
|
534
|
+
method: 'storage.setMetadata',
|
|
535
|
+
path: this.name,
|
|
536
|
+
patch: patchOf(metadata, { strict: true }),
|
|
537
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
538
|
+
}));
|
|
539
|
+
return [fileMetadataOf(full)];
|
|
540
|
+
}
|
|
541
|
+
/** The host mints the token; the Node host serves the URL, a SharedWorker host has nowhere to. */
|
|
542
|
+
async [DOWNLOAD_URL]() {
|
|
543
|
+
const { path } = await this.missingAsNoSuchObject(this.channel.op({
|
|
544
|
+
method: 'storage.getDownloadURL',
|
|
545
|
+
path: this.name,
|
|
546
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
547
|
+
}));
|
|
548
|
+
const route = await this.channel.byteRoute?.();
|
|
549
|
+
const routed = route !== undefined;
|
|
550
|
+
if (routed)
|
|
551
|
+
return new URL(path, route.baseUrl).href;
|
|
552
|
+
const [[metadata], [bytes]] = await Promise.all([this.getMetadata(), this.download()]);
|
|
553
|
+
return dataUri(bytes, metadata.contentType);
|
|
554
|
+
}
|
|
555
|
+
/** Mirror the gcs/firebase-admin (and local arm) `No such object` message for a missing file. */
|
|
556
|
+
async missingAsNoSuchObject(pending) {
|
|
557
|
+
try {
|
|
558
|
+
return await pending;
|
|
307
559
|
}
|
|
308
560
|
catch (err) {
|
|
309
|
-
if (isObjectNotFound(err))
|
|
310
|
-
// Mirror the gcs/firebase-admin (and local arm) message shape so
|
|
311
|
-
// consumer catch-blocks that string-match `No such object` work
|
|
312
|
-
// identically across arms.
|
|
561
|
+
if (isObjectNotFound(err))
|
|
313
562
|
throw new Error(`No such object: ${this.bucket.name}/${this.name}`);
|
|
314
|
-
}
|
|
315
563
|
throw err;
|
|
316
564
|
}
|
|
317
|
-
return [Buffer.from(wire.dataB64, 'base64')];
|
|
318
565
|
}
|
|
319
566
|
async delete() {
|
|
320
567
|
try {
|
|
@@ -353,31 +600,124 @@ class RemoteFile {
|
|
|
353
600
|
async getSignedUrl(options) {
|
|
354
601
|
return [stubSignedUrl(this.bucket.name, this.name, options)];
|
|
355
602
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
603
|
+
}
|
|
604
|
+
/** A host's metadata in `@google-cloud/storage`'s shape: the download tokens become the custom key production uses. */
|
|
605
|
+
function fileMetadataOf(host) {
|
|
606
|
+
const custom = { ...(host.customMetadata ?? {}) };
|
|
607
|
+
if (host.downloadTokens)
|
|
608
|
+
custom[DOWNLOAD_TOKENS_KEY] = host.downloadTokens;
|
|
609
|
+
const settable = {};
|
|
610
|
+
for (const field of SETTABLE_FIELDS) {
|
|
611
|
+
const value = host[field];
|
|
612
|
+
if (value !== undefined)
|
|
613
|
+
settable[field] = value;
|
|
364
614
|
}
|
|
615
|
+
const hasCustom = Object.keys(custom).length > 0;
|
|
616
|
+
return {
|
|
617
|
+
name: host.fullPath,
|
|
618
|
+
bucket: host.bucket,
|
|
619
|
+
generation: host.generation,
|
|
620
|
+
metageneration: host.metageneration,
|
|
621
|
+
size: String(host.size),
|
|
622
|
+
timeCreated: host.timeCreated,
|
|
623
|
+
updated: host.updated,
|
|
624
|
+
...settable,
|
|
625
|
+
...(host.md5Hash !== undefined ? { md5Hash: host.md5Hash } : {}),
|
|
626
|
+
...(hasCustom ? { metadata: custom } : {}),
|
|
627
|
+
};
|
|
365
628
|
}
|
|
366
629
|
/** Is this relayed error the worker's `storage/object-not-found`? */
|
|
367
630
|
function isObjectNotFound(err) {
|
|
368
631
|
return err?.code === 'storage/object-not-found';
|
|
369
632
|
}
|
|
370
|
-
/** Over-cap rejection (code `
|
|
371
|
-
|
|
372
|
-
function payloadTooLarge(sizeBytes, what) {
|
|
633
|
+
/** Over-cap rejection (code `storage/quota-exceeded`) — mirrors the worker host's message shape. */
|
|
634
|
+
function quotaExceeded(sizeBytes, what) {
|
|
373
635
|
const err = new Error(`pyric-admin/storage: ${what} is ${sizeBytes} bytes — over the ` +
|
|
374
|
-
`${
|
|
375
|
-
|
|
376
|
-
'split the object or keep it under the cap.');
|
|
377
|
-
err.code = 'payload-too-large';
|
|
636
|
+
`${MAX_STORAGE_OBJECT_BYTES / (1024 * 1024)} MiB maximum storage object cap (MAX_STORAGE_OBJECT_BYTES).`);
|
|
637
|
+
err.code = 'storage/quota-exceeded';
|
|
378
638
|
return err;
|
|
379
639
|
}
|
|
380
|
-
|
|
640
|
+
/**
|
|
641
|
+
* Split `@google-cloud/storage`-shaped metadata into a {@link MetadataPatch}.
|
|
642
|
+
* `setMetadata` is strict and refuses fields the sandbox does not keep;
|
|
643
|
+
* `save` has always stored what it was given and ignores them.
|
|
644
|
+
*/
|
|
645
|
+
function patchOf(metadata, { strict }) {
|
|
646
|
+
const source = metadata;
|
|
647
|
+
const unsupported = Object.keys(source).filter(key => key !== 'metadata' && !SETTABLE_FIELDS.includes(key));
|
|
648
|
+
const refused = strict && unsupported.length > 0;
|
|
649
|
+
if (refused) {
|
|
650
|
+
throw new Error(`not implemented in pyric-admin/storage sandbox backend: setMetadata of ${unsupported.join(', ')}. ` +
|
|
651
|
+
`The sandbox keeps ${SETTABLE_FIELDS.join(', ')}, and custom metadata under \`metadata\`.`);
|
|
652
|
+
}
|
|
653
|
+
const patch = { settable: {}, customMetadata: {} };
|
|
654
|
+
for (const field of SETTABLE_FIELDS) {
|
|
655
|
+
const value = source[field];
|
|
656
|
+
if (typeof value === 'string')
|
|
657
|
+
patch.settable[field] = value;
|
|
658
|
+
}
|
|
659
|
+
const custom = source.metadata;
|
|
660
|
+
const hasCustom = custom !== null && typeof custom === 'object';
|
|
661
|
+
if (!hasCustom)
|
|
662
|
+
return patch;
|
|
663
|
+
for (const [key, value] of Object.entries(custom)) {
|
|
664
|
+
if (value === undefined)
|
|
665
|
+
continue;
|
|
666
|
+
const next = value === null ? null : String(value);
|
|
667
|
+
if (key === DOWNLOAD_TOKENS_KEY)
|
|
668
|
+
patch.downloadTokens = next;
|
|
669
|
+
else
|
|
670
|
+
patch.customMetadata[key] = next;
|
|
671
|
+
}
|
|
672
|
+
return patch;
|
|
673
|
+
}
|
|
674
|
+
/** A `data:` URI carrying `bytes`, the download URL where no host serves them over HTTP. */
|
|
675
|
+
function dataUri(bytes, contentType) {
|
|
676
|
+
return `data:${contentType ?? 'application/octet-stream'};base64,${base64Of(bytes)}`;
|
|
677
|
+
}
|
|
678
|
+
/** `bytes` as base64, for a frame. */
|
|
679
|
+
function base64Of(bytes) {
|
|
680
|
+
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('base64');
|
|
681
|
+
}
|
|
682
|
+
/** The inclusive range `start..end` of `bytes`, as `@google-cloud/storage` reads it. */
|
|
683
|
+
function byteRange(bytes, options) {
|
|
684
|
+
const end = options.end === undefined ? undefined : options.end + 1;
|
|
685
|
+
return bytes.subarray(options.start ?? 0, end);
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* A writable stream that spools its bytes to a temporary file and, once it
|
|
689
|
+
* finishes, hands that file to `commit`. The file is removed either way.
|
|
690
|
+
*/
|
|
691
|
+
function spooledWriteStream(commit) {
|
|
692
|
+
let directory;
|
|
693
|
+
let sink;
|
|
694
|
+
return new Writable({
|
|
695
|
+
construct(callback) {
|
|
696
|
+
mkdtemp(join(tmpdir(), 'pyric-admin-upload-')).then(created => {
|
|
697
|
+
directory = created;
|
|
698
|
+
sink = createFileWriteStream(join(created, 'object'));
|
|
699
|
+
callback();
|
|
700
|
+
}, callback);
|
|
701
|
+
},
|
|
702
|
+
write(chunk, _encoding, callback) {
|
|
703
|
+
sink.write(chunk, callback);
|
|
704
|
+
},
|
|
705
|
+
final(callback) {
|
|
706
|
+
sink.end(() => {
|
|
707
|
+
commit(join(directory, 'object')).then(() => callback(), callback);
|
|
708
|
+
});
|
|
709
|
+
},
|
|
710
|
+
destroy(error, callback) {
|
|
711
|
+
sink?.destroy();
|
|
712
|
+
const created = directory;
|
|
713
|
+
if (created === undefined) {
|
|
714
|
+
callback(error);
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
rm(created, { recursive: true, force: true }).finally(() => callback(error));
|
|
718
|
+
},
|
|
719
|
+
});
|
|
720
|
+
}
|
|
381
721
|
/**
|
|
382
722
|
* The deterministic sandbox signed-URL stub, shared by the local and remote
|
|
383
723
|
* arms so their output is byte-identical (the URL is never served — it's a
|