pyric-admin 0.1.0-alpha.21 → 0.1.0-alpha.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +12 -4
  2. package/dist/app-check/index.d.ts +3 -0
  3. package/dist/app-check/index.d.ts.map +1 -0
  4. package/dist/app-check/index.js +11 -0
  5. package/dist/app-check/index.js.map +1 -0
  6. package/dist/data-connect/index.d.ts +3 -0
  7. package/dist/data-connect/index.d.ts.map +1 -0
  8. package/dist/data-connect/index.js +11 -0
  9. package/dist/data-connect/index.js.map +1 -0
  10. package/dist/deferred.d.ts +21 -0
  11. package/dist/deferred.d.ts.map +1 -0
  12. package/dist/deferred.js +29 -0
  13. package/dist/deferred.js.map +1 -0
  14. package/dist/eventarc/index.d.ts +3 -0
  15. package/dist/eventarc/index.d.ts.map +1 -0
  16. package/dist/eventarc/index.js +11 -0
  17. package/dist/eventarc/index.js.map +1 -0
  18. package/dist/extensions/index.d.ts +3 -0
  19. package/dist/extensions/index.d.ts.map +1 -0
  20. package/dist/extensions/index.js +11 -0
  21. package/dist/extensions/index.js.map +1 -0
  22. package/dist/functions/index.d.ts +3 -0
  23. package/dist/functions/index.d.ts.map +1 -0
  24. package/dist/functions/index.js +11 -0
  25. package/dist/functions/index.js.map +1 -0
  26. package/dist/installations/index.d.ts +3 -0
  27. package/dist/installations/index.d.ts.map +1 -0
  28. package/dist/installations/index.js +11 -0
  29. package/dist/installations/index.js.map +1 -0
  30. package/dist/instance-id/index.d.ts +3 -0
  31. package/dist/instance-id/index.d.ts.map +1 -0
  32. package/dist/instance-id/index.js +11 -0
  33. package/dist/instance-id/index.js.map +1 -0
  34. package/dist/machine-learning/index.d.ts +3 -0
  35. package/dist/machine-learning/index.d.ts.map +1 -0
  36. package/dist/machine-learning/index.js +11 -0
  37. package/dist/machine-learning/index.js.map +1 -0
  38. package/dist/messaging/index.d.ts +0 -1
  39. package/dist/messaging/index.d.ts.map +1 -1
  40. package/dist/messaging/index.js +7 -8
  41. package/dist/messaging/index.js.map +1 -1
  42. package/dist/phone-number-verification/index.d.ts +3 -0
  43. package/dist/phone-number-verification/index.d.ts.map +1 -0
  44. package/dist/phone-number-verification/index.js +11 -0
  45. package/dist/phone-number-verification/index.js.map +1 -0
  46. package/dist/project-management/index.d.ts +3 -0
  47. package/dist/project-management/index.d.ts.map +1 -0
  48. package/dist/project-management/index.js +11 -0
  49. package/dist/project-management/index.js.map +1 -0
  50. package/dist/remote-config/index.d.ts +3 -0
  51. package/dist/remote-config/index.d.ts.map +1 -0
  52. package/dist/remote-config/index.js +11 -0
  53. package/dist/remote-config/index.js.map +1 -0
  54. package/dist/security-rules/index.d.ts +3 -0
  55. package/dist/security-rules/index.d.ts.map +1 -0
  56. package/dist/security-rules/index.js +11 -0
  57. package/dist/security-rules/index.js.map +1 -0
  58. package/dist/storage/index.d.ts +88 -16
  59. package/dist/storage/index.d.ts.map +1 -1
  60. package/dist/storage/index.js +409 -69
  61. package/dist/storage/index.js.map +1 -1
  62. package/package.json +50 -2
  63. package/src/app-check/index.ts +14 -0
  64. package/src/data-connect/index.ts +14 -0
  65. package/src/deferred.ts +33 -0
  66. package/src/eventarc/index.ts +14 -0
  67. package/src/extensions/index.ts +14 -0
  68. package/src/functions/index.ts +14 -0
  69. package/src/installations/index.ts +15 -0
  70. package/src/instance-id/index.ts +14 -0
  71. package/src/machine-learning/index.ts +14 -0
  72. package/src/messaging/index.ts +2 -9
  73. package/src/phone-number-verification/index.ts +14 -0
  74. package/src/project-management/index.ts +15 -0
  75. package/src/remote-config/index.ts +15 -0
  76. package/src/security-rules/index.ts +14 -0
  77. package/src/storage/index.ts +520 -89
@@ -7,10 +7,10 @@
7
7
  *
8
8
  * - **Remote sandbox path** — a handle branded by `@pyric/cli`'
9
9
  * `connectRemoteSandbox()`/`remoteSandbox()` relays every data
10
- * operation over the bridge to the browser-hosted SharedWorker's
11
- * object store (admin lens pinned — rules bypass). Single bucket;
12
- * 8 MiB per-op byte cap; `getSignedUrl` stays the local stub. See
13
- * the remote arm section below.
10
+ * operation to the sandbox's host (admin lens pinned — rules bypass).
11
+ * Bytes move over the host's HTTP byte route when it has one, and as
12
+ * frames when it does not (a browser tab's SharedWorker). Single bucket;
13
+ * `getSignedUrl` stays the local stub. See the remote arm section below.
14
14
  *
15
15
  * - **Sandbox path** — returns an in-process {@link Storage} backed
16
16
  * by an in-memory `Map<bucketName, Map<path, FileEntry>>`. State
@@ -23,23 +23,37 @@
23
23
  * - `storage.bucket(name?)` → {@link Bucket}-shaped handle
24
24
  * - `bucket.file(path)` → {@link File}-shaped handle
25
25
  * - `file.save(data, options?)` — `Buffer | string | Uint8Array`
26
- * - `file.download(options?)` → `[Buffer]`
26
+ * - `file.download(options?)` → `[Buffer]`, whole or `start..end`
27
+ * - `file.createReadStream(options?)`, whole or `start..end`
28
+ * - `file.createWriteStream(options?)`
29
+ * - `file.getMetadata()` / `file.setMetadata(metadata)`: settable fields
30
+ * and custom metadata; a custom key set to `null` is removed
31
+ * - `getDownloadURL(file)`: mints a token into
32
+ * `firebaseStorageDownloadTokens` when the file has none. The Node
33
+ * host returns its HTTP URL; elsewhere the URL is a `data:` URI
27
34
  * - `file.delete()` — idempotent
28
35
  * - `file.exists()` → `[boolean]`
29
36
  * - `file.getSignedUrl(options)` → `['pyric-sandbox-storage://…']`
30
37
  *
31
38
  * **Deferred in the sandbox backend** (throws `"not implemented in
32
- * pyric-admin/storage sandbox backend"`): streaming uploads
33
- * (`createWriteStream`), resumable uploads, signed cookies, IAM
34
- * policies, lifecycle rules, ACLs, copy/move, notifications.
39
+ * pyric-admin/storage sandbox backend"`): resumable uploads, signed
40
+ * cookies, IAM policies, lifecycle rules, ACLs, copy/move,
41
+ * notifications.
35
42
  */
36
43
 
44
+ import { createWriteStream as createFileWriteStream, openAsBlob, type WriteStream } from 'node:fs';
45
+ import { randomUUID } from 'node:crypto';
46
+ import { mkdtemp, readFile, rm } from 'node:fs/promises';
47
+ import { tmpdir } from 'node:os';
48
+ import { join } from 'node:path';
49
+ import { Readable, Writable } from 'node:stream';
37
50
  import {
38
51
  isRemoteSandbox,
39
52
  type RemoteSandbox,
40
53
  type RemoteSandboxChannel,
41
54
  type Sandbox,
42
55
  } from 'pyric/sandbox';
56
+ import { fetchFromByteRoute, uploadOverByteRoute } from 'pyric/storage/internal';
43
57
 
44
58
  import {
45
59
  ADMIN_APP_TARGET,
@@ -83,9 +97,9 @@ export interface Bucket {
83
97
  * / getSignedUrl, etc.) so common consumer code retains the familiar shape.
84
98
  *
85
99
  * The sandbox backend implements the methods documented here. Any
86
- * other `File` method from `@google-cloud/storage` (`createWriteStream`,
87
- * `createReadStream`, `copy`, `move`, `setMetadata` beyond the basic
88
- * `save` options, etc.) throws on the sandbox path — see module header.
100
+ * other `File` method from `@google-cloud/storage` (`copy`, `move`,
101
+ * `setMetadata` beyond the basic `save` options, etc.) is not implemented
102
+ * on the sandbox path — see module header.
89
103
  */
90
104
  export interface File {
91
105
  /** Name (path) of the file within its bucket. */
@@ -101,11 +115,30 @@ export interface File {
101
115
  */
102
116
  save(data: Buffer | string | Uint8Array, options?: SaveOptions): Promise<void>;
103
117
  /**
104
- * Read the file's bytes. Returns a `[Buffer]` tuple to mirror
105
- * `@google-cloud/storage`'s `File.download` (which returns
106
- * `[Buffer, ...]`). Throws if the file does not exist.
118
+ * Read the file's bytes, or the inclusive range `start..end` of them.
119
+ * Returns a `[Buffer]` tuple to mirror `@google-cloud/storage`'s
120
+ * `File.download` (which returns `[Buffer, ...]`). Throws if the file
121
+ * does not exist.
107
122
  */
108
123
  download(options?: DownloadOptions): Promise<[Buffer]>;
124
+ /**
125
+ * A readable stream of the file's bytes, or of the inclusive range
126
+ * `start..end`. A missing file surfaces as the stream's error.
127
+ */
128
+ createReadStream(options?: CreateReadStreamOptions): Readable;
129
+ /**
130
+ * A writable stream whose bytes replace the file's content once the
131
+ * stream finishes, as {@link File.save} does.
132
+ */
133
+ createWriteStream(options?: CreateWriteStreamOptions): Writable;
134
+ /** The file's metadata, as a `[metadata]` tuple. Throws if the file does not exist. */
135
+ getMetadata(): Promise<[FileMetadata]>;
136
+ /**
137
+ * Change the file's metadata: the named settable fields, and the custom
138
+ * keys under `metadata`, where `null` removes a key and the others stay.
139
+ * Removing `firebaseStorageDownloadTokens` revokes the file's download URL.
140
+ */
141
+ setMetadata(metadata: FileMetadataUpdate): Promise<[FileMetadata]>;
109
142
  /**
110
143
  * Remove the file from its bucket. Idempotent — deleting a missing
111
144
  * file is a no-op (matches `@google-cloud/storage`'s
@@ -148,10 +181,58 @@ export interface SaveOptions {
148
181
 
149
182
  /** Options bag for {@link File.download}. Subset of `@google-cloud/storage`'s `DownloadOptions`. */
150
183
  export interface DownloadOptions {
184
+ /** First byte to read, inclusive. */
185
+ start?: number;
186
+ /** Last byte to read, inclusive. */
187
+ end?: number;
151
188
  /** The sandbox accepts but ignores `validation`. */
152
189
  validation?: 'md5' | 'crc32c' | boolean;
153
190
  }
154
191
 
192
+ /** Options bag for {@link File.createReadStream}. Subset of `@google-cloud/storage`'s `CreateReadStreamOptions`. */
193
+ export type CreateReadStreamOptions = DownloadOptions;
194
+
195
+ /** Options bag for {@link File.createWriteStream}. Subset of `@google-cloud/storage`'s `CreateWriteStreamOptions`. */
196
+ export interface CreateWriteStreamOptions {
197
+ /** Stored alongside the file, as {@link SaveOptions.metadata} is. */
198
+ metadata?: Record<string, unknown>;
199
+ /** Content type stored on the file. */
200
+ contentType?: string;
201
+ }
202
+
203
+ /** The custom metadata key holding a file's download tokens, comma-separated. */
204
+ const DOWNLOAD_TOKENS_KEY = 'firebaseStorageDownloadTokens';
205
+
206
+ /** Settable fields the sandbox keeps, besides custom metadata. */
207
+ const SETTABLE_FIELDS = ['contentType', 'cacheControl', 'contentDisposition', 'contentEncoding', 'contentLanguage'] as const;
208
+ type SettableField = (typeof SETTABLE_FIELDS)[number];
209
+
210
+ /** A file's metadata. Subset of `@google-cloud/storage`'s `FileMetadata`. */
211
+ export interface FileMetadata {
212
+ name: string;
213
+ bucket: string;
214
+ generation: string;
215
+ metageneration: string;
216
+ /** Size in bytes, as a decimal string. */
217
+ size: string;
218
+ timeCreated: string;
219
+ updated: string;
220
+ contentType?: string;
221
+ cacheControl?: string;
222
+ contentDisposition?: string;
223
+ contentEncoding?: string;
224
+ contentLanguage?: string;
225
+ md5Hash?: string;
226
+ /** Custom metadata, including `firebaseStorageDownloadTokens` once the file has a download URL. Absent when there is none. */
227
+ metadata?: Record<string, string>;
228
+ }
229
+
230
+ /** What {@link File.setMetadata} takes. Subset of `@google-cloud/storage`'s `FileMetadata`. */
231
+ export type FileMetadataUpdate = Partial<Pick<FileMetadata, SettableField>> & {
232
+ /** Custom keys to set; `null` removes a key. Values are stored as strings. */
233
+ metadata?: Record<string, string | number | boolean | null>;
234
+ };
235
+
155
236
  /** Options bag for {@link File.getSignedUrl}. Mirrors `@google-cloud/storage`'s shape. */
156
237
  export interface GetSignedUrlOptions {
157
238
  /** `'read' | 'write' | 'delete' | 'resumable'`. Sandbox stamps it into the URL only as a hint. */
@@ -204,13 +285,28 @@ export function getStorage(app?: StorageApp): Storage {
204
285
  );
205
286
  }
206
287
 
288
+ /** Each arm's own download URL. */
289
+ const DOWNLOAD_URL = Symbol('pyric-admin/storage download URL');
290
+
291
+ interface DownloadUrlSource {
292
+ [DOWNLOAD_URL](): Promise<string>;
293
+ }
294
+
207
295
  /**
208
- * Link-compatible mirror of `firebase-admin/storage`'s `getDownloadURL(file)`.
209
- * Returns a deterministic sandbox storage stub URL from `file.getSignedUrl()`.
296
+ * Mirror of `firebase-admin/storage`'s `getDownloadURL(file)`. A file without
297
+ * a download token gets one in `firebaseStorageDownloadTokens`, as production
298
+ * mints it. On the Node host the URL is its HTTP byte route URL, which serves
299
+ * ranges and stops working when the token is removed. In process and on a
300
+ * SharedWorker host there is no HTTP origin, so the URL is a `data:` URI that
301
+ * carries the bytes.
210
302
  */
211
303
  export async function getDownloadURL(file: File): Promise<string> {
212
- const [url] = await file.getSignedUrl({ action: 'read', expires: '2099-01-01' });
213
- return url;
304
+ const source = file as File & Partial<DownloadUrlSource>;
305
+ const served = typeof source[DOWNLOAD_URL] === 'function';
306
+ if (!served) {
307
+ throw new Error('pyric-admin/storage: getDownloadURL expects a File from getStorage().bucket().file().');
308
+ }
309
+ return source[DOWNLOAD_URL]!();
214
310
  }
215
311
 
216
312
  // ─── Sandbox path ───────────────────────────────────────────────────────
@@ -222,11 +318,23 @@ export async function getDownloadURL(file: File): Promise<string> {
222
318
  */
223
319
  const DEFAULT_SANDBOX_BUCKET = 'pyric-default';
224
320
 
225
- /** A single file's bytes + opaque metadata in the in-memory store. */
321
+ /** A single file's bytes and metadata in the in-memory store. */
226
322
  interface FileEntry {
227
323
  data: Uint8Array;
228
- metadata: Record<string, unknown>;
229
- contentType?: string;
324
+ settable: Partial<Record<SettableField, string>>;
325
+ custom: Record<string, string>;
326
+ generation: string;
327
+ metageneration: number;
328
+ timeCreated: string;
329
+ updated: string;
330
+ }
331
+
332
+ let generationSequence = 0;
333
+
334
+ /** A generation in the form `@google-cloud/storage` reports: microseconds since the epoch. */
335
+ function nextGeneration(): string {
336
+ generationSequence = (generationSequence + 1) % 1000;
337
+ return `${Date.now()}${String(generationSequence).padStart(3, '0')}`;
230
338
  }
231
339
 
232
340
  /** Per-sandbox state: bucket name → (file path → entry). */
@@ -323,16 +431,79 @@ class SandboxFile implements File {
323
431
  );
324
432
  }
325
433
  const bytes = toBytes(data);
326
- const metadata = options.metadata ?? {};
327
- const entry: FileEntry = {
434
+ const saved = patchOf(options.metadata ?? {}, { strict: false });
435
+ const contentType = options.contentType ?? saved.settable.contentType;
436
+ const now = new Date().toISOString();
437
+ const custom: Record<string, string> = {};
438
+ for (const [key, value] of Object.entries(saved.customMetadata)) if (value !== null) custom[key] = value;
439
+ const hasTokens = typeof saved.downloadTokens === 'string';
440
+ if (hasTokens) custom[DOWNLOAD_TOKENS_KEY] = saved.downloadTokens!;
441
+ this.files.set(this.name, {
328
442
  data: bytes,
329
- metadata,
330
- ...(options.contentType !== undefined ? { contentType: options.contentType } : {}),
443
+ settable: { ...saved.settable, ...(contentType !== undefined ? { contentType } : {}) },
444
+ custom,
445
+ generation: nextGeneration(),
446
+ metageneration: 1,
447
+ timeCreated: now,
448
+ updated: now,
449
+ });
450
+ }
451
+
452
+ async getMetadata(): Promise<[FileMetadata]> {
453
+ return [this.metadataOf(this.entry())];
454
+ }
455
+
456
+ async setMetadata(metadata: FileMetadataUpdate): Promise<[FileMetadata]> {
457
+ const entry = this.entry();
458
+ const patch = patchOf(metadata, { strict: true });
459
+ Object.assign(entry.settable, patch.settable);
460
+ for (const [key, value] of Object.entries(patch.customMetadata)) {
461
+ if (value === null) delete entry.custom[key];
462
+ else entry.custom[key] = value;
463
+ }
464
+ if (patch.downloadTokens === null) delete entry.custom[DOWNLOAD_TOKENS_KEY];
465
+ else if (patch.downloadTokens !== undefined) entry.custom[DOWNLOAD_TOKENS_KEY] = patch.downloadTokens;
466
+ this.touch(entry);
467
+ return [this.metadataOf(entry)];
468
+ }
469
+
470
+ async [DOWNLOAD_URL](): Promise<string> {
471
+ const entry = this.entry();
472
+ const tokenless = !entry.custom[DOWNLOAD_TOKENS_KEY];
473
+ if (tokenless) {
474
+ entry.custom[DOWNLOAD_TOKENS_KEY] = randomUUID();
475
+ this.touch(entry);
476
+ }
477
+ return dataUri(entry.data, entry.settable.contentType);
478
+ }
479
+
480
+ private entry(): FileEntry {
481
+ const entry = this.files.get(this.name);
482
+ if (!entry) throw new Error(`No such object: ${this.bucket.name}/${this.name}`);
483
+ return entry;
484
+ }
485
+
486
+ private touch(entry: FileEntry): void {
487
+ entry.metageneration += 1;
488
+ entry.updated = new Date().toISOString();
489
+ }
490
+
491
+ private metadataOf(entry: FileEntry): FileMetadata {
492
+ const hasCustom = Object.keys(entry.custom).length > 0;
493
+ return {
494
+ name: this.name,
495
+ bucket: this.bucket.name,
496
+ generation: entry.generation,
497
+ metageneration: String(entry.metageneration),
498
+ size: String(entry.data.byteLength),
499
+ timeCreated: entry.timeCreated,
500
+ updated: entry.updated,
501
+ ...entry.settable,
502
+ ...(hasCustom ? { metadata: { ...entry.custom } } : {}),
331
503
  };
332
- this.files.set(this.name, entry);
333
504
  }
334
505
 
335
- async download(_options: DownloadOptions = {}): Promise<[Buffer]> {
506
+ async download(options: DownloadOptions = {}): Promise<[Buffer]> {
336
507
  const entry = this.files.get(this.name);
337
508
  if (!entry) {
338
509
  // Mirror the gcs/firebase-admin error message shape so consumer
@@ -341,7 +512,19 @@ class SandboxFile implements File {
341
512
  `No such object: ${this.bucket.name}/${this.name}`,
342
513
  );
343
514
  }
344
- return [Buffer.from(entry.data)];
515
+ return [Buffer.from(byteRange(entry.data, options))];
516
+ }
517
+
518
+ createReadStream(options: CreateReadStreamOptions = {}): Readable {
519
+ const read = async function* (file: SandboxFile): AsyncGenerator<Uint8Array> {
520
+ const [bytes] = await file.download(options);
521
+ yield bytes;
522
+ };
523
+ return Readable.from(read(this), { objectMode: false });
524
+ }
525
+
526
+ createWriteStream(options: CreateWriteStreamOptions = {}): Writable {
527
+ return spooledWriteStream(async spooled => this.save(await readFile(spooled), options));
345
528
  }
346
529
 
347
530
  async delete(): Promise<void> {
@@ -356,28 +539,13 @@ class SandboxFile implements File {
356
539
  return [stubSignedUrl(this.bucket.name, this.name, options)];
357
540
  }
358
541
 
359
- // ─── Deferred surface (declared so TS callers see a clear error) ────
360
-
361
- /** @deprecated Streaming writes are deferred — see module header. */
362
- createWriteStream(): never {
363
- throw new Error(
364
- 'not implemented in pyric-admin/storage sandbox backend: createWriteStream',
365
- );
366
- }
367
-
368
- /** @deprecated Streaming reads are deferred — see module header. */
369
- createReadStream(): never {
370
- throw new Error(
371
- 'not implemented in pyric-admin/storage sandbox backend: createReadStream',
372
- );
373
- }
374
542
  }
375
543
 
376
544
  // ─── Remote sandbox arm (remote sandbox, slice 2) ───────────────────────
377
545
  //
378
- // The app's `Sandbox` is a Node-side handle onto the browser-hosted
379
- // SharedWorker sandbox. Every data operation relays over the handle's
380
- // worker channel with `actAs: { mode: 'admin' }` pinned — firebase-admin's
546
+ // The app's `Sandbox` is a Node-side handle onto a hosted sandbox or a
547
+ // browser tab's SharedWorker sandbox. Every data operation relays over the
548
+ // handle's worker channel with `actAs: { mode: 'admin' }` pinned — firebase-admin's
381
549
  // rules-bypass semantics against the ONE object store the app + Studio +
382
550
  // agents share (the host resolves the lens to `pyric/storage/internal`'s
383
551
  // admin plane). There is deliberately NO local state here: a `WeakMap`
@@ -390,21 +558,24 @@ class SandboxFile implements File {
390
558
  // ("the data store is shared" — bucket names only round-trip in
391
559
  // metadata), so `bucket('non-default')` throws instead of silently
392
560
  // merging buckets. The default bucket name matches the local arm.
393
- // - byte payloads are capped at 8 MiB per op (whole-object buffering
394
- // over four relay hops; streaming stays unsupported on both sandbox arms).
561
+ // - a host with an HTTP byte route takes and serves bytes there; a
562
+ // SharedWorker host has none and takes frames, in 4 MiB parts past that.
395
563
  // `getSignedUrl` does NOT relay: it stays the byte-identical local stub.
396
564
 
397
565
  /** firebase-admin's rules-bypass lens, pinned on every relayed operation. */
398
566
  const STORAGE_REMOTE_ADMIN_LENS = { mode: 'admin' } as const;
399
567
 
400
568
  /**
401
- * Raw per-op byte cap for relayed storage payloads. MUST mirror
402
- * `@pyric/cli`' `MAX_STORAGE_OP_BYTES` (serve/worker/protocol.ts) — the
569
+ * Raw per-part byte cap for storage payloads sent as frames. MUST mirror
570
+ * `@pyric/cli`' `MAX_STORAGE_PART_BYTES` (serve/worker/protocol.ts) — the
403
571
  * worker host enforces the same cap on its end. Inlined (like the RTDB
404
572
  * push-id generator) because `pyric-admin` deliberately does not depend on
405
573
  * `@pyric/cli`.
406
574
  */
407
- const MAX_REMOTE_STORAGE_OP_BYTES = 8 * 1024 * 1024;
575
+ const MAX_STORAGE_PART_BYTES = 4 * 1024 * 1024;
576
+
577
+ /** Maximum whole-object size supported by the sandbox backend (512 MiB). Matches MAX_STORAGE_OBJECT_BYTES. */
578
+ const MAX_STORAGE_OBJECT_BYTES = 512 * 1024 * 1024;
408
579
 
409
580
  /** One remote `Storage` per remote handle (handles only — never data). */
410
581
  const remoteStorageBySandbox = new WeakMap<Sandbox, Storage>();
@@ -471,38 +642,184 @@ class RemoteFile implements File {
471
642
  'not implemented in pyric-admin/storage remote sandbox backend: resumable uploads',
472
643
  );
473
644
  }
474
- const bytes = toBytes(data);
475
- if (bytes.byteLength > MAX_REMOTE_STORAGE_OP_BYTES) {
476
- throw payloadTooLarge(bytes.byteLength, `save() payload for '${this.name}'`);
645
+ const rawLength = typeof data === 'string' ? Buffer.byteLength(data, 'utf8') : data.byteLength;
646
+ if (rawLength > MAX_STORAGE_OBJECT_BYTES) {
647
+ throw quotaExceeded(rawLength, `save() payload for '${this.name}'`);
477
648
  }
478
- await this.channel.op({
479
- method: 'storage.putBytes',
480
- path: this.name,
481
- dataB64: Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('base64'),
649
+ await this.upload(new Blob([toBytes(data) as Uint8Array<ArrayBuffer>]), options);
650
+ }
651
+
652
+ async download(options: DownloadOptions = {}): Promise<[Buffer]> {
653
+ const parts: Uint8Array[] = [];
654
+ for await (const part of this.read(options)) parts.push(part);
655
+ return [Buffer.concat(parts)];
656
+ }
657
+
658
+ createReadStream(options: CreateReadStreamOptions = {}): Readable {
659
+ return Readable.from(this.read(options), { objectMode: false });
660
+ }
661
+
662
+ createWriteStream(options: CreateWriteStreamOptions = {}): Writable {
663
+ return spooledWriteStream(async spooled => {
664
+ const data = await openAsBlob(spooled);
665
+ if (data.size > MAX_STORAGE_OBJECT_BYTES) {
666
+ throw quotaExceeded(data.size, `createWriteStream() payload for '${this.name}'`);
667
+ }
668
+ await this.upload(data, options);
669
+ });
670
+ }
671
+
672
+ /** Store `data` over the host's byte route, or as one frame when the host has none. */
673
+ private async upload(data: Blob, options: CreateWriteStreamOptions): Promise<void> {
674
+ const request = {
482
675
  ...(options.contentType !== undefined ? { contentType: options.contentType } : {}),
483
676
  ...(options.metadata !== undefined ? { metadata: options.metadata } : {}),
677
+ };
678
+ const route = await this.channel.byteRoute?.();
679
+ const routed = route !== undefined;
680
+ if (routed) {
681
+ await uploadOverByteRoute(this.channel, route, { path: this.name, data, ...request, actAs: STORAGE_REMOTE_ADMIN_LENS });
682
+ return;
683
+ }
684
+ const bytes = new Uint8Array(await data.arrayBuffer());
685
+ if (bytes.byteLength <= MAX_STORAGE_PART_BYTES) {
686
+ await this.channel.op({
687
+ method: 'storage.putBytes',
688
+ path: this.name,
689
+ dataB64: base64Of(bytes),
690
+ ...request,
691
+ actAs: STORAGE_REMOTE_ADMIN_LENS,
692
+ });
693
+ return;
694
+ }
695
+ // A SharedWorker host takes a larger object in parts (ADR 0015).
696
+ const { uploadId } = (await this.channel.op({
697
+ method: 'storage.beginUpload',
698
+ path: this.name,
699
+ size: bytes.byteLength,
700
+ ...request,
484
701
  actAs: STORAGE_REMOTE_ADMIN_LENS,
485
- });
702
+ })) as { uploadId: string };
703
+ try {
704
+ for (let offset = 0, index = 0; offset < bytes.byteLength; offset += MAX_STORAGE_PART_BYTES, index++) {
705
+ await this.channel.op({
706
+ method: 'storage.putPart',
707
+ uploadId,
708
+ partIndex: index,
709
+ dataB64: base64Of(bytes.subarray(offset, offset + MAX_STORAGE_PART_BYTES)),
710
+ actAs: STORAGE_REMOTE_ADMIN_LENS,
711
+ });
712
+ }
713
+ await this.channel.op({ method: 'storage.finishUpload', uploadId, actAs: STORAGE_REMOTE_ADMIN_LENS });
714
+ } catch (err) {
715
+ try {
716
+ await this.channel.op({ method: 'storage.abortUpload', uploadId, actAs: STORAGE_REMOTE_ADMIN_LENS });
717
+ } catch {
718
+ // secondary abort best effort
719
+ }
720
+ throw err;
721
+ }
486
722
  }
487
723
 
488
- async download(_options: DownloadOptions = {}): Promise<[Buffer]> {
489
- let wire: RemoteGetBytesResult;
724
+ /** The object's bytes, or the inclusive range `start..end`, as they arrive. */
725
+ private async *read(options: DownloadOptions): AsyncGenerator<Uint8Array> {
726
+ const metadata = await this.missingAsNoSuchObject(this.channel.op({
727
+ method: 'storage.getMetadata',
728
+ path: this.name,
729
+ actAs: STORAGE_REMOTE_ADMIN_LENS,
730
+ })) as { bucket: string; size: number; generation: string };
731
+ const route = await this.channel.byteRoute?.();
732
+ const routed = route !== undefined;
733
+ if (!routed) {
734
+ yield* this.readFrames(metadata, options);
735
+ return;
736
+ }
737
+ const response = await this.missingAsNoSuchObject(fetchFromByteRoute(route, {
738
+ bucket: metadata.bucket,
739
+ path: this.name,
740
+ start: options.start,
741
+ end: options.end,
742
+ }));
743
+ const body = response.body;
744
+ if (body === null) return;
745
+ const reader = body.getReader();
490
746
  try {
491
- wire = (await this.channel.op({
747
+ for (;;) {
748
+ const { done, value } = await reader.read();
749
+ if (done) return;
750
+ yield value;
751
+ }
752
+ } finally {
753
+ reader.releaseLock();
754
+ }
755
+ }
756
+
757
+ /** Read from a SharedWorker host, which sends bytes as frames: whole, or in ranged parts (ADR 0015). */
758
+ private async *readFrames(metadata: { size: number; generation: string }, options: DownloadOptions): AsyncGenerator<Uint8Array> {
759
+ if (metadata.size <= MAX_STORAGE_PART_BYTES) {
760
+ const wire = await this.missingAsNoSuchObject(this.channel.op({
492
761
  method: 'storage.getBytes',
493
762
  path: this.name,
494
763
  actAs: STORAGE_REMOTE_ADMIN_LENS,
495
764
  })) as RemoteGetBytesResult;
765
+ yield byteRange(Buffer.from(wire.dataB64, 'base64'), options);
766
+ return;
767
+ }
768
+ const last = Math.min(metadata.size, options.end === undefined ? metadata.size : options.end + 1);
769
+ for (let offset = options.start ?? 0; offset < last; offset += MAX_STORAGE_PART_BYTES) {
770
+ const wire = await this.missingAsNoSuchObject(this.channel.op({
771
+ method: 'storage.getBytes',
772
+ path: this.name,
773
+ offset,
774
+ length: Math.min(MAX_STORAGE_PART_BYTES, last - offset),
775
+ expectedGeneration: metadata.generation,
776
+ actAs: STORAGE_REMOTE_ADMIN_LENS,
777
+ })) as RemoteGetBytesResult;
778
+ yield Buffer.from(wire.dataB64, 'base64');
779
+ }
780
+ }
781
+
782
+ async getMetadata(): Promise<[FileMetadata]> {
783
+ const full = await this.missingAsNoSuchObject(this.channel.op({
784
+ method: 'storage.getMetadata',
785
+ path: this.name,
786
+ actAs: STORAGE_REMOTE_ADMIN_LENS,
787
+ })) as HostMetadata;
788
+ return [fileMetadataOf(full)];
789
+ }
790
+
791
+ async setMetadata(metadata: FileMetadataUpdate): Promise<[FileMetadata]> {
792
+ const full = await this.missingAsNoSuchObject(this.channel.op({
793
+ method: 'storage.setMetadata',
794
+ path: this.name,
795
+ patch: patchOf(metadata, { strict: true }),
796
+ actAs: STORAGE_REMOTE_ADMIN_LENS,
797
+ })) as HostMetadata;
798
+ return [fileMetadataOf(full)];
799
+ }
800
+
801
+ /** The host mints the token; the Node host serves the URL, a SharedWorker host has nowhere to. */
802
+ async [DOWNLOAD_URL](): Promise<string> {
803
+ const { path } = await this.missingAsNoSuchObject(this.channel.op({
804
+ method: 'storage.getDownloadURL',
805
+ path: this.name,
806
+ actAs: STORAGE_REMOTE_ADMIN_LENS,
807
+ })) as { path: string };
808
+ const route = await this.channel.byteRoute?.();
809
+ const routed = route !== undefined;
810
+ if (routed) return new URL(path, route.baseUrl).href;
811
+ const [[metadata], [bytes]] = await Promise.all([this.getMetadata(), this.download()]);
812
+ return dataUri(bytes, metadata.contentType);
813
+ }
814
+
815
+ /** Mirror the gcs/firebase-admin (and local arm) `No such object` message for a missing file. */
816
+ private async missingAsNoSuchObject<T>(pending: Promise<T>): Promise<T> {
817
+ try {
818
+ return await pending;
496
819
  } catch (err) {
497
- if (isObjectNotFound(err)) {
498
- // Mirror the gcs/firebase-admin (and local arm) message shape so
499
- // consumer catch-blocks that string-match `No such object` work
500
- // identically across arms.
501
- throw new Error(`No such object: ${this.bucket.name}/${this.name}`);
502
- }
820
+ if (isObjectNotFound(err)) throw new Error(`No such object: ${this.bucket.name}/${this.name}`);
503
821
  throw err;
504
822
  }
505
- return [Buffer.from(wire.dataB64, 'base64')];
506
823
  }
507
824
 
508
825
  async delete(): Promise<void> {
@@ -540,22 +857,49 @@ class RemoteFile implements File {
540
857
  async getSignedUrl(options: GetSignedUrlOptions): Promise<[string]> {
541
858
  return [stubSignedUrl(this.bucket.name, this.name, options)];
542
859
  }
860
+ }
543
861
 
544
- // ─── Deferred surface (remediating throws, remote-flavored) ─────────
545
-
546
- createWriteStream(): never {
547
- throw new Error(
548
- 'not implemented in pyric-admin/storage remote sandbox backend: createWriteStream — ' +
549
- 'streams cannot span the bridge relay; use file.save(buffer) (≤ 8 MiB) instead.',
550
- );
551
- }
862
+ /** Object metadata as a host reports it (`pyric/storage`'s `FullMetadata`, with its download tokens). */
863
+ interface HostMetadata {
864
+ bucket: string;
865
+ fullPath: string;
866
+ generation: string;
867
+ metageneration: string;
868
+ size: number;
869
+ timeCreated: string;
870
+ updated: string;
871
+ md5Hash?: string;
872
+ contentType?: string;
873
+ cacheControl?: string;
874
+ contentDisposition?: string;
875
+ contentEncoding?: string;
876
+ contentLanguage?: string;
877
+ customMetadata?: Record<string, string>;
878
+ downloadTokens?: string;
879
+ }
552
880
 
553
- createReadStream(): never {
554
- throw new Error(
555
- 'not implemented in pyric-admin/storage remote sandbox backend: createReadStream — ' +
556
- 'streams cannot span the bridge relay; use file.download() (≤ 8 MiB) instead.',
557
- );
881
+ /** A host's metadata in `@google-cloud/storage`'s shape: the download tokens become the custom key production uses. */
882
+ function fileMetadataOf(host: HostMetadata): FileMetadata {
883
+ const custom = { ...(host.customMetadata ?? {}) };
884
+ if (host.downloadTokens) custom[DOWNLOAD_TOKENS_KEY] = host.downloadTokens;
885
+ const settable: Partial<Record<SettableField, string>> = {};
886
+ for (const field of SETTABLE_FIELDS) {
887
+ const value = host[field];
888
+ if (value !== undefined) settable[field] = value;
558
889
  }
890
+ const hasCustom = Object.keys(custom).length > 0;
891
+ return {
892
+ name: host.fullPath,
893
+ bucket: host.bucket,
894
+ generation: host.generation,
895
+ metageneration: host.metageneration,
896
+ size: String(host.size),
897
+ timeCreated: host.timeCreated,
898
+ updated: host.updated,
899
+ ...settable,
900
+ ...(host.md5Hash !== undefined ? { md5Hash: host.md5Hash } : {}),
901
+ ...(hasCustom ? { metadata: custom } : {}),
902
+ };
559
903
  }
560
904
 
561
905
  /** Is this relayed error the worker's `storage/object-not-found`? */
@@ -563,21 +907,108 @@ function isObjectNotFound(err: unknown): boolean {
563
907
  return (err as { code?: unknown })?.code === 'storage/object-not-found';
564
908
  }
565
909
 
566
- /** Over-cap rejection (code `payload-too-large`) — mirrors the worker host's
567
- * message shape and names the streaming gap. */
568
- function payloadTooLarge(sizeBytes: number, what: string): Error & { code: string } {
910
+ /** Over-cap rejection (code `storage/quota-exceeded`) — mirrors the worker host's message shape. */
911
+ function quotaExceeded(sizeBytes: number, what: string): Error & { code: string } {
569
912
  const err = new Error(
570
913
  `pyric-admin/storage: ${what} is ${sizeBytes} bytes — over the ` +
571
- `${MAX_REMOTE_STORAGE_OP_BYTES / (1024 * 1024)} MiB remote storage op cap. ` +
572
- 'Streaming/resumable transfers are not supported on the sandbox backend; ' +
573
- 'split the object or keep it under the cap.',
914
+ `${MAX_STORAGE_OBJECT_BYTES / (1024 * 1024)} MiB maximum storage object cap (MAX_STORAGE_OBJECT_BYTES).`,
574
915
  ) as Error & { code: string };
575
- err.code = 'payload-too-large';
916
+ err.code = 'storage/quota-exceeded';
576
917
  return err;
577
918
  }
578
919
 
579
920
  // ─── Helpers ────────────────────────────────────────────────────────────
580
921
 
922
+ /** A metadata change in the form hosts take: settable fields, custom keys, and the download tokens apart. */
923
+ interface MetadataPatch {
924
+ settable: Partial<Record<SettableField, string>>;
925
+ customMetadata: Record<string, string | null>;
926
+ downloadTokens?: string | null;
927
+ }
928
+
929
+ /**
930
+ * Split `@google-cloud/storage`-shaped metadata into a {@link MetadataPatch}.
931
+ * `setMetadata` is strict and refuses fields the sandbox does not keep;
932
+ * `save` has always stored what it was given and ignores them.
933
+ */
934
+ function patchOf(metadata: FileMetadataUpdate | Record<string, unknown>, { strict }: { strict: boolean }): MetadataPatch {
935
+ const source = metadata as Record<string, unknown>;
936
+ const unsupported = Object.keys(source).filter(key => key !== 'metadata' && !(SETTABLE_FIELDS as readonly string[]).includes(key));
937
+ const refused = strict && unsupported.length > 0;
938
+ if (refused) {
939
+ throw new Error(
940
+ `not implemented in pyric-admin/storage sandbox backend: setMetadata of ${unsupported.join(', ')}. ` +
941
+ `The sandbox keeps ${SETTABLE_FIELDS.join(', ')}, and custom metadata under \`metadata\`.`,
942
+ );
943
+ }
944
+ const patch: MetadataPatch = { settable: {}, customMetadata: {} };
945
+ for (const field of SETTABLE_FIELDS) {
946
+ const value = source[field];
947
+ if (typeof value === 'string') patch.settable[field] = value;
948
+ }
949
+ const custom = source.metadata;
950
+ const hasCustom = custom !== null && typeof custom === 'object';
951
+ if (!hasCustom) return patch;
952
+ for (const [key, value] of Object.entries(custom as Record<string, unknown>)) {
953
+ if (value === undefined) continue;
954
+ const next = value === null ? null : String(value);
955
+ if (key === DOWNLOAD_TOKENS_KEY) patch.downloadTokens = next;
956
+ else patch.customMetadata[key] = next;
957
+ }
958
+ return patch;
959
+ }
960
+
961
+ /** A `data:` URI carrying `bytes`, the download URL where no host serves them over HTTP. */
962
+ function dataUri(bytes: Uint8Array, contentType: string | undefined): string {
963
+ return `data:${contentType ?? 'application/octet-stream'};base64,${base64Of(bytes)}`;
964
+ }
965
+
966
+ /** `bytes` as base64, for a frame. */
967
+ function base64Of(bytes: Uint8Array): string {
968
+ return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('base64');
969
+ }
970
+
971
+ /** The inclusive range `start..end` of `bytes`, as `@google-cloud/storage` reads it. */
972
+ function byteRange(bytes: Uint8Array, options: DownloadOptions): Uint8Array {
973
+ const end = options.end === undefined ? undefined : options.end + 1;
974
+ return bytes.subarray(options.start ?? 0, end);
975
+ }
976
+
977
+ /**
978
+ * A writable stream that spools its bytes to a temporary file and, once it
979
+ * finishes, hands that file to `commit`. The file is removed either way.
980
+ */
981
+ function spooledWriteStream(commit: (spooled: string) => Promise<void>): Writable {
982
+ let directory: string | undefined;
983
+ let sink: WriteStream | undefined;
984
+ return new Writable({
985
+ construct(callback) {
986
+ mkdtemp(join(tmpdir(), 'pyric-admin-upload-')).then(created => {
987
+ directory = created;
988
+ sink = createFileWriteStream(join(created, 'object'));
989
+ callback();
990
+ }, callback);
991
+ },
992
+ write(chunk: Buffer, _encoding, callback) {
993
+ sink!.write(chunk, callback);
994
+ },
995
+ final(callback) {
996
+ sink!.end(() => {
997
+ commit(join(directory!, 'object')).then(() => callback(), callback);
998
+ });
999
+ },
1000
+ destroy(error, callback) {
1001
+ sink?.destroy();
1002
+ const created = directory;
1003
+ if (created === undefined) {
1004
+ callback(error);
1005
+ return;
1006
+ }
1007
+ rm(created, { recursive: true, force: true }).finally(() => callback(error));
1008
+ },
1009
+ });
1010
+ }
1011
+
581
1012
  /**
582
1013
  * The deterministic sandbox signed-URL stub, shared by the local and remote
583
1014
  * arms so their output is byte-identical (the URL is never served — it's a