pyric-admin 0.1.0-alpha.22 → 0.1.0-alpha.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -5
- package/dist/admin.d.ts +64 -12
- package/dist/admin.d.ts.map +1 -1
- package/dist/admin.js +53 -8
- package/dist/admin.js.map +1 -1
- package/dist/app/index.d.ts +14 -1
- package/dist/app/index.d.ts.map +1 -1
- package/dist/app/index.js +58 -16
- package/dist/app/index.js.map +1 -1
- package/dist/app-check/index.d.ts +15 -0
- package/dist/app-check/index.d.ts.map +1 -0
- package/dist/app-check/index.js +16 -0
- package/dist/app-check/index.js.map +1 -0
- package/dist/auth/index.d.ts.map +1 -1
- package/dist/auth/index.js +11 -0
- package/dist/auth/index.js.map +1 -1
- package/dist/data-connect/index.d.ts +25 -0
- package/dist/data-connect/index.d.ts.map +1 -0
- package/dist/data-connect/index.js +48 -0
- package/dist/data-connect/index.js.map +1 -0
- package/dist/database/index.d.ts +24 -19
- package/dist/database/index.d.ts.map +1 -1
- package/dist/database/index.js +133 -99
- package/dist/database/index.js.map +1 -1
- package/dist/deferred.d.ts +45 -0
- package/dist/deferred.d.ts.map +1 -0
- package/dist/deferred.js +71 -0
- package/dist/deferred.js.map +1 -0
- package/dist/eventarc/index.d.ts +15 -0
- package/dist/eventarc/index.d.ts.map +1 -0
- package/dist/eventarc/index.js +17 -0
- package/dist/eventarc/index.js.map +1 -0
- package/dist/extensions/index.d.ts +15 -0
- package/dist/extensions/index.d.ts.map +1 -0
- package/dist/extensions/index.js +17 -0
- package/dist/extensions/index.js.map +1 -0
- package/dist/firestore/index.d.ts +6 -0
- package/dist/firestore/index.d.ts.map +1 -1
- package/dist/firestore/index.js +25 -3
- package/dist/firestore/index.js.map +1 -1
- package/dist/functions/index.d.ts +15 -0
- package/dist/functions/index.d.ts.map +1 -0
- package/dist/functions/index.js +17 -0
- package/dist/functions/index.js.map +1 -0
- package/dist/installations/index.d.ts +15 -0
- package/dist/installations/index.d.ts.map +1 -0
- package/dist/installations/index.js +17 -0
- package/dist/installations/index.js.map +1 -0
- package/dist/instance-id/index.d.ts +15 -0
- package/dist/instance-id/index.d.ts.map +1 -0
- package/dist/instance-id/index.js +17 -0
- package/dist/instance-id/index.js.map +1 -0
- package/dist/machine-learning/index.d.ts +21 -0
- package/dist/machine-learning/index.d.ts.map +1 -0
- package/dist/machine-learning/index.js +25 -0
- package/dist/machine-learning/index.js.map +1 -0
- package/dist/messaging/index.d.ts +0 -1
- package/dist/messaging/index.d.ts.map +1 -1
- package/dist/messaging/index.js +8 -8
- package/dist/messaging/index.js.map +1 -1
- package/dist/phone-number-verification/index.d.ts +14 -0
- package/dist/phone-number-verification/index.d.ts.map +1 -0
- package/dist/phone-number-verification/index.js +16 -0
- package/dist/phone-number-verification/index.js.map +1 -0
- package/dist/project-management/index.d.ts +23 -0
- package/dist/project-management/index.d.ts.map +1 -0
- package/dist/project-management/index.js +25 -0
- package/dist/project-management/index.js.map +1 -0
- package/dist/remote-config/index.d.ts +23 -0
- package/dist/remote-config/index.d.ts.map +1 -0
- package/dist/remote-config/index.js +26 -0
- package/dist/remote-config/index.js.map +1 -0
- package/dist/security-rules/index.d.ts +25 -0
- package/dist/security-rules/index.d.ts.map +1 -0
- package/dist/security-rules/index.js +29 -0
- package/dist/security-rules/index.js.map +1 -0
- package/dist/storage/deferred-members.d.ts +14 -0
- package/dist/storage/deferred-members.d.ts.map +1 -0
- package/dist/storage/deferred-members.js +55 -0
- package/dist/storage/deferred-members.js.map +1 -0
- package/dist/storage/delete-files.d.ts +13 -0
- package/dist/storage/delete-files.d.ts.map +1 -0
- package/dist/storage/delete-files.js +39 -0
- package/dist/storage/delete-files.js.map +1 -0
- package/dist/storage/file-query.d.ts +52 -0
- package/dist/storage/file-query.d.ts.map +1 -0
- package/dist/storage/file-query.js +94 -0
- package/dist/storage/file-query.js.map +1 -0
- package/dist/storage/get-files.d.ts +15 -0
- package/dist/storage/get-files.d.ts.map +1 -0
- package/dist/storage/get-files.js +33 -0
- package/dist/storage/get-files.js.map +1 -0
- package/dist/storage/index.d.ts +159 -23
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +688 -149
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/remote-names.d.ts +19 -0
- package/dist/storage/remote-names.d.ts.map +1 -0
- package/dist/storage/remote-names.js +26 -0
- package/dist/storage/remote-names.js.map +1 -0
- package/package.json +50 -2
- package/src/admin.ts +58 -8
- package/src/app/index.ts +71 -19
- package/src/app-check/index.ts +32 -0
- package/src/auth/auth.test.ts +27 -0
- package/src/auth/index.ts +8 -0
- package/src/data-connect/index.ts +75 -0
- package/src/database/database.test.ts +1 -0
- package/src/database/index.ts +182 -108
- package/src/deferred.ts +96 -0
- package/src/eventarc/index.ts +37 -0
- package/src/extensions/index.ts +37 -0
- package/src/firestore/index.ts +30 -3
- package/src/functions/index.ts +37 -0
- package/src/installations/index.ts +38 -0
- package/src/instance-id/index.ts +38 -0
- package/src/machine-learning/index.ts +51 -0
- package/src/messaging/index.ts +3 -9
- package/src/phone-number-verification/index.ts +36 -0
- package/src/project-management/index.ts +55 -0
- package/src/remote-config/index.ts +58 -0
- package/src/security-rules/index.ts +59 -0
- package/src/storage/deferred-members.ts +67 -0
- package/src/storage/delete-files.ts +40 -0
- package/src/storage/file-query.ts +135 -0
- package/src/storage/get-files.ts +40 -0
- package/src/storage/index.ts +918 -183
- package/src/storage/remote-names.ts +34 -0
- package/src/storage/storage.test.ts +78 -5
package/dist/storage/index.js
CHANGED
|
@@ -7,10 +7,15 @@
|
|
|
7
7
|
*
|
|
8
8
|
* - **Remote sandbox path** — a handle branded by `@pyric/cli`'
|
|
9
9
|
* `connectRemoteSandbox()`/`remoteSandbox()` relays every data
|
|
10
|
-
* operation
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* the
|
|
10
|
+
* operation to the sandbox's host (admin lens pinned — rules bypass).
|
|
11
|
+
* Bytes move over the host's HTTP byte route when it has one, and as
|
|
12
|
+
* frames when it does not (a browser tab's SharedWorker). Each bucket
|
|
13
|
+
* name is its own bucket on the host; the default bucket is the host's
|
|
14
|
+
* default one. `createResumableUpload` gives a session URL on the byte
|
|
15
|
+
* route. `getSignedUrl` stays the local stub. See the remote arm section below.
|
|
16
|
+
*
|
|
17
|
+
* - **Default bucket**: `bucket()` is the bucket the app's `storageBucket`
|
|
18
|
+
* option names, or `'pyric-default'` without one, on both paths.
|
|
14
19
|
*
|
|
15
20
|
* - **Sandbox path** — returns an in-process {@link Storage} backed
|
|
16
21
|
* by an in-memory `Map<bucketName, Map<path, FileEntry>>`. State
|
|
@@ -23,19 +28,53 @@
|
|
|
23
28
|
* - `storage.bucket(name?)` → {@link Bucket}-shaped handle
|
|
24
29
|
* - `bucket.file(path)` → {@link File}-shaped handle
|
|
25
30
|
* - `file.save(data, options?)` — `Buffer | string | Uint8Array`
|
|
26
|
-
* - `file.download(options?)` → `[Buffer]`
|
|
31
|
+
* - `file.download(options?)` → `[Buffer]`, whole or `start..end`
|
|
32
|
+
* - `file.createReadStream(options?)`, whole or `start..end`
|
|
33
|
+
* - `file.createWriteStream(options?)`
|
|
34
|
+
* - `save` and `createWriteStream` take `firebaseStorageDownloadTokens`
|
|
35
|
+
* under `metadata.metadata` as the file's download tokens, which grant
|
|
36
|
+
* its download URL as `getDownloadURL` mints them
|
|
37
|
+
* - `file.getMetadata()` / `file.setMetadata(metadata)`: settable fields
|
|
38
|
+
* and custom metadata; a custom key set to `null` is removed
|
|
39
|
+
* - `getDownloadURL(file)`: mints a token into
|
|
40
|
+
* `firebaseStorageDownloadTokens` when the file has none. The Node
|
|
41
|
+
* host returns its HTTP URL; elsewhere the URL is a `data:` URI
|
|
27
42
|
* - `file.delete()` — idempotent
|
|
28
43
|
* - `file.exists()` → `[boolean]`
|
|
29
44
|
* - `file.getSignedUrl(options)` → `['pyric-sandbox-storage://…']`
|
|
45
|
+
* - `file.copy(destination, options?)`, within a bucket or across buckets
|
|
46
|
+
* - `bucket.getFiles(query?)` → `[files, nextQuery, apiResponse]`, with
|
|
47
|
+
* `prefix`, `delimiter`, `maxResults`, `pageToken`, `autoPaginate`
|
|
48
|
+
* - `bucket.deleteFiles(query?)`, honoring `force`
|
|
30
49
|
*
|
|
31
50
|
* **Deferred in the sandbox backend** (throws `"not implemented in
|
|
32
|
-
* pyric-admin/storage sandbox backend"`):
|
|
33
|
-
*
|
|
34
|
-
* policies, lifecycle rules, ACLs,
|
|
51
|
+
* pyric-admin/storage sandbox backend"`): `file.createResumableUpload`
|
|
52
|
+
* upload sessions, which need an HTTP host, signed cookies, IAM
|
|
53
|
+
* policies, lifecycle rules, ACLs, move, notifications. Every other
|
|
54
|
+
* `Bucket` and `File` method is a deferred member listed in
|
|
55
|
+
* `deferred-members.ts`.
|
|
35
56
|
*/
|
|
57
|
+
import { createWriteStream as createFileWriteStream, openAsBlob } from 'node:fs';
|
|
58
|
+
import { randomUUID } from 'node:crypto';
|
|
59
|
+
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
|
60
|
+
import { tmpdir } from 'node:os';
|
|
61
|
+
import { join } from 'node:path';
|
|
62
|
+
import { Readable, Writable } from 'node:stream';
|
|
36
63
|
import { isRemoteSandbox, } from 'pyric/sandbox';
|
|
64
|
+
import { deleteObject as storageDeleteObject, getMetadata as storageGetMetadata, ref as storageRef, } from 'pyric/storage';
|
|
65
|
+
import { copyObject, fetchFromByteRoute, getAdminStorageSandbox, getStorageService, patchObjectMetadata, uploadObject, uploadOverByteRoute, } from 'pyric/storage/internal';
|
|
37
66
|
import { ADMIN_APP_TARGET, getApp, } from '../app/index.js';
|
|
38
67
|
import { assertAdminAppActive } from '../app/lifecycle.js';
|
|
68
|
+
import { defineDeferredMembers } from 'pyric/app/internal';
|
|
69
|
+
import { BUCKET_DEFERRED_MEMBERS, FILE_DEFERRED_MEMBERS } from './deferred-members.js';
|
|
70
|
+
import { deleteListedFiles } from './delete-files.js';
|
|
71
|
+
import { assertQuery } from './file-query.js';
|
|
72
|
+
import { getFilesResponse } from './get-files.js';
|
|
73
|
+
import { namesUnderPrefix } from './remote-names.js';
|
|
74
|
+
/** The custom metadata key holding a file's download tokens, comma-separated. */
|
|
75
|
+
const DOWNLOAD_TOKENS_KEY = 'firebaseStorageDownloadTokens';
|
|
76
|
+
/** Settable fields the sandbox keeps, besides custom metadata. */
|
|
77
|
+
const SETTABLE_FIELDS = ['contentType', 'cacheControl', 'contentDisposition', 'contentEncoding', 'contentLanguage'];
|
|
39
78
|
/**
|
|
40
79
|
* Get the {@link Storage} service for the given app.
|
|
41
80
|
*
|
|
@@ -53,10 +92,11 @@ export function getStorage(app) {
|
|
|
53
92
|
// must never touch a remote handle — local state keyed off a remote
|
|
54
93
|
// handle would be a private server-side store the browser never sees,
|
|
55
94
|
// and `onEvent` throws on remote handles by design.
|
|
95
|
+
const defaultBucket = resolved.options.storageBucket ?? DEFAULT_SANDBOX_BUCKET;
|
|
56
96
|
if (isRemoteSandbox(resolved.sandbox)) {
|
|
57
|
-
return getRemoteStorage(resolved.sandbox);
|
|
97
|
+
return getRemoteStorage(resolved.sandbox, defaultBucket);
|
|
58
98
|
}
|
|
59
|
-
return getSandboxStorage(resolved);
|
|
99
|
+
return getSandboxStorage(resolved, defaultBucket);
|
|
60
100
|
}
|
|
61
101
|
// Defensive: the union is closed at the type level. A runtime value
|
|
62
102
|
// that lands here means a caller forged a handle without going
|
|
@@ -64,13 +104,23 @@ export function getStorage(app) {
|
|
|
64
104
|
throw new TypeError('pyric-admin/storage: getStorage expected a PyricAdminApp from `initializeApp`; ' +
|
|
65
105
|
'received a value with no recognized ADMIN_APP_TARGET brand.');
|
|
66
106
|
}
|
|
107
|
+
/** Each arm's own download URL. */
|
|
108
|
+
const DOWNLOAD_URL = Symbol('pyric-admin/storage download URL');
|
|
67
109
|
/**
|
|
68
|
-
*
|
|
69
|
-
*
|
|
110
|
+
* Mirror of `firebase-admin/storage`'s `getDownloadURL(file)`. A file without
|
|
111
|
+
* a download token gets one in `firebaseStorageDownloadTokens`, as production
|
|
112
|
+
* mints it. On the Node host the URL is its HTTP byte route URL, which serves
|
|
113
|
+
* ranges and stops working when the token is removed. In process and on a
|
|
114
|
+
* SharedWorker host there is no HTTP origin, so the URL is a `data:` URI that
|
|
115
|
+
* carries the bytes.
|
|
70
116
|
*/
|
|
71
117
|
export async function getDownloadURL(file) {
|
|
72
|
-
const
|
|
73
|
-
|
|
118
|
+
const source = file;
|
|
119
|
+
const served = typeof source[DOWNLOAD_URL] === 'function';
|
|
120
|
+
if (!served) {
|
|
121
|
+
throw new Error('pyric-admin/storage: getDownloadURL expects a File from getStorage().bucket().file().');
|
|
122
|
+
}
|
|
123
|
+
return source[DOWNLOAD_URL]();
|
|
74
124
|
}
|
|
75
125
|
// ─── Sandbox path ───────────────────────────────────────────────────────
|
|
76
126
|
/**
|
|
@@ -81,131 +131,240 @@ export async function getDownloadURL(file) {
|
|
|
81
131
|
const DEFAULT_SANDBOX_BUCKET = 'pyric-default';
|
|
82
132
|
/**
|
|
83
133
|
* State + reset-handler registry keyed on `Sandbox` so a single
|
|
84
|
-
* Sandbox shares its storage across every `getStorage` call
|
|
85
|
-
* `WeakMap` lets a discarded `Sandbox`
|
|
134
|
+
* Sandbox shares its storage across every `getStorage` call and with
|
|
135
|
+
* `pyric/storage`. The `WeakMap` lets a discarded `Sandbox` be GC'd
|
|
86
136
|
* naturally.
|
|
87
137
|
*/
|
|
88
138
|
const SANDBOX_STATE = new WeakMap();
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
if (!map) {
|
|
100
|
-
map = new Map();
|
|
101
|
-
SANDBOX_STATE.set(sandbox, map);
|
|
102
|
-
}
|
|
103
|
-
if (!RESET_HOOKED.has(sandbox)) {
|
|
104
|
-
RESET_HOOKED.add(sandbox);
|
|
139
|
+
function ensureSandboxStorageState(sandbox, defaultBucket) {
|
|
140
|
+
let state = SANDBOX_STATE.get(sandbox);
|
|
141
|
+
if (!state) {
|
|
142
|
+
state = {
|
|
143
|
+
dbName: `pyric-admin-storage:${randomUUID()}`,
|
|
144
|
+
defaultBucketNames: new Set([DEFAULT_SANDBOX_BUCKET]),
|
|
145
|
+
buckets: new Set([DEFAULT_SANDBOX_BUCKET]),
|
|
146
|
+
pendingReset: undefined,
|
|
147
|
+
};
|
|
148
|
+
SANDBOX_STATE.set(sandbox, state);
|
|
105
149
|
sandbox.onEvent((event) => {
|
|
106
150
|
if (event.kind === 'session_boundary' && event.phase === 'reset') {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
151
|
+
const current = SANDBOX_STATE.get(sandbox);
|
|
152
|
+
if (!current)
|
|
153
|
+
return;
|
|
154
|
+
const previous = current.pendingReset ?? Promise.resolve();
|
|
155
|
+
const bucketsToReset = [...current.buckets];
|
|
156
|
+
const nextReset = previous
|
|
157
|
+
.then(async () => {
|
|
158
|
+
await Promise.all(bucketsToReset.map(async (bucket) => {
|
|
159
|
+
const handle = getAdminStorageSandbox(sandbox, {
|
|
160
|
+
dbName: current.dbName,
|
|
161
|
+
bucket,
|
|
162
|
+
});
|
|
163
|
+
const service = await getStorageService(handle);
|
|
164
|
+
await service.backend.reset(bucket);
|
|
165
|
+
}));
|
|
166
|
+
})
|
|
167
|
+
.catch(() => {
|
|
168
|
+
// Best-effort reset across open bucket handles
|
|
169
|
+
});
|
|
170
|
+
current.pendingReset = nextReset;
|
|
112
171
|
}
|
|
113
172
|
});
|
|
114
173
|
}
|
|
115
|
-
|
|
174
|
+
state.defaultBucketNames.add(defaultBucket);
|
|
175
|
+
return state;
|
|
116
176
|
}
|
|
117
|
-
function getSandboxStorage(app) {
|
|
177
|
+
function getSandboxStorage(app, defaultBucket) {
|
|
118
178
|
const sandbox = app.sandbox;
|
|
119
|
-
const
|
|
120
|
-
return new SandboxStorage(
|
|
179
|
+
const state = ensureSandboxStorageState(sandbox, defaultBucket);
|
|
180
|
+
return new SandboxStorage(sandbox, state, defaultBucket);
|
|
121
181
|
}
|
|
122
182
|
/**
|
|
123
|
-
* Sandbox `Storage` implementation.
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
* `@google-cloud/storage` returns lightweight per-call handles.
|
|
183
|
+
* Sandbox `Storage` implementation. Delegates to the per-sandbox
|
|
184
|
+
* `pyric/storage` admin plane (`getAdminStorageSandbox`) so `pyric-admin/storage`,
|
|
185
|
+
* `pyric/storage`, and `captureFullState` share one object store.
|
|
127
186
|
*/
|
|
128
187
|
class SandboxStorage {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
188
|
+
sandbox;
|
|
189
|
+
state;
|
|
190
|
+
defaultBucket;
|
|
191
|
+
constructor(sandbox, state, defaultBucket) {
|
|
192
|
+
this.sandbox = sandbox;
|
|
193
|
+
this.state = state;
|
|
194
|
+
this.defaultBucket = defaultBucket;
|
|
132
195
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
196
|
+
storeBucketOf(bucketName) {
|
|
197
|
+
if (bucketName === DEFAULT_SANDBOX_BUCKET ||
|
|
198
|
+
bucketName === this.defaultBucket ||
|
|
199
|
+
this.state.defaultBucketNames.has(bucketName)) {
|
|
200
|
+
return DEFAULT_SANDBOX_BUCKET;
|
|
201
|
+
}
|
|
202
|
+
return bucketName;
|
|
203
|
+
}
|
|
204
|
+
async waitForReset() {
|
|
205
|
+
while (this.state.pendingReset !== undefined) {
|
|
206
|
+
const pending = this.state.pendingReset;
|
|
207
|
+
await pending;
|
|
208
|
+
if (this.state.pendingReset === pending) {
|
|
209
|
+
this.state.pendingReset = undefined;
|
|
210
|
+
}
|
|
139
211
|
}
|
|
140
|
-
|
|
212
|
+
}
|
|
213
|
+
bucket(name) {
|
|
214
|
+
const bucketName = name ?? this.defaultBucket;
|
|
215
|
+
const storeBucket = this.storeBucketOf(bucketName);
|
|
216
|
+
this.state.buckets.add(storeBucket);
|
|
217
|
+
return new SandboxBucket(bucketName, storeBucket, this);
|
|
141
218
|
}
|
|
142
219
|
}
|
|
143
220
|
class SandboxBucket {
|
|
144
221
|
name;
|
|
145
|
-
|
|
146
|
-
|
|
222
|
+
storeBucket;
|
|
223
|
+
storage;
|
|
224
|
+
constructor(name, storeBucket, storage) {
|
|
147
225
|
this.name = name;
|
|
148
|
-
this.
|
|
226
|
+
this.storeBucket = storeBucket;
|
|
227
|
+
this.storage = storage;
|
|
228
|
+
}
|
|
229
|
+
adminStorage() {
|
|
230
|
+
this.storage.state.buckets.add(this.storeBucket);
|
|
231
|
+
return getAdminStorageSandbox(this.storage.sandbox, {
|
|
232
|
+
dbName: this.storage.state.dbName,
|
|
233
|
+
bucket: this.storeBucket,
|
|
234
|
+
});
|
|
149
235
|
}
|
|
150
236
|
file(path) {
|
|
151
|
-
return new SandboxFile(path, this
|
|
237
|
+
return new SandboxFile(path, this);
|
|
238
|
+
}
|
|
239
|
+
async getFiles(query = {}) {
|
|
240
|
+
assertQuery(query, 'getFiles');
|
|
241
|
+
return getFilesResponse(await this.namesUnder(query.prefix ?? ''), query, (name) => this.file(name));
|
|
242
|
+
}
|
|
243
|
+
async deleteFiles(query = {}) {
|
|
244
|
+
assertQuery(query, 'deleteFiles');
|
|
245
|
+
await deleteListedFiles(await this.namesUnder(query.prefix ?? ''), query, (name) => this.file(name).delete());
|
|
246
|
+
}
|
|
247
|
+
async namesUnder(prefix) {
|
|
248
|
+
await this.storage.waitForReset();
|
|
249
|
+
const service = await getStorageService(this.adminStorage());
|
|
250
|
+
const items = await service.backend.listByPrefix(prefix, this.storeBucket);
|
|
251
|
+
return items.map((item) => item.fullPath);
|
|
152
252
|
}
|
|
153
253
|
}
|
|
154
254
|
class SandboxFile {
|
|
155
255
|
name;
|
|
156
256
|
bucket;
|
|
157
|
-
|
|
158
|
-
constructor(name, bucket, files) {
|
|
257
|
+
constructor(name, bucket) {
|
|
159
258
|
this.name = name;
|
|
160
259
|
this.bucket = bucket;
|
|
161
|
-
this.files = files;
|
|
162
260
|
}
|
|
261
|
+
/**
|
|
262
|
+
* Store `data` in one write. `resumable` only chooses how
|
|
263
|
+
* `@google-cloud/storage` sends the bytes; the stored object is the same.
|
|
264
|
+
*/
|
|
163
265
|
async save(data, options = {}) {
|
|
164
|
-
|
|
165
|
-
|
|
266
|
+
const rawLength = typeof data === 'string' ? Buffer.byteLength(data, 'utf8') : data.byteLength;
|
|
267
|
+
if (rawLength > MAX_STORAGE_OBJECT_BYTES) {
|
|
268
|
+
throw quotaExceeded(rawLength, `save() payload for '${this.name}'`);
|
|
166
269
|
}
|
|
270
|
+
await this.bucket.storage.waitForReset();
|
|
167
271
|
const bytes = toBytes(data);
|
|
168
|
-
const
|
|
169
|
-
const
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
...(
|
|
272
|
+
const saved = savedMetadataOf(options.metadata ?? {});
|
|
273
|
+
const contentType = options.contentType ?? saved.settable.contentType;
|
|
274
|
+
const settable = {
|
|
275
|
+
...saved.settable,
|
|
276
|
+
...(contentType !== undefined ? { contentType } : {}),
|
|
277
|
+
...(Object.keys(saved.custom).length > 0 ? { customMetadata: saved.custom } : {}),
|
|
173
278
|
};
|
|
174
|
-
this.
|
|
279
|
+
await uploadObject(storageRef(this.bucket.adminStorage(), this.name), bytes, settable, { ...(saved.downloadTokens !== undefined ? { downloadTokens: saved.downloadTokens } : {}) });
|
|
175
280
|
}
|
|
176
|
-
async
|
|
177
|
-
|
|
178
|
-
|
|
281
|
+
async getMetadata() {
|
|
282
|
+
await this.bucket.storage.waitForReset();
|
|
283
|
+
const full = (await this.missingAsNoSuchObject(storageGetMetadata(storageRef(this.bucket.adminStorage(), this.name))));
|
|
284
|
+
return [fileMetadataOf({ ...full, bucket: this.bucket.name })];
|
|
285
|
+
}
|
|
286
|
+
async setMetadata(metadata) {
|
|
287
|
+
await this.bucket.storage.waitForReset();
|
|
288
|
+
const patch = patchOf(metadata, { strict: true });
|
|
289
|
+
const full = (await this.missingAsNoSuchObject(patchObjectMetadata(storageRef(this.bucket.adminStorage(), this.name), patch)));
|
|
290
|
+
return [fileMetadataOf({ ...full, bucket: this.bucket.name })];
|
|
291
|
+
}
|
|
292
|
+
async [DOWNLOAD_URL]() {
|
|
293
|
+
await this.bucket.storage.waitForReset();
|
|
294
|
+
const r = storageRef(this.bucket.adminStorage(), this.name);
|
|
295
|
+
const full = (await this.missingAsNoSuchObject(storageGetMetadata(r)));
|
|
296
|
+
const token = (full.downloadTokens ?? '').split(',').find((existing) => existing !== '');
|
|
297
|
+
if (token === undefined) {
|
|
298
|
+
await patchObjectMetadata(r, { downloadTokens: randomUUID() });
|
|
299
|
+
}
|
|
300
|
+
const [bytes] = await this.download();
|
|
301
|
+
return dataUri(bytes, full.contentType);
|
|
302
|
+
}
|
|
303
|
+
async missingAsNoSuchObject(pending) {
|
|
304
|
+
try {
|
|
305
|
+
return await pending;
|
|
306
|
+
}
|
|
307
|
+
catch (err) {
|
|
308
|
+
if (isObjectNotFound(err))
|
|
309
|
+
throw new Error(`No such object: ${this.bucket.name}/${this.name}`);
|
|
310
|
+
throw err;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
async download(options = {}) {
|
|
314
|
+
await this.bucket.storage.waitForReset();
|
|
315
|
+
const service = await getStorageService(this.bucket.adminStorage());
|
|
316
|
+
const object = await service.backend.getObject(this.name, this.bucket.storeBucket);
|
|
317
|
+
if (!object) {
|
|
179
318
|
// Mirror the gcs/firebase-admin error message shape so consumer
|
|
180
319
|
// catch-blocks that string-match `No such object` keep working.
|
|
181
320
|
throw new Error(`No such object: ${this.bucket.name}/${this.name}`);
|
|
182
321
|
}
|
|
183
|
-
|
|
322
|
+
const bytes = new Uint8Array(await object.blob.arrayBuffer());
|
|
323
|
+
return [Buffer.from(byteRange(bytes, options))];
|
|
324
|
+
}
|
|
325
|
+
createReadStream(options = {}) {
|
|
326
|
+
const read = async function* (file) {
|
|
327
|
+
const [bytes] = await file.download(options);
|
|
328
|
+
yield bytes;
|
|
329
|
+
};
|
|
330
|
+
return Readable.from(read(this), { objectMode: false });
|
|
331
|
+
}
|
|
332
|
+
createWriteStream(options = {}) {
|
|
333
|
+
return spooledWriteStream(async (spooled) => this.save(await readFile(spooled), options));
|
|
334
|
+
}
|
|
335
|
+
async createResumableUpload() {
|
|
336
|
+
throw new Error('not implemented in pyric-admin/storage sandbox backend: resumable upload sessions (createResumableUpload). ' +
|
|
337
|
+
'The in-process sandbox has no HTTP host to take the bytes; upload with file.save() or file.createWriteStream().');
|
|
338
|
+
}
|
|
339
|
+
async copy(destination, options = {}) {
|
|
340
|
+
const target = copyTargetOf(this, destination, this.bucket.storage);
|
|
341
|
+
const local = target instanceof SandboxFile && target.bucket.storage.sandbox === this.bucket.storage.sandbox;
|
|
342
|
+
if (!local)
|
|
343
|
+
throw new Error('pyric-admin/storage: File.copy copies to a File from the same getStorage() sandbox.');
|
|
344
|
+
await this.bucket.storage.waitForReset();
|
|
345
|
+
const patch = patchOf(copyMetadataOf(options), { strict: false });
|
|
346
|
+
const full = (await this.missingAsNoSuchObject(copyObject(storageRef(this.bucket.adminStorage(), this.name), storageRef(target.bucket.adminStorage(), target.name), patch)));
|
|
347
|
+
return [target, fileMetadataOf({ ...full, bucket: target.bucket.name })];
|
|
184
348
|
}
|
|
185
349
|
async delete() {
|
|
186
|
-
this.
|
|
350
|
+
await this.bucket.storage.waitForReset();
|
|
351
|
+
await storageDeleteObject(storageRef(this.bucket.adminStorage(), this.name));
|
|
187
352
|
}
|
|
188
353
|
async exists() {
|
|
189
|
-
|
|
354
|
+
await this.bucket.storage.waitForReset();
|
|
355
|
+
const service = await getStorageService(this.bucket.adminStorage());
|
|
356
|
+
const stored = await service.backend.getMetadata(this.name, this.bucket.storeBucket);
|
|
357
|
+
return [stored !== undefined];
|
|
190
358
|
}
|
|
191
359
|
async getSignedUrl(options) {
|
|
192
360
|
return [stubSignedUrl(this.bucket.name, this.name, options)];
|
|
193
361
|
}
|
|
194
|
-
// ─── Deferred surface (declared so TS callers see a clear error) ────
|
|
195
|
-
/** @deprecated Streaming writes are deferred — see module header. */
|
|
196
|
-
createWriteStream() {
|
|
197
|
-
throw new Error('not implemented in pyric-admin/storage sandbox backend: createWriteStream');
|
|
198
|
-
}
|
|
199
|
-
/** @deprecated Streaming reads are deferred — see module header. */
|
|
200
|
-
createReadStream() {
|
|
201
|
-
throw new Error('not implemented in pyric-admin/storage sandbox backend: createReadStream');
|
|
202
|
-
}
|
|
203
362
|
}
|
|
204
363
|
// ─── Remote sandbox arm (remote sandbox, slice 2) ───────────────────────
|
|
205
364
|
//
|
|
206
|
-
// The app's `Sandbox` is a Node-side handle onto
|
|
207
|
-
// SharedWorker sandbox. Every data operation relays over the
|
|
208
|
-
// worker channel with `actAs: { mode: 'admin' }` pinned — firebase-admin's
|
|
365
|
+
// The app's `Sandbox` is a Node-side handle onto a hosted sandbox or a
|
|
366
|
+
// browser tab's SharedWorker sandbox. Every data operation relays over the
|
|
367
|
+
// handle's worker channel with `actAs: { mode: 'admin' }` pinned — firebase-admin's
|
|
209
368
|
// rules-bypass semantics against the ONE object store the app + Studio +
|
|
210
369
|
// agents share (the host resolves the lens to `pyric/storage/internal`'s
|
|
211
370
|
// admin plane). There is deliberately NO local state here: a `WeakMap`
|
|
@@ -213,62 +372,97 @@ class SandboxFile {
|
|
|
213
372
|
// the browser never sees, and the local arm's `onEvent` reset hook throws
|
|
214
373
|
// on remote handles by design.
|
|
215
374
|
//
|
|
375
|
+
// Buckets: every op names its bucket, and gives the app's name for the
|
|
376
|
+
// default bucket, so the host stores each named bucket apart and answers to
|
|
377
|
+
// the configured default name, in ops and in byte route URLs alike.
|
|
378
|
+
//
|
|
216
379
|
// Divergences from the local arm, all LOUD:
|
|
217
|
-
// -
|
|
218
|
-
//
|
|
219
|
-
//
|
|
220
|
-
//
|
|
221
|
-
// - byte payloads are capped at 8 MiB per op (whole-object buffering
|
|
222
|
-
// over four relay hops; streaming stays unsupported on both sandbox arms).
|
|
380
|
+
// - a host with an HTTP byte route takes and serves bytes there; a
|
|
381
|
+
// SharedWorker host has none and takes frames, in 4 MiB parts past that.
|
|
382
|
+
// - a host with a byte route gives `createResumableUpload` session URLs;
|
|
383
|
+
// the local arm and a SharedWorker host reject it.
|
|
223
384
|
// `getSignedUrl` does NOT relay: it stays the byte-identical local stub.
|
|
224
385
|
/** firebase-admin's rules-bypass lens, pinned on every relayed operation. */
|
|
225
386
|
const STORAGE_REMOTE_ADMIN_LENS = { mode: 'admin' };
|
|
226
387
|
/**
|
|
227
|
-
* Raw per-
|
|
228
|
-
* `@pyric/cli`' `
|
|
388
|
+
* Raw per-part byte cap for storage payloads sent as frames. MUST mirror
|
|
389
|
+
* `@pyric/cli`' `MAX_STORAGE_PART_BYTES` (serve/worker/protocol.ts) — the
|
|
229
390
|
* worker host enforces the same cap on its end. Inlined (like the RTDB
|
|
230
391
|
* push-id generator) because `pyric-admin` deliberately does not depend on
|
|
231
392
|
* `@pyric/cli`.
|
|
232
393
|
*/
|
|
233
|
-
const
|
|
234
|
-
/**
|
|
394
|
+
const MAX_STORAGE_PART_BYTES = 4 * 1024 * 1024;
|
|
395
|
+
/** Maximum whole-object size supported by the sandbox backend (512 MiB). Matches MAX_STORAGE_OBJECT_BYTES. */
|
|
396
|
+
const MAX_STORAGE_OBJECT_BYTES = 512 * 1024 * 1024;
|
|
397
|
+
/** One remote `Storage` per remote handle and default bucket name (handles only, never data). */
|
|
235
398
|
const remoteStorageBySandbox = new WeakMap();
|
|
236
|
-
function getRemoteStorage(sandbox) {
|
|
237
|
-
let
|
|
399
|
+
function getRemoteStorage(sandbox, defaultBucket) {
|
|
400
|
+
let byDefault = remoteStorageBySandbox.get(sandbox);
|
|
401
|
+
if (!byDefault) {
|
|
402
|
+
byDefault = new Map();
|
|
403
|
+
remoteStorageBySandbox.set(sandbox, byDefault);
|
|
404
|
+
}
|
|
405
|
+
let storage = byDefault.get(defaultBucket);
|
|
238
406
|
if (!storage) {
|
|
239
|
-
storage = new RemoteStorage(sandbox.channel);
|
|
240
|
-
|
|
407
|
+
storage = new RemoteStorage(sandbox.channel, defaultBucket);
|
|
408
|
+
byDefault.set(defaultBucket, storage);
|
|
241
409
|
}
|
|
242
410
|
return storage;
|
|
243
411
|
}
|
|
244
412
|
class RemoteStorage {
|
|
245
413
|
channel;
|
|
246
|
-
|
|
414
|
+
defaultBucket;
|
|
415
|
+
constructor(channel, defaultBucket) {
|
|
247
416
|
this.channel = channel;
|
|
417
|
+
this.defaultBucket = defaultBucket;
|
|
248
418
|
}
|
|
419
|
+
/** Each bucket name is its own bucket on the host; the default one is the host's default bucket. */
|
|
249
420
|
bucket(name) {
|
|
250
|
-
|
|
251
|
-
// be faithfully relayed — throw loudly instead of silently merging
|
|
252
|
-
// buckets (the local arm has REAL multi-bucket isolation; this is the
|
|
253
|
-
// sharpest local/remote divergence, so it must be explicit).
|
|
254
|
-
if (name !== undefined && name !== DEFAULT_SANDBOX_BUCKET) {
|
|
255
|
-
throw new Error(`pyric-admin/storage: the remote (browser) sandbox has a single bucket — ` +
|
|
256
|
-
`bucket('${name}') cannot be isolated. Use bucket() (the default ` +
|
|
257
|
-
`'${DEFAULT_SANDBOX_BUCKET}' bucket) instead.`);
|
|
258
|
-
}
|
|
259
|
-
return new RemoteBucket(DEFAULT_SANDBOX_BUCKET, this.channel);
|
|
421
|
+
return new RemoteBucket(name ?? this.defaultBucket, this.channel, this);
|
|
260
422
|
}
|
|
261
423
|
}
|
|
262
424
|
class RemoteBucket {
|
|
263
425
|
name;
|
|
264
426
|
channel;
|
|
265
|
-
|
|
427
|
+
storage;
|
|
428
|
+
constructor(name, channel, storage) {
|
|
266
429
|
this.name = name;
|
|
267
430
|
this.channel = channel;
|
|
431
|
+
this.storage = storage;
|
|
432
|
+
}
|
|
433
|
+
/**
|
|
434
|
+
* How ops name this bucket: the default bucket has no `bucket`, and every op
|
|
435
|
+
* gives the app's name for the default bucket, so the host answers to it.
|
|
436
|
+
*/
|
|
437
|
+
get wire() {
|
|
438
|
+
const defaultName = this.storage.defaultBucket;
|
|
439
|
+
const namesDefault = defaultName !== DEFAULT_SANDBOX_BUCKET;
|
|
440
|
+
const isDefault = this.name === defaultName;
|
|
441
|
+
return {
|
|
442
|
+
...(isDefault ? {} : { bucket: this.name }),
|
|
443
|
+
...(namesDefault ? { defaultBucket: defaultName } : {}),
|
|
444
|
+
};
|
|
268
445
|
}
|
|
269
446
|
file(path) {
|
|
270
447
|
return new RemoteFile(path, this, this.channel);
|
|
271
448
|
}
|
|
449
|
+
async getFiles(query = {}) {
|
|
450
|
+
assertQuery(query, 'getFiles');
|
|
451
|
+
return getFilesResponse(await this.namesUnder(query.prefix ?? ''), query, name => this.file(name));
|
|
452
|
+
}
|
|
453
|
+
async deleteFiles(query = {}) {
|
|
454
|
+
assertQuery(query, 'deleteFiles');
|
|
455
|
+
await deleteListedFiles(await this.namesUnder(query.prefix ?? ''), query, name => this.file(name).delete());
|
|
456
|
+
}
|
|
457
|
+
/** The host's object names under `prefix`, walked folder by folder with `storage.listAll`. */
|
|
458
|
+
namesUnder(prefix) {
|
|
459
|
+
return namesUnderPrefix(prefix, async (path) => (await this.channel.op({
|
|
460
|
+
method: 'storage.listAll',
|
|
461
|
+
...this.wire,
|
|
462
|
+
path,
|
|
463
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
464
|
+
})));
|
|
465
|
+
}
|
|
272
466
|
}
|
|
273
467
|
class RemoteFile {
|
|
274
468
|
name;
|
|
@@ -279,47 +473,246 @@ class RemoteFile {
|
|
|
279
473
|
this.bucket = bucket;
|
|
280
474
|
this.channel = channel;
|
|
281
475
|
}
|
|
476
|
+
/**
|
|
477
|
+
* Store `data` in one upload. `resumable` only chooses how
|
|
478
|
+
* `@google-cloud/storage` sends the bytes; the stored object is the same.
|
|
479
|
+
*/
|
|
282
480
|
async save(data, options = {}) {
|
|
283
|
-
|
|
284
|
-
|
|
481
|
+
const rawLength = typeof data === 'string' ? Buffer.byteLength(data, 'utf8') : data.byteLength;
|
|
482
|
+
if (rawLength > MAX_STORAGE_OBJECT_BYTES) {
|
|
483
|
+
throw quotaExceeded(rawLength, `save() payload for '${this.name}'`);
|
|
285
484
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
485
|
+
await this.upload(new Blob([toBytes(data)]), options);
|
|
486
|
+
}
|
|
487
|
+
async download(options = {}) {
|
|
488
|
+
const parts = [];
|
|
489
|
+
for await (const part of this.read(options))
|
|
490
|
+
parts.push(part);
|
|
491
|
+
return [Buffer.concat(parts)];
|
|
492
|
+
}
|
|
493
|
+
createReadStream(options = {}) {
|
|
494
|
+
return Readable.from(this.read(options), { objectMode: false });
|
|
495
|
+
}
|
|
496
|
+
createWriteStream(options = {}) {
|
|
497
|
+
return spooledWriteStream(async (spooled) => {
|
|
498
|
+
const data = await openAsBlob(spooled);
|
|
499
|
+
if (data.size > MAX_STORAGE_OBJECT_BYTES) {
|
|
500
|
+
throw quotaExceeded(data.size, `createWriteStream() payload for '${this.name}'`);
|
|
501
|
+
}
|
|
502
|
+
await this.upload(data, options);
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
/**
|
|
506
|
+
* A session URL on the host's byte route that takes this object's bytes,
|
|
507
|
+
* as `@google-cloud/storage` returns a resumable upload URI. A browser sends
|
|
508
|
+
* them in one `PUT`, or in `Content-Range` slices whose first one names the
|
|
509
|
+
* total size, and the object is created with `metadata` when the last byte
|
|
510
|
+
* arrives. With `origin`, the URL answers that page origin's cross-origin
|
|
511
|
+
* requests. A host without a byte route (a browser tab's SharedWorker) has
|
|
512
|
+
* no URL to give, so this rejects there.
|
|
513
|
+
*/
|
|
514
|
+
async createResumableUpload(options = {}) {
|
|
515
|
+
const route = await this.channel.byteRoute?.();
|
|
516
|
+
const routed = route !== undefined;
|
|
517
|
+
if (!routed) {
|
|
518
|
+
throw new Error('not implemented in pyric-admin/storage remote sandbox backend: resumable upload sessions (createResumableUpload) ' +
|
|
519
|
+
'on a host without an HTTP byte route. Upload with file.save() or file.createWriteStream().');
|
|
289
520
|
}
|
|
290
|
-
await this.channel.op({
|
|
291
|
-
method: 'storage.
|
|
521
|
+
const { uploadUrl } = (await this.channel.op({
|
|
522
|
+
method: 'storage.createUploadSession',
|
|
523
|
+
...this.bucket.wire,
|
|
292
524
|
path: this.name,
|
|
293
|
-
|
|
294
|
-
...(options.
|
|
295
|
-
...(options.metadata !== undefined ? { metadata: options.metadata } : {}),
|
|
525
|
+
...uploadRequestOf({ ...(options.metadata !== undefined ? { metadata: options.metadata } : {}) }),
|
|
526
|
+
...(options.origin !== undefined ? { origin: options.origin } : {}),
|
|
296
527
|
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
297
|
-
});
|
|
528
|
+
}));
|
|
529
|
+
return [new URL(uploadUrl, route.baseUrl).href];
|
|
530
|
+
}
|
|
531
|
+
/** Copy this object, with its metadata, to `destination`; `options` change the copy's metadata. */
|
|
532
|
+
async copy(destination, options = {}) {
|
|
533
|
+
const target = copyTargetOf(this, destination, this.bucket.storage);
|
|
534
|
+
const remote = target instanceof RemoteFile && target.channel === this.channel;
|
|
535
|
+
if (!remote)
|
|
536
|
+
throw new Error('pyric-admin/storage: File.copy copies to a File from the same remote sandbox.');
|
|
537
|
+
const destinationWire = target.bucket.wire;
|
|
538
|
+
const full = await this.missingAsNoSuchObject(this.channel.op({
|
|
539
|
+
method: 'storage.copyObject',
|
|
540
|
+
...this.bucket.wire,
|
|
541
|
+
path: this.name,
|
|
542
|
+
...(destinationWire.bucket !== undefined ? { destinationBucket: destinationWire.bucket } : {}),
|
|
543
|
+
destinationPath: target.name,
|
|
544
|
+
patch: patchOf(copyMetadataOf(options), { strict: false }),
|
|
545
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
546
|
+
}));
|
|
547
|
+
return [target, fileMetadataOf(full)];
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* Store `data` over the host's byte route, or as one frame when the host has
|
|
551
|
+
* none. `firebaseStorageDownloadTokens` becomes the object's download tokens,
|
|
552
|
+
* as `setMetadata` sends it, and the other custom keys stay custom metadata.
|
|
553
|
+
*/
|
|
554
|
+
async upload(data, options) {
|
|
555
|
+
const request = { ...this.bucket.wire, ...uploadRequestOf(options) };
|
|
556
|
+
const route = await this.channel.byteRoute?.();
|
|
557
|
+
const routed = route !== undefined;
|
|
558
|
+
if (routed) {
|
|
559
|
+
await uploadOverByteRoute(this.channel, route, { path: this.name, data, ...request, actAs: STORAGE_REMOTE_ADMIN_LENS });
|
|
560
|
+
return;
|
|
561
|
+
}
|
|
562
|
+
const bytes = new Uint8Array(await data.arrayBuffer());
|
|
563
|
+
if (bytes.byteLength <= MAX_STORAGE_PART_BYTES) {
|
|
564
|
+
await this.channel.op({
|
|
565
|
+
method: 'storage.putBytes',
|
|
566
|
+
path: this.name,
|
|
567
|
+
dataB64: base64Of(bytes),
|
|
568
|
+
...request,
|
|
569
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
570
|
+
});
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
// A SharedWorker host takes a larger object in parts (ADR 0015).
|
|
574
|
+
const { uploadId } = (await this.channel.op({
|
|
575
|
+
method: 'storage.beginUpload',
|
|
576
|
+
path: this.name,
|
|
577
|
+
size: bytes.byteLength,
|
|
578
|
+
...request,
|
|
579
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
580
|
+
}));
|
|
581
|
+
try {
|
|
582
|
+
for (let offset = 0, index = 0; offset < bytes.byteLength; offset += MAX_STORAGE_PART_BYTES, index++) {
|
|
583
|
+
await this.channel.op({
|
|
584
|
+
method: 'storage.putPart',
|
|
585
|
+
uploadId,
|
|
586
|
+
partIndex: index,
|
|
587
|
+
dataB64: base64Of(bytes.subarray(offset, offset + MAX_STORAGE_PART_BYTES)),
|
|
588
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
await this.channel.op({ method: 'storage.finishUpload', uploadId, actAs: STORAGE_REMOTE_ADMIN_LENS });
|
|
592
|
+
}
|
|
593
|
+
catch (err) {
|
|
594
|
+
try {
|
|
595
|
+
await this.channel.op({ method: 'storage.abortUpload', uploadId, actAs: STORAGE_REMOTE_ADMIN_LENS });
|
|
596
|
+
}
|
|
597
|
+
catch {
|
|
598
|
+
// secondary abort best effort
|
|
599
|
+
}
|
|
600
|
+
throw err;
|
|
601
|
+
}
|
|
298
602
|
}
|
|
299
|
-
|
|
300
|
-
|
|
603
|
+
/** The object's bytes, or the inclusive range `start..end`, as they arrive. */
|
|
604
|
+
async *read(options) {
|
|
605
|
+
const metadata = await this.missingAsNoSuchObject(this.channel.op({
|
|
606
|
+
method: 'storage.getMetadata',
|
|
607
|
+
...this.bucket.wire,
|
|
608
|
+
path: this.name,
|
|
609
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
610
|
+
}));
|
|
611
|
+
const route = await this.channel.byteRoute?.();
|
|
612
|
+
const routed = route !== undefined;
|
|
613
|
+
if (!routed) {
|
|
614
|
+
yield* this.readFrames(metadata, options);
|
|
615
|
+
return;
|
|
616
|
+
}
|
|
617
|
+
const response = await this.missingAsNoSuchObject(fetchFromByteRoute(route, {
|
|
618
|
+
bucket: metadata.bucket,
|
|
619
|
+
...this.bucket.wire,
|
|
620
|
+
path: this.name,
|
|
621
|
+
start: options.start,
|
|
622
|
+
end: options.end,
|
|
623
|
+
}));
|
|
624
|
+
const body = response.body;
|
|
625
|
+
if (body === null)
|
|
626
|
+
return;
|
|
627
|
+
const reader = body.getReader();
|
|
301
628
|
try {
|
|
302
|
-
|
|
629
|
+
for (;;) {
|
|
630
|
+
const { done, value } = await reader.read();
|
|
631
|
+
if (done)
|
|
632
|
+
return;
|
|
633
|
+
yield value;
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
finally {
|
|
637
|
+
reader.releaseLock();
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
/** Read from a SharedWorker host, which sends bytes as frames: whole, or in ranged parts (ADR 0015). */
|
|
641
|
+
async *readFrames(metadata, options) {
|
|
642
|
+
if (metadata.size <= MAX_STORAGE_PART_BYTES) {
|
|
643
|
+
const wire = await this.missingAsNoSuchObject(this.channel.op({
|
|
303
644
|
method: 'storage.getBytes',
|
|
645
|
+
...this.bucket.wire,
|
|
304
646
|
path: this.name,
|
|
305
647
|
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
306
648
|
}));
|
|
649
|
+
yield byteRange(Buffer.from(wire.dataB64, 'base64'), options);
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
const last = Math.min(metadata.size, options.end === undefined ? metadata.size : options.end + 1);
|
|
653
|
+
for (let offset = options.start ?? 0; offset < last; offset += MAX_STORAGE_PART_BYTES) {
|
|
654
|
+
const wire = await this.missingAsNoSuchObject(this.channel.op({
|
|
655
|
+
method: 'storage.getBytes',
|
|
656
|
+
...this.bucket.wire,
|
|
657
|
+
path: this.name,
|
|
658
|
+
offset,
|
|
659
|
+
length: Math.min(MAX_STORAGE_PART_BYTES, last - offset),
|
|
660
|
+
expectedGeneration: metadata.generation,
|
|
661
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
662
|
+
}));
|
|
663
|
+
yield Buffer.from(wire.dataB64, 'base64');
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
async getMetadata() {
|
|
667
|
+
const full = await this.missingAsNoSuchObject(this.channel.op({
|
|
668
|
+
method: 'storage.getMetadata',
|
|
669
|
+
...this.bucket.wire,
|
|
670
|
+
path: this.name,
|
|
671
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
672
|
+
}));
|
|
673
|
+
return [fileMetadataOf(full)];
|
|
674
|
+
}
|
|
675
|
+
async setMetadata(metadata) {
|
|
676
|
+
const full = await this.missingAsNoSuchObject(this.channel.op({
|
|
677
|
+
method: 'storage.setMetadata',
|
|
678
|
+
...this.bucket.wire,
|
|
679
|
+
path: this.name,
|
|
680
|
+
patch: patchOf(metadata, { strict: true }),
|
|
681
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
682
|
+
}));
|
|
683
|
+
return [fileMetadataOf(full)];
|
|
684
|
+
}
|
|
685
|
+
/** The host mints the token; the Node host serves the URL, a SharedWorker host has nowhere to. */
|
|
686
|
+
async [DOWNLOAD_URL]() {
|
|
687
|
+
const { path } = await this.missingAsNoSuchObject(this.channel.op({
|
|
688
|
+
method: 'storage.getDownloadURL',
|
|
689
|
+
...this.bucket.wire,
|
|
690
|
+
path: this.name,
|
|
691
|
+
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
692
|
+
}));
|
|
693
|
+
const route = await this.channel.byteRoute?.();
|
|
694
|
+
const routed = route !== undefined;
|
|
695
|
+
if (routed)
|
|
696
|
+
return new URL(path, route.baseUrl).href;
|
|
697
|
+
const [[metadata], [bytes]] = await Promise.all([this.getMetadata(), this.download()]);
|
|
698
|
+
return dataUri(bytes, metadata.contentType);
|
|
699
|
+
}
|
|
700
|
+
/** Mirror the gcs/firebase-admin (and local arm) `No such object` message for a missing file. */
|
|
701
|
+
async missingAsNoSuchObject(pending) {
|
|
702
|
+
try {
|
|
703
|
+
return await pending;
|
|
307
704
|
}
|
|
308
705
|
catch (err) {
|
|
309
|
-
if (isObjectNotFound(err))
|
|
310
|
-
// Mirror the gcs/firebase-admin (and local arm) message shape so
|
|
311
|
-
// consumer catch-blocks that string-match `No such object` work
|
|
312
|
-
// identically across arms.
|
|
706
|
+
if (isObjectNotFound(err))
|
|
313
707
|
throw new Error(`No such object: ${this.bucket.name}/${this.name}`);
|
|
314
|
-
}
|
|
315
708
|
throw err;
|
|
316
709
|
}
|
|
317
|
-
return [Buffer.from(wire.dataB64, 'base64')];
|
|
318
710
|
}
|
|
319
711
|
async delete() {
|
|
320
712
|
try {
|
|
321
713
|
await this.channel.op({
|
|
322
714
|
method: 'storage.deleteObject',
|
|
715
|
+
...this.bucket.wire,
|
|
323
716
|
path: this.name,
|
|
324
717
|
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
325
718
|
});
|
|
@@ -337,6 +730,7 @@ class RemoteFile {
|
|
|
337
730
|
try {
|
|
338
731
|
await this.channel.op({
|
|
339
732
|
method: 'storage.getMetadata',
|
|
733
|
+
...this.bucket.wire,
|
|
340
734
|
path: this.name,
|
|
341
735
|
actAs: STORAGE_REMOTE_ADMIN_LENS,
|
|
342
736
|
});
|
|
@@ -353,31 +747,166 @@ class RemoteFile {
|
|
|
353
747
|
async getSignedUrl(options) {
|
|
354
748
|
return [stubSignedUrl(this.bucket.name, this.name, options)];
|
|
355
749
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
750
|
+
}
|
|
751
|
+
/** A host's metadata in `@google-cloud/storage`'s shape: the download tokens become the custom key production uses. */
|
|
752
|
+
function fileMetadataOf(host) {
|
|
753
|
+
const custom = { ...(host.customMetadata ?? {}) };
|
|
754
|
+
if (host.downloadTokens)
|
|
755
|
+
custom[DOWNLOAD_TOKENS_KEY] = host.downloadTokens;
|
|
756
|
+
const settable = {};
|
|
757
|
+
for (const field of SETTABLE_FIELDS) {
|
|
758
|
+
const value = host[field];
|
|
759
|
+
if (value !== undefined)
|
|
760
|
+
settable[field] = value;
|
|
364
761
|
}
|
|
762
|
+
const hasCustom = Object.keys(custom).length > 0;
|
|
763
|
+
return {
|
|
764
|
+
name: host.fullPath,
|
|
765
|
+
bucket: host.bucket,
|
|
766
|
+
generation: host.generation,
|
|
767
|
+
metageneration: host.metageneration,
|
|
768
|
+
size: String(host.size),
|
|
769
|
+
timeCreated: host.timeCreated,
|
|
770
|
+
updated: host.updated,
|
|
771
|
+
...settable,
|
|
772
|
+
...(host.md5Hash !== undefined ? { md5Hash: host.md5Hash } : {}),
|
|
773
|
+
...(hasCustom ? { metadata: custom } : {}),
|
|
774
|
+
};
|
|
365
775
|
}
|
|
366
776
|
/** Is this relayed error the worker's `storage/object-not-found`? */
|
|
367
777
|
function isObjectNotFound(err) {
|
|
368
778
|
return err?.code === 'storage/object-not-found';
|
|
369
779
|
}
|
|
370
|
-
/** Over-cap rejection (code `
|
|
371
|
-
|
|
372
|
-
function payloadTooLarge(sizeBytes, what) {
|
|
780
|
+
/** Over-cap rejection (code `storage/quota-exceeded`) — mirrors the worker host's message shape. */
|
|
781
|
+
function quotaExceeded(sizeBytes, what) {
|
|
373
782
|
const err = new Error(`pyric-admin/storage: ${what} is ${sizeBytes} bytes — over the ` +
|
|
374
|
-
`${
|
|
375
|
-
|
|
376
|
-
'split the object or keep it under the cap.');
|
|
377
|
-
err.code = 'payload-too-large';
|
|
783
|
+
`${MAX_STORAGE_OBJECT_BYTES / (1024 * 1024)} MiB maximum storage object cap (MAX_STORAGE_OBJECT_BYTES).`);
|
|
784
|
+
err.code = 'storage/quota-exceeded';
|
|
378
785
|
return err;
|
|
379
786
|
}
|
|
380
787
|
// ─── Helpers ────────────────────────────────────────────────────────────
|
|
788
|
+
/**
|
|
789
|
+
* The File a copy writes: a path in the source's bucket, a `gs://` URL, the
|
|
790
|
+
* source's path in another Bucket, or a File.
|
|
791
|
+
*/
|
|
792
|
+
function copyTargetOf(source, destination, storage) {
|
|
793
|
+
if (typeof destination === 'string') {
|
|
794
|
+
const url = /^gs:\/\/([^/]+)\/(.+)$/.exec(destination);
|
|
795
|
+
return url === null ? source.bucket.file(destination) : storage.bucket(url[1]).file(url[2]);
|
|
796
|
+
}
|
|
797
|
+
const isFile = 'save' in destination;
|
|
798
|
+
return isFile ? destination : destination.file(source.name);
|
|
799
|
+
}
|
|
800
|
+
/** {@link CopyOptions} as the metadata `save` takes: settable fields, and custom keys under `metadata`. */
|
|
801
|
+
function copyMetadataOf(options) {
|
|
802
|
+
const { metadata, ...settable } = options;
|
|
803
|
+
return { ...settable, ...(metadata !== undefined ? { metadata } : {}) };
|
|
804
|
+
}
|
|
805
|
+
/** What an upload sends besides its bytes: settable fields and custom metadata, with the download tokens apart. */
|
|
806
|
+
function uploadRequestOf(options) {
|
|
807
|
+
const saved = options.metadata === undefined ? undefined : savedMetadataOf(options.metadata);
|
|
808
|
+
const hasCustom = saved !== undefined && Object.keys(saved.custom).length > 0;
|
|
809
|
+
return {
|
|
810
|
+
...(options.contentType !== undefined ? { contentType: options.contentType } : {}),
|
|
811
|
+
...(saved !== undefined ? { metadata: { ...saved.settable, ...(hasCustom ? { metadata: saved.custom } : {}) } } : {}),
|
|
812
|
+
...(saved?.downloadTokens !== undefined ? { downloadTokens: saved.downloadTokens } : {}),
|
|
813
|
+
};
|
|
814
|
+
}
|
|
815
|
+
/**
|
|
816
|
+
* Split `@google-cloud/storage`-shaped metadata into a {@link MetadataPatch}.
|
|
817
|
+
* `setMetadata` is strict and refuses fields the sandbox does not keep;
|
|
818
|
+
* `save` has always stored what it was given and ignores them.
|
|
819
|
+
*/
|
|
820
|
+
function patchOf(metadata, { strict }) {
|
|
821
|
+
const source = metadata;
|
|
822
|
+
const unsupported = Object.keys(source).filter(key => key !== 'metadata' && !SETTABLE_FIELDS.includes(key));
|
|
823
|
+
const refused = strict && unsupported.length > 0;
|
|
824
|
+
if (refused) {
|
|
825
|
+
throw new Error(`not implemented in pyric-admin/storage sandbox backend: setMetadata of ${unsupported.join(', ')}. ` +
|
|
826
|
+
`The sandbox keeps ${SETTABLE_FIELDS.join(', ')}, and custom metadata under \`metadata\`.`);
|
|
827
|
+
}
|
|
828
|
+
const patch = { settable: {}, customMetadata: {} };
|
|
829
|
+
for (const field of SETTABLE_FIELDS) {
|
|
830
|
+
const value = source[field];
|
|
831
|
+
if (typeof value === 'string')
|
|
832
|
+
patch.settable[field] = value;
|
|
833
|
+
}
|
|
834
|
+
const custom = source.metadata;
|
|
835
|
+
const hasCustom = custom !== null && typeof custom === 'object';
|
|
836
|
+
if (!hasCustom)
|
|
837
|
+
return patch;
|
|
838
|
+
for (const [key, value] of Object.entries(custom)) {
|
|
839
|
+
if (value === undefined)
|
|
840
|
+
continue;
|
|
841
|
+
const next = value === null ? null : String(value);
|
|
842
|
+
if (key === DOWNLOAD_TOKENS_KEY)
|
|
843
|
+
patch.downloadTokens = next;
|
|
844
|
+
else
|
|
845
|
+
patch.customMetadata[key] = next;
|
|
846
|
+
}
|
|
847
|
+
return patch;
|
|
848
|
+
}
|
|
849
|
+
/**
|
|
850
|
+
* The metadata `save` and `createWriteStream` create an object with. It is the
|
|
851
|
+
* {@link patchOf} split `setMetadata` uses, so `firebaseStorageDownloadTokens`
|
|
852
|
+
* becomes the object's download tokens on every path; a `null` key is not stored.
|
|
853
|
+
*/
|
|
854
|
+
function savedMetadataOf(metadata) {
|
|
855
|
+
const patch = patchOf(metadata, { strict: false });
|
|
856
|
+
const custom = {};
|
|
857
|
+
for (const [key, value] of Object.entries(patch.customMetadata))
|
|
858
|
+
if (value !== null)
|
|
859
|
+
custom[key] = value;
|
|
860
|
+
const tokens = patch.downloadTokens;
|
|
861
|
+
return { settable: patch.settable, custom, ...(typeof tokens === 'string' ? { downloadTokens: tokens } : {}) };
|
|
862
|
+
}
|
|
863
|
+
/** A `data:` URI carrying `bytes`, the download URL where no host serves them over HTTP. */
|
|
864
|
+
function dataUri(bytes, contentType) {
|
|
865
|
+
return `data:${contentType ?? 'application/octet-stream'};base64,${base64Of(bytes)}`;
|
|
866
|
+
}
|
|
867
|
+
/** `bytes` as base64, for a frame. */
|
|
868
|
+
function base64Of(bytes) {
|
|
869
|
+
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('base64');
|
|
870
|
+
}
|
|
871
|
+
/** The inclusive range `start..end` of `bytes`, as `@google-cloud/storage` reads it. */
|
|
872
|
+
function byteRange(bytes, options) {
|
|
873
|
+
const end = options.end === undefined ? undefined : options.end + 1;
|
|
874
|
+
return bytes.subarray(options.start ?? 0, end);
|
|
875
|
+
}
|
|
876
|
+
/**
|
|
877
|
+
* A writable stream that spools its bytes to a temporary file and, once it
|
|
878
|
+
* finishes, hands that file to `commit`. The file is removed either way.
|
|
879
|
+
*/
|
|
880
|
+
function spooledWriteStream(commit) {
|
|
881
|
+
let directory;
|
|
882
|
+
let sink;
|
|
883
|
+
return new Writable({
|
|
884
|
+
construct(callback) {
|
|
885
|
+
mkdtemp(join(tmpdir(), 'pyric-admin-upload-')).then(created => {
|
|
886
|
+
directory = created;
|
|
887
|
+
sink = createFileWriteStream(join(created, 'object'));
|
|
888
|
+
callback();
|
|
889
|
+
}, callback);
|
|
890
|
+
},
|
|
891
|
+
write(chunk, _encoding, callback) {
|
|
892
|
+
sink.write(chunk, callback);
|
|
893
|
+
},
|
|
894
|
+
final(callback) {
|
|
895
|
+
sink.end(() => {
|
|
896
|
+
commit(join(directory, 'object')).then(() => callback(), callback);
|
|
897
|
+
});
|
|
898
|
+
},
|
|
899
|
+
destroy(error, callback) {
|
|
900
|
+
sink?.destroy();
|
|
901
|
+
const created = directory;
|
|
902
|
+
if (created === undefined) {
|
|
903
|
+
callback(error);
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
rm(created, { recursive: true, force: true }).finally(() => callback(error));
|
|
907
|
+
},
|
|
908
|
+
});
|
|
909
|
+
}
|
|
381
910
|
/**
|
|
382
911
|
* The deterministic sandbox signed-URL stub, shared by the local and remote
|
|
383
912
|
* arms so their output is byte-identical (the URL is never served — it's a
|
|
@@ -418,4 +947,14 @@ function normalizeExpires(expires) {
|
|
|
418
947
|
// strictness because the value only feeds the deterministic sandbox stub.
|
|
419
948
|
return new Date(expires).getTime();
|
|
420
949
|
}
|
|
950
|
+
// ─── Deferred members ────────────────────────────────────────────────────
|
|
951
|
+
//
|
|
952
|
+
// The `Bucket` and `File` methods neither arm models fail with an error that
|
|
953
|
+
// names the method instead of reading as `undefined`. See deferred-members.ts.
|
|
954
|
+
for (const bucket of [SandboxBucket, RemoteBucket]) {
|
|
955
|
+
defineDeferredMembers(bucket.prototype, 'pyric-admin/storage', 'Bucket', BUCKET_DEFERRED_MEMBERS);
|
|
956
|
+
}
|
|
957
|
+
for (const file of [SandboxFile, RemoteFile]) {
|
|
958
|
+
defineDeferredMembers(file.prototype, 'pyric-admin/storage', 'File', FILE_DEFERRED_MEMBERS);
|
|
959
|
+
}
|
|
421
960
|
//# sourceMappingURL=index.js.map
|