sealkeep 0.9.0 → 0.11.1

package/dist/src/providers/gdrive.js

@@ -1,8 +1,9 @@
 import { createHash, randomBytes } from "node:crypto";
 import { spawn } from "node:child_process";
 import { createServer } from "node:http";
-import { fail, VaultlineError } from "../errors.js";
+import { fail, SealkeepError } from "../errors.js";
 import { envVar } from "../env.js";
+import { collectChunks, oneChunk, responseBodyChunks } from "../byte-stream.js";
 /**
  * Google Drive as a bring-your-own storage provider (Drive API v3).
  *
@@ -21,7 +22,7 @@ import { envVar } from "../env.js";
  */
 export const GDRIVE_SCOPE = "https://www.googleapis.com/auth/drive.file";
 /** Placeholder until a first-party client id ships. Point SEALKEEP_GDRIVE_CLIENT_ID at your own Desktop-app OAuth client to connect today. */
-export const GDRIVE_CLIENT_ID_PLACEHOLDER = "000000000000-vaultline-placeholder.apps.googleusercontent.com";
+export const GDRIVE_CLIENT_ID_PLACEHOLDER = "000000000000-sealkeep-placeholder.apps.googleusercontent.com";
 export function gdriveClientId(env = process.env) {
     return envVar("GDRIVE_CLIENT_ID", env)?.trim() || GDRIVE_CLIENT_ID_PLACEHOLDER;
 }
@@ -69,6 +70,34 @@ export async function exchangeCodeForTokens(input) {
         ?? fail("unauthorized", "Google returned no refresh token. Remove Sealkeep at https://myaccount.google.com/permissions and connect again.");
     return { refreshToken, accessToken: data.access_token ?? "", expiresIn: data.expires_in ?? 0 };
 }
+/**
+ * Resolve the stable Drive identity represented by an access token.
+ *
+ * `about.get(fields=user(permissionId))` is available under the existing
+ * `drive.file` grant.  A permission id is an opaque grantee identifier, not a
+ * credential and not personally-readable account metadata, so it is safe to
+ * retain beside the refresh token and include in destination fingerprints.
+ */
+export async function gdriveAccountId(accessToken, options = {}) {
+    const token = accessToken.trim();
+    if (!token)
+        fail("unauthorized", "Google returned no access token, so the connected Drive account could not be identified.");
+    const url = new URL("/drive/v3/about", options.apiBase ?? GDRIVE_API_BASE);
+    url.searchParams.set("fields", "user(permissionId)");
+    const response = await (options.fetchImpl ?? fetch)(url, {
+        headers: { authorization: `Bearer ${token}` },
+    });
+    if (!response.ok) {
+        const code = response.status === 401 || response.status === 403 ? "unauthorized" : "internal";
+        fail(code, `Google Drive could not identify the connected account: ${response.status} ${response.statusText}`, { status: response.status });
+    }
+    const data = await response.json();
+    const accountId = typeof data.user?.permissionId === "string" ? data.user.permissionId.trim() : "";
+    if (!accountId || accountId.length > 512 || /[\u0000-\u001f\u007f]/.test(accountId)) {
+        fail("unauthorized", "Google Drive returned no usable account identity. Disconnect and reconnect the Drive target.");
+    }
+    return accountId;
+}
 /**
  * Fire-and-forget platform opener. Failing to open a browser is never failing
  * to connect — the consent URL is reported to the caller either way — so this
@@ -108,7 +137,7 @@ export async function connectGdrive(options = {}) {
     const consentUrl = gdriveConsentUrl({ clientId, redirectUri, challenge, state, authUrl: options.authUrl });
     let timer;
     const code = new Promise((resolve, reject) => {
-        timer = setTimeout(() => reject(new VaultlineError("unauthorized", `No response from the browser within ${Math.round((options.timeoutMs ?? 300_000) / 1000)}s. Run the connect command again.`)), options.timeoutMs ?? 300_000);
+        timer = setTimeout(() => reject(new SealkeepError("unauthorized", `No response from the browser within ${Math.round((options.timeoutMs ?? 300_000) / 1000)}s. Run the connect command again.`)), options.timeoutMs ?? 300_000);
         server.on("request", (request, response) => {
             const url = new URL(request.url ?? "/", `http://127.0.0.1:${port}`);
             if (url.pathname !== "/callback") {
@@ -119,7 +148,7 @@ export async function connectGdrive(options = {}) {
             const deny = (message) => {
                 response.writeHead(400, { "content-type": "text/html; charset=utf-8" });
                 response.end("<p>Sealkeep could not finish connecting. Return to the terminal.</p>");
-                reject(new VaultlineError("unauthorized", message));
+                reject(new SealkeepError("unauthorized", message));
             };
             if (url.searchParams.get("error"))
                 return deny(`Google reported: ${url.searchParams.get("error")}`);
@@ -145,7 +174,8 @@ export async function connectGdrive(options = {}) {
         await Promise.resolve((options.opener ?? launchBrowser)(consentUrl)).catch(() => { });
         const authorizationCode = await code;
         const tokens = await exchangeCodeForTokens({ code: authorizationCode, verifier, clientId, redirectUri, tokenUrl: options.tokenUrl, fetchImpl: options.fetchImpl });
-        return { kind: "gdrive", refreshToken: tokens.refreshToken, clientId };
+        const accountId = await (options.accountIdResolver ?? gdriveAccountId)(tokens.accessToken);
+        return { kind: "gdrive", refreshToken: tokens.refreshToken, clientId, accountId };
     }
     finally {
         if (timer)
@@ -159,7 +189,7 @@ const FOLDER_MIME = "application/vnd.google-apps.folder";
 /**
  * Drive upload client.
  *
- * A lease's objectKey (e.g. `vaultline/<id>.vlarchive`) has no path meaning in
+ * A lease's objectKey (e.g. `vaultline/<id>.skarchive`) has no path meaning in
  * Drive, so the mapping is: files live flat in one folder (or the hidden
  * appDataFolder space), named by the key's basename, and each carries
  * `appProperties.vaultlineKey = <full objectKey>`. head/download locate by that
@@ -198,7 +228,7 @@ export class GdriveUploadClient {
     tokenUrl() { return this.endpoint.host ? `${this.base()}/token` : GDRIVE_TOKEN_URL; }
     space() { return this.endpoint.appData ? "appDataFolder" : "drive"; }
     /** Access token minted from the refresh token, cached until shortly before expiry so it never dies mid-chunk. */
-    async token() {
+    async token(signal) {
         if (this.accessToken && this.now() < this.accessToken.expiresAt)
             return this.accessToken.token;
         // A cloud-exchanged credential refreshes through the account's server —
@@ -211,7 +241,8 @@ export class GdriveUploadClient {
         const response = await this.fetchImpl(this.tokenUrl(), {
             method: "POST",
             headers: { "content-type": "application/x-www-form-urlencoded" },
-            body: new URLSearchParams({ grant_type: "refresh_token", refresh_token: this.credentials.refreshToken, client_id: this.credentials.clientId }).toString()
+            body: new URLSearchParams({ grant_type: "refresh_token", refresh_token: this.credentials.refreshToken, client_id: this.credentials.clientId }).toString(),
+            signal,
         });
         if (!response.ok)
             fail("unauthorized", `Google Drive refused to mint an access token (${response.status}). Reconnect with: sealkeep storage connect gdrive`, { status: response.status });
@@ -221,40 +252,55 @@ export class GdriveUploadClient {
         return token;
     }
     async api(path, init = {}) {
-        const token = await this.token();
+        const token = await this.token(init.signal);
         return this.fetchImpl(`${this.base()}${path}`, { ...init, headers: { authorization: `Bearer ${token}`, ...init.headers } });
     }
-    async list(query) {
+    async list(query, signal) {
         const params = new URLSearchParams({ q: query, spaces: this.space(), fields: "files(id,name,size,md5Checksum)", pageSize: "10" });
-        const response = await this.api(`/drive/v3/files?${params.toString()}`);
+        const response = await this.api(`/drive/v3/files?${params.toString()}`, { signal });
         if (!response.ok)
             fail("internal", `Google Drive file lookup failed: ${response.status} ${response.statusText}`, { status: response.status });
         const data = await response.json();
         return data.files ?? [];
     }
-    /** Exact lookup by the appProperties key. Two files claiming one key is a state we refuse to guess about. */
-    async findByKey(objectKey) {
-        const files = await this.list(`appProperties has { key = 'vaultlineKey' and value = '${escapeQuery(objectKey)}' } and trashed = false`);
+    /**
+     * Exact lookup inside this configured physical destination. Object keys are
+     * intentionally reusable across Drive folders, so the immutable parent id
+     * and Drive space are part of every lookup rather than merely creation
+     * metadata.
+     */
+    async findByKey(objectKey, signal, knownParentId) {
+        const parentId = knownParentId ?? await this.parentId(signal, false);
+        if (!parentId)
+            return null;
+        const files = await this.list(`appProperties has { key = 'vaultlineKey' and value = '${escapeQuery(objectKey)}' } and '${escapeQuery(parentId)}' in parents and trashed = false`, signal);
         if (files.length > 1)
             fail("internal", `Google Drive holds ${files.length} files claiming ${objectKey}; refusing to pick one`, { objectKey, count: files.length });
         return files[0] ?? null;
     }
-    /** Finds or creates the archive folder; the id is cached per instance. */
-    async parentId() {
+    /** Resolves the archive folder to its immutable Drive id, creating it only for uploads. */
+    async parentId(signal, create = true) {
         if (this.endpoint.appData)
             return "appDataFolder";
         if (this.folderId)
             return this.folderId;
         const name = this.endpoint.folderName;
-        const existing = (await this.list(`name = '${escapeQuery(name)}' and mimeType = '${FOLDER_MIME}' and trashed = false`))[0]?.id;
+        const folders = await this.list(`name = '${escapeQuery(name)}' and mimeType = '${FOLDER_MIME}' and trashed = false`, signal);
+        if (folders.length > 1) {
+            fail("internal", `Google Drive holds ${folders.length} folders named "${name}"; refusing to guess which configured destination owns the archive`, { folderName: name, count: folders.length });
+        }
+        const existing = folders[0]?.id;
         if (existing) {
             this.folderId = existing;
             return existing;
         }
+        if (!create)
+            return null;
         const response = await this.api("/drive/v3/files?fields=id", {
             method: "POST",
             headers: { "content-type": "application/json; charset=UTF-8" },
-            body: JSON.stringify({ name, mimeType: FOLDER_MIME })
+            body: JSON.stringify({ name, mimeType: FOLDER_MIME }),
+            signal,
         });
         if (!response.ok)
             fail("internal", `Could not create the "${name}" folder in Google Drive: ${response.status} ${response.statusText}`, { status: response.status });
@@ -268,42 +314,147 @@ export class GdriveUploadClient {
      * appProperties lookup stops being exact). The session URI is a bearer
      * capability: used immediately, never persisted, never logged.
      */
-    async initiateResumable(objectKey, existingFileId, totalBytes) {
+    async initiateResumable(objectKey, parentId, existingFileId, totalBytes, signal) {
         const metadata = existingFileId
             ? { appProperties: { vaultlineKey: objectKey } }
-            : { name: objectKey.split("/").pop() ?? objectKey, parents: [await this.parentId()], appProperties: { vaultlineKey: objectKey } };
+            : { name: objectKey.split("/").pop() ?? objectKey, parents: [parentId], appProperties: { vaultlineKey: objectKey } };
         const response = await this.api(existingFileId ? `/upload/drive/v3/files/${existingFileId}?uploadType=resumable` : "/upload/drive/v3/files?uploadType=resumable", {
             method: existingFileId ? "PATCH" : "POST",
-            headers: { "content-type": "application/json; charset=UTF-8", "x-upload-content-length": String(totalBytes) },
-            body: JSON.stringify(metadata)
+            headers: {
+                "content-type": "application/json; charset=UTF-8",
+                "x-upload-content-length": String(totalBytes),
+                "x-upload-content-type": "application/octet-stream",
+            },
+            body: JSON.stringify(metadata),
+            signal,
         });
         if (!response.ok)
             fail("internal", `Could not start a Drive resumable upload for ${objectKey}: ${response.status} ${response.statusText}`, { objectKey, status: response.status });
         return response.headers.get("location") ?? fail("internal", `Drive's resumable initiation for ${objectKey} returned no session URI`, { objectKey });
     }
     async upload(lease, ciphertext) {
-        const objectKey = lease.objectKey;
-        const existing = await this.findByKey(objectKey);
-        const session = await this.initiateResumable(objectKey, existing?.id ?? null, ciphertext.length);
-        for (let offset = 0; offset < Math.max(ciphertext.length, 1); offset += this.chunkBytes) {
-            const slice = ciphertext.subarray(offset, Math.min(offset + this.chunkBytes, ciphertext.length));
-            const last = offset + slice.length >= ciphertext.length;
-            const range = ciphertext.length === 0 ? "bytes */0" : `bytes ${offset}-${offset + slice.length - 1}/${ciphertext.length}`;
-            const response = await this.fetchImpl(session, { method: "PUT", body: new Uint8Array(slice), headers: { "content-range": range } });
-            // 308 is "resume incomplete", the expected status between chunks.
-            if (!response.ok && !(response.status === 308 && !last)) {
-                fail("internal", `Drive resumable upload for ${objectKey} failed at ${range}: ${response.status} ${response.statusText}`, { objectKey, status: response.status });
+        return this.uploadStream(lease.objectKey, oneChunk(ciphertext), { bytes: ciphertext.length });
+    }
+    /**
+     * Uploads a known-length object through Drive's resumable protocol while
+     * retaining at most one configured resumable part. The archive pipeline
+     * measures a file in a bounded first pass, then this second pass computes
+     * Drive's whole-object MD5 as bytes are copied into fixed-size PUTs.
+     *
+     * Drive requires every non-final chunk to be a positive multiple of 256 KiB.
+     * The configured `chunkBytes` already enforces that rule; the last PUT may be
+     * short. A 308 response is accepted only for a non-final PUT, and the final
+     * PUT must be a success, so an early-finalized or half-accepted session can
+     * never be reported as complete.
+     */
+    async uploadStream(objectKey, chunks, options = {}) {
+        const totalBytes = options.bytes;
+        if (!Number.isSafeInteger(totalBytes) || (totalBytes ?? -1) < 0) {
+            fail("invalid_argument", `Google Drive needs the archive's exact byte length before a resumable upload can start`, { objectKey, bytes: totalBytes });
+        }
+        const total = totalBytes;
+        options.signal?.throwIfAborted();
+        const parentId = await this.parentId(options.signal, true)
+            ?? fail("internal", `Could not resolve the configured Drive destination for ${objectKey}`, { objectKey });
+        const existing = await this.findByKey(objectKey, options.signal, parentId);
+        options.signal?.throwIfAborted();
+        let session = await this.initiateResumable(objectKey, parentId, existing?.id ?? null, total, options.signal);
+        const md5 = createHash("md5");
+        // One fixed part, reused only after fetch has consumed the preceding body.
+        // Input chunks are copied piecemeal, so even a badly shaped producer cannot
+        // make this client concatenate the whole archive internally.
+        const part = Buffer.allocUnsafe(Math.min(this.chunkBytes, Math.max(total, 1)));
+        let partBytes = 0;
+        let consumed = 0;
+        let sent = 0;
+        const putPart = async (piece, final) => {
+            const partStart = sent;
+            const partEnd = partStart + piece.length;
+            let attemptsWithoutProgress = 0;
+            do {
+                options.signal?.throwIfAborted();
+                const bodyOffset = sent - partStart;
+                const body = piece.subarray(bodyOffset);
+                const range = total === 0 ? "bytes */0" : `bytes ${sent}-${partEnd - 1}/${total}`;
+                const response = await this.fetchImpl(session, {
+                    method: "PUT",
+                    // This is a view over the fixed part buffer, not another part-sized copy.
+                    body: new Uint8Array(body.buffer, body.byteOffset, body.byteLength),
+                    headers: { "content-length": String(body.length), "content-range": range },
+                    redirect: "manual",
+                    signal: options.signal,
+                });
+                const movedSession = response.headers.get("location");
+                if (movedSession)
+                    session = movedSession;
+                if (response.ok) {
+                    if (!final) {
+                        fail("internal", `Drive finalized the resumable upload for ${objectKey} before its declared final byte`, { objectKey, status: response.status, sentBytes: partEnd, expectedBytes: total });
+                    }
+                    sent = partEnd;
+                    return;
+                }
+                if (response.status !== 308) {
+                    fail("internal", `Drive resumable upload for ${objectKey} failed at ${range}: ${response.status} ${response.statusText}`, { objectKey, status: response.status });
+                }
+                // Drive's 308 Range is the authoritative committed prefix. It can be
+                // shorter than the attempted PUT after a connection interruption, in
+                // which case the still-retained suffix is sent again from that offset.
+                const acknowledged = /^bytes=0-(\d+)$/.exec(response.headers.get("range") ?? "");
+                const confirmed = acknowledged ? Number(acknowledged[1]) + 1 : 0;
+                if (!Number.isSafeInteger(confirmed) || confirmed < sent || confirmed > partEnd) {
+                    fail("internal", `Drive returned an invalid committed-byte range while uploading ${objectKey}`, { objectKey, range: response.headers.get("range"), sentBytes: sent, partEnd });
+                }
+                if (confirmed === sent)
+                    attemptsWithoutProgress += 1;
+                else
+                    attemptsWithoutProgress = 0;
+                if (attemptsWithoutProgress >= 3) {
+                    fail("internal", `Drive made no progress after three resumable PUT attempts for ${objectKey}`, { objectKey, sentBytes: sent });
+                }
+                sent = confirmed;
+                if (sent === partEnd) {
+                    if (final) {
+                        fail("internal", `Drive kept the final resumable PUT for ${objectKey} incomplete after acknowledging every byte`, { objectKey, sentBytes: sent });
+                    }
+                    return;
+                }
+            } while (sent < partEnd || (final && total === 0));
+        };
+        for await (const chunk of chunks) {
+            options.signal?.throwIfAborted();
+            if (chunk.length === 0)
+                continue;
+            consumed += chunk.length;
+            if (consumed > total) {
+                fail("ciphertext_integrity_failed", `The stream for ${objectKey} produced more than its declared ${total} bytes`, { objectKey, expectedBytes: total, consumedBytes: consumed });
             }
-            if (last)
-                break;
+            md5.update(chunk);
+            let offset = 0;
+            while (offset < chunk.length) {
+                // A full part is held until another source byte proves it is non-final.
+                // That prevents a generator which later throws or overruns its declared
+                // length from finalizing a truncated object first.
+                if (partBytes === part.length) {
+                    await putPart(part.subarray(0, partBytes), false);
+                    partBytes = 0;
+                }
+                const copied = Math.min(part.length - partBytes, chunk.length - offset);
+                chunk.copy(part, partBytes, offset, offset + copied);
+                partBytes += copied;
+                offset += copied;
+            }
+        }
+        if (consumed !== total) {
+            fail("ciphertext_integrity_failed", `The stream for ${objectKey} ended after ${consumed} of its declared ${total} bytes`, { objectKey, expectedBytes: total, consumedBytes: consumed });
+        }
+        await putPart(part.subarray(0, partBytes), true);
+        if (sent !== total) {
+            fail("ciphertext_integrity_failed", `Drive accepted ${sent} of ${total} bytes for ${objectKey}`, { objectKey, expectedBytes: total, sentBytes: sent });
         }
-        // The whole-object proof Drive reports back is `md5Checksum` in file
-        // metadata, as lowercase hex — it computes no SHA-256, and unlike GCS's
-        // base64 header this one is hex. The verifier in upload.ts compares this
-        // value against head()'s, so both sides must speak that same hex dialect;
-        // MD5 here is transport proof, not trust — the ciphertext is
-        // AEAD-authenticated and restore re-checks its SHA-256 independently.
-        return { remoteChecksum: createHash("md5").update(ciphertext).digest("hex"), bytes: ciphertext.length };
+        // Drive reports binary-object MD5 as lowercase hex. This is transport proof,
+        // not trust: ciphertext is AEAD-authenticated and restore verifies SHA-256.
+        return { remoteChecksum: md5.digest("hex"), bytes: consumed };
     }
     async head(lease) {
         const file = await this.findByKey(lease.objectKey);
@@ -315,22 +466,44 @@ export class GdriveUploadClient {
         return { exists: true, bytes: Number(file.size ?? "0"), checksum: file.md5Checksum };
     }
     /** Reads an object back for restore or offload-proof: locate by key, then `alt=media`. */
-    /** Removes the file holding this object key. Absent already means done. */
+    /**
+     * Removes the file holding this object key in this exact destination. An
+     * empty successful scoped listing proves absence. A DELETE 404 is rechecked
+     * in the same scope before callers may discard their ledger metadata.
+     */
     async deleteByKey(objectKey) {
-        const existing = await this.findByKey(objectKey);
+        const parentId = await this.parentId(undefined, false);
+        if (!parentId) {
+            fail("internal", `The configured Drive folder "${this.endpoint.folderName}" cannot be found, so absence of ${objectKey} cannot be proven. Nothing was changed here.`, { objectKey, folderName: this.endpoint.folderName });
+        }
+        const existing = await this.findByKey(objectKey, undefined, parentId);
         if (!existing)
-            return;
+            return "absent";
         const response = await this.api(`/drive/v3/files/${existing.id}`, { method: "DELETE" });
-        if (!response.ok && response.status !== 404) {
+        if (response.ok) {
+            if (!await this.findByKey(objectKey, undefined, parentId))
+                return "deleted";
+            fail("internal", `Drive accepted deletion of ${objectKey}, but an object with that key is still present in the configured destination. Nothing was changed here.`, { objectKey });
+        }
+        if (response.status === 404) {
+            if (!await this.findByKey(objectKey, undefined, parentId))
+                return "absent";
+            fail("internal", `Drive reported ${objectKey} missing during deletion, but it is still present in the configured destination. Nothing was changed here.`, { objectKey });
+        }
+        if (!response.ok) {
             fail("internal", `Drive refused the delete: ${response.status} ${response.statusText}. The stored copy may still be there.`);
         }
+        return "deleted";
+    }
+    async download(lease, options = {}) {
+        return collectChunks(await this.downloadStream(lease, options), Number.MAX_SAFE_INTEGER);
     }
-    async download(lease) {
-        const file = await this.findByKey(lease.objectKey)
+    async downloadStream(lease, options = {}) {
+        const file = await this.findByKey(lease.objectKey, options.signal)
             ?? fail("archive_not_found", `Google Drive holds no file for ${lease.objectKey}`, { key: lease.objectKey });
-        const response = await this.api(`/drive/v3/files/${file.id}?alt=media`);
+        const response = await this.api(`/drive/v3/files/${file.id}?alt=media`, { signal: options.signal });
         if (!response.ok)
             fail("internal", `Could not download ${lease.objectKey}: ${response.status} ${response.statusText}`, { key: lease.objectKey, status: response.status });
-        return Buffer.from(await response.arrayBuffer());
+        return responseBodyChunks(response);
     }
 }

package/dist/src/providers/index.d.ts

@@ -3,6 +3,7 @@ import { type ProviderConfig, type ProviderKind, type ProviderUploadClient, type
 import { S3UploadClient } from "./s3.js";
 import { GcsUploadClient } from "./gcs.js";
 import { GdriveUploadClient } from "./gdrive.js";
+import { type StorageFetch } from "./safe-storage-fetch.js";
 export type EndpointOverrides = {
     host?: string;
     port?: number;
@@ -10,6 +11,8 @@ export type EndpointOverrides = {
     pathStyle?: boolean;
     partBytes?: number;
     allowSmallParts?: boolean;
+    /** Runtime-only test/transport seam. Never persisted with routing metadata. */
+    fetchImpl?: StorageFetch;
 };
 /**
  * "gdrive" rides alongside control-plane's ProviderKind rather than inside it:
@@ -21,10 +24,15 @@ export type UploadProviderKind = ProviderKind | "gdrive";
 export type UploadProviderConfig = Omit<ProviderConfig, "provider"> & {
     provider: UploadProviderKind;
 };
+/** One canonical rule for selecting Drive's hidden per-app storage space. */
+export declare function isGdriveAppDataPrefix(prefix: string | undefined): boolean;
+/** The one origin a custom endpoint client may contact. */
+export declare function storageFetchForOverrides(overrides: EndpointOverrides, env?: NodeJS.ProcessEnv): StorageFetch | undefined;
 /**
- * Builds the client for a configured provider. This is the only place a credential
- * is handed to a network client, and the feature flag is checked here rather than
- * at import time so an operator can turn uploads off without a restart.
+ * Builds the client for a configured provider. This is the only place a
+ * credential is handed to a network client, and capability is checked per
+ * call: an ordinary CLI needs explicit environment permission while a daemon
+ * may use only the grant held for its own lifetime.
  */
 export declare function createUploadClient(config: UploadProviderConfig, credentials: ProviderCredentials, overrides?: EndpointOverrides, env?: NodeJS.ProcessEnv): ProviderUploadClient;
 export declare function uploadClientFromStore(dataDir: string, storageConfigId: string, config: UploadProviderConfig, overrides?: EndpointOverrides, backend?: BackendName): Promise<ProviderUploadClient>;

package/dist/src/providers/index.js

@@ -5,36 +5,56 @@ import { DEFAULT_LEASE_TTL_MS, signerEnabled } from "../control-plane.js";
 import { S3UploadClient } from "./s3.js";
 import { GcsUploadClient } from "./gcs.js";
 import { GdriveUploadClient } from "./gdrive.js";
+import { envFlag } from "../env.js";
+import { createPinnedStorageFetch } from "./safe-storage-fetch.js";
+/** One canonical rule for selecting Drive's hidden per-app storage space. */
+export function isGdriveAppDataPrefix(prefix) {
+    return (prefix ?? "").replace(/^\/+|\/+$/g, "") === "appdata";
+}
 function isAwsStyle(credentials) {
     return "accessKeyId" in credentials;
 }
 function isGdriveCredentials(credentials) {
     return "kind" in credentials && credentials.kind === "gdrive";
 }
+/** The one origin a custom endpoint client may contact. */
+export function storageFetchForOverrides(overrides, env = process.env) {
+    if (overrides.fetchImpl)
+        return overrides.fetchImpl;
+    if (!overrides.host)
+        return undefined;
+    const host = overrides.host.includes(":") && !overrides.host.startsWith("[")
+        ? `[${overrides.host}]`
+        : overrides.host;
+    const origin = `${overrides.protocol ?? "https"}://${host}${overrides.port ? `:${overrides.port}` : ""}`;
+    return createPinnedStorageFetch(origin, { allowPrivate: envFlag("ALLOW_PRIVATE_STORAGE_ENDPOINT", env) });
+}
 /**
- * Builds the client for a configured provider. This is the only place a credential
- * is handed to a network client, and the feature flag is checked here rather than
- * at import time so an operator can turn uploads off without a restart.
+ * Builds the client for a configured provider. This is the only place a
+ * credential is handed to a network client, and capability is checked per
+ * call: an ordinary CLI needs explicit environment permission while a daemon
+ * may use only the grant held for its own lifetime.
  */
-export function createUploadClient(config, credentials, overrides = {}, env = process.env) {
+export function createUploadClient(config, credentials, overrides = {}, env) {
     if (!signerEnabled(env))
         fail("signer_not_configured", "Uploads are disabled. Set SEALKEEP_ENABLE_SIGNER=1 to enable a configured provider client.", { provider: config.provider });
+    const fetchImpl = storageFetchForOverrides(overrides, env ?? process.env);
     if (config.provider === "gdrive") {
         if (!isGdriveCredentials(credentials))
             fail("invalid_argument", "Google Drive needs an OAuth credential. Run: sealkeep storage connect gdrive");
         // Drive has no buckets: the configured bucket names the archive folder, and
         // a prefix of "appdata" selects the hidden per-app appDataFolder space.
-        return new GdriveUploadClient({ folderName: config.bucket, appData: config.prefix.replace(/^\/+|\/+$/g, "") === "appdata", host: overrides.host, port: overrides.port, protocol: overrides.protocol }, credentials);
+        return new GdriveUploadClient({ folderName: config.bucket, appData: isGdriveAppDataPrefix(config.prefix), host: overrides.host, port: overrides.port, protocol: overrides.protocol }, credentials, fetchImpl ? { fetchImpl } : {});
     }
     if (config.provider === "gcs") {
         if (isAwsStyle(credentials) || isGdriveCredentials(credentials))
             fail("invalid_argument", "GCS needs a service-account credential, not an access key or OAuth token");
-        return new GcsUploadClient({ bucket: config.bucket, host: overrides.host, port: overrides.port, protocol: overrides.protocol }, credentials);
+        return new GcsUploadClient({ bucket: config.bucket, host: overrides.host, port: overrides.port, protocol: overrides.protocol }, credentials, undefined, fetchImpl);
     }
     if (!isAwsStyle(credentials))
         fail("invalid_argument", `${config.provider} needs an access key credential, not a service account or OAuth token`);
     const endpoint = { provider: config.provider === "vaultline" ? "s3" : config.provider, region: config.region ?? "auto", bucket: config.bucket, host: overrides.host, port: overrides.port, protocol: overrides.protocol, pathStyle: overrides.pathStyle };
-    return new S3UploadClient(config.provider, endpoint, credentials, overrides.partBytes, overrides.allowSmallParts);
+    return new S3UploadClient(config.provider, endpoint, credentials, overrides.partBytes, overrides.allowSmallParts, fetchImpl);
 }
 export async function uploadClientFromStore(dataDir, storageConfigId, config, overrides = {}, backend) {
     const credentials = await loadProviderCredentials(dataDir, storageConfigId, backend);
@@ -52,7 +72,7 @@ export async function uploadClientFromStore(dataDir, storageConfigId, config, ov
  * The object key is derived from the configured prefix and the archive id; a caller
  * cannot choose it, which keeps one account's objects inside its own prefix.
  */
-export function createActiveLease(config, input, env = process.env) {
+export function createActiveLease(config, input, env) {
     if (!signerEnabled(env))
         fail("signer_not_configured", "Active leases require an enabled signer", { provider: config.provider });
     if (!config.bucket || !config.prefix)
@@ -63,10 +83,10 @@ export function createActiveLease(config, input, env = process.env) {
         // UploadLease.provider is still control-plane's narrower union; the cast
         // goes away with the UploadProviderKind widening above.
         archiveId, provider: config.provider,
-        objectKey: `${config.prefix.replace(/^\/+|\/+$/g, "")}/${archiveId}.vlarchive`,
+        objectKey: `${config.prefix.replace(/^\/+|\/+$/g, "")}/${archiveId}.skarchive`,
         expiresAt: new Date(issuedAt + (input.ttlMs ?? DEFAULT_LEASE_TTL_MS)).toISOString(),
         method: config.provider === "gcs" || config.provider === "gdrive" ? "RESUMABLE" : "MULTIPART",
-        uploadUrl: `${config.provider}://${config.bucket}/${config.prefix.replace(/^\/+|\/+$/g, "")}/${archiveId}.vlarchive`,
+        uploadUrl: `${config.provider}://${config.bucket}/${config.prefix.replace(/^\/+|\/+$/g, "")}/${archiveId}.skarchive`,
         requiredHeaders: { "content-type": "application/vnd.vaultline.ciphertext", "x-vaultline-ciphertext-sha256": input.ciphertextSha256, "content-length": String(input.bytes) },
         status: "active", uploadable: true
     };

package/dist/src/providers/s3.d.ts

@@ -1,5 +1,7 @@
 import { type Credentials } from "./sigv4.js";
 import type { ProviderKind, ProviderUploadClient, UploadLease } from "../control-plane.js";
+import { type ByteStream } from "../byte-stream.js";
+import type { StorageFetch } from "./safe-storage-fetch.js";
 export type S3Endpoint = {
     region: string;
     bucket: string;
@@ -46,6 +48,8 @@ export type S3StreamOptions = {
      *  This call is the durability point a resume journal writes at: once it
      *  returns, a crash can be resumed from that state. */
     onState?: (state: S3StreamState) => void | Promise<void>;
+    /** Cancellation is observed by the incoming iterator between bounded parts. */
+    signal?: AbortSignal;
 };
 /**
  * The byte position a resumed stream must start feeding from: one full part per
@@ -79,10 +83,11 @@ export declare class S3UploadClient implements ProviderUploadClient {
     private readonly endpoint;
     private readonly credentials;
     private readonly partBytes;
+    private readonly fetchImpl;
     constructor(kind: ProviderKind, endpoint: S3Endpoint, credentials: Credentials, partBytes?: number, 
     /** S3 requires every part except the last to be at least 5 MiB. Local emulators and
      *  MinIO do not, so this is the deliberate opt-out for those targets only. */
-    allowSmallParts?: boolean);
+    allowSmallParts?: boolean, fetchImpl?: StorageFetch);
     private sign;
     private put;
     /**
@@ -91,22 +96,33 @@ export declare class S3UploadClient implements ProviderUploadClient {
      * a plain checksummed PUT does everything multipart machinery did — which
      * is exactly why the chunk layout needs none of it.
      */
-    putObject(key: string, body: Buffer): Promise<{
+    putObject(key: string, body: Buffer, options?: {
+        signal?: AbortSignal;
+    }): Promise<{
         checksum: string;
     }>;
-    headObject(key: string): Promise<{
+    headObject(key: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<{
         exists: boolean;
         bytes: number;
         checksum?: string;
     }>;
-    getObject(key: string): Promise<Buffer>;
+    getObject(key: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<Buffer>;
+    /** Object-key streaming counterpart used by the background index store. */
+    getObjectStream(key: string, options?: {
+        signal?: AbortSignal;
+    }): Promise<ByteStream>;
+    private downloadKeyStream;
     deleteObject(key: string): Promise<void>;
-    initiateMultipart(key: string): Promise<string>;
+    initiateMultipart(key: string, signal?: AbortSignal): Promise<string>;
     completeMultipart(key: string, uploadId: string, parts: {
         partNumber: number;
         etag: string;
         checksum: string;
-    }[]): Promise<string>;
+    }[], signal?: AbortSignal): Promise<string>;
     /**
      * What the provider ACTUALLY holds for an in-flight multipart — the resume
      * path's source of truth. A journal can lag the provider by exactly the
@@ -119,7 +135,7 @@ export declare class S3UploadClient implements ProviderUploadClient {
         etag: string;
         bytes: number;
     }[]>;
-    abortMultipart(key: string, uploadId: string): Promise<void>;
+    abortMultipart(key: string, uploadId: string, signal?: AbortSignal): Promise<void>;
     /** Uploads ciphertext and returns the checksum the provider confirmed. */
     upload(lease: UploadLease, ciphertext: Buffer): Promise<{
         remoteChecksum: string;
@@ -157,7 +173,13 @@ export declare class S3UploadClient implements ProviderUploadClient {
      * GET with the short-lived credential it was vended, exactly as it does for
      * the upload.
      */
-    download(lease: UploadLease): Promise<Buffer>;
+    download(lease: UploadLease, options?: {
+        signal?: AbortSignal;
+    }): Promise<Buffer>;
+    /** The production restore/transfer door: response chunks flow straight to disk. */
+    downloadStream(lease: UploadLease, options?: {
+        signal?: AbortSignal;
+    }): Promise<ByteStream>;
     /**
      * Confirms the object exists remotely and reports what the provider stored.
      *