wrangler 2.0.9 → 2.0.14

package/src/sites.tsx

@@ -137,7 +137,13 @@ export async function syncAssets(
   const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
 
   const manifest: Record<string, string> = {};
-  const toUpload: KeyValue[] = [];
+
+  // A batch of uploads where each bucket has to be less than 100mb
+  const uploadBuckets: KeyValue[][] = [];
+  // The "live" bucket that we'll keep filling until it's just below 100mb
+  let uploadBucket: KeyValue[] = [];
+  // A size counter for the live bucket
+  let uploadBucketSize = 0;
 
   const include = createPatternMatcher(siteAssets.includePatterns, false);
   const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
@@ -148,7 +154,7 @@ export async function syncAssets(
     siteAssets.assetDirectory
   );
   for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
-    const assetFile = path.relative(siteAssets.baseDirectory, absAssetFile);
+    const assetFile = path.relative(assetDirectory, absAssetFile);
     if (!include(assetFile)) {
       continue;
     }
@@ -156,17 +162,32 @@ export async function syncAssets(
       continue;
     }
 
-    await validateAssetSize(absAssetFile, assetFile);
     logger.log(`Reading ${assetFile}...`);
     const content = await readFile(absAssetFile, "base64");
-
+    await validateAssetSize(absAssetFile, assetFile);
+    // while KV accepts files that are 25 MiB **before** b64 encoding
+    // the overall bucket size must be below 100 MiB **after** b64 encoding
+    const assetSize = Buffer.from(content).length;
     const assetKey = hashAsset(hasher, assetFile, content);
     validateAssetKey(assetKey);
 
     // now put each of the files into kv
     if (!namespaceKeys.has(assetKey)) {
       logger.log(`Uploading as ${assetKey}...`);
-      toUpload.push({
+
+      // Check if adding this asset to the bucket would
+      // push it over the 100 MiB limit KV bulk API limit
+      if (uploadBucketSize + assetSize > 100 * 1024 * 1024) {
+        // If so, move the current bucket into the batch,
+        // and reset the counter/bucket
+        uploadBuckets.push(uploadBucket);
+        uploadBucketSize = 0;
+        uploadBucket = [];
+      }
+
+      // Update the bucket and the size counter
+      uploadBucketSize += assetSize;
+      uploadBucket.push({
         key: assetKey,
         value: content,
         base64: true,
@@ -175,27 +196,31 @@ export async function syncAssets(
       logger.log(`Skipping - already uploaded.`);
     }
 
-    // remove the key from the set so we know what we've already uploaded
+    // Remove the key from the set so we know what we've already uploaded
     namespaceKeys.delete(assetKey);
 
-    // prevent causing different manifest keys on windows
-    const maifestKey = urlSafe(
-      path.relative(siteAssets.assetDirectory, absAssetFile)
-    );
-    manifest[maifestKey] = assetKey;
+    // Prevent different manifest keys on windows
+    const manifestKey = urlSafe(path.relative(assetDirectory, absAssetFile));
+    manifest[manifestKey] = assetKey;
   }
 
+  // Add the last (potentially only) bucket to the batch
+  uploadBuckets.push(uploadBucket);
+
   // keys now contains all the files we're deleting
   for (const key of namespaceKeys) {
     logger.log(`Deleting ${key} from the asset store...`);
   }
 
-  await Promise.all([
-    // upload all the new assets
-    putKVBulkKeyValue(accountId, namespace, toUpload),
-    // delete all the unused assets
-    deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys)),
-  ]);
+  // upload each bucket in parallel
+  const bucketsToPut = [];
+  for (const bucket of uploadBuckets) {
+    bucketsToPut.push(putKVBulkKeyValue(accountId, namespace, bucket));
+  }
+  await Promise.all(bucketsToPut);
+
+  // then delete all the assets that aren't used anymore
+  await deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys));
 
   logger.log("↗️  Done syncing assets");
 
@@ -214,10 +239,16 @@ function createPatternMatcher(
   }
 }
 
+/**
+ * validate that the passed-in file is below 25 MiB
+ * **PRIOR** to base64 encoding. 25 MiB is a KV limit
+ * @param absFilePath
+ * @param relativeFilePath
+ */
 async function validateAssetSize(
   absFilePath: string,
   relativeFilePath: string
-) {
+): Promise<void> {
   const { size } = await stat(absFilePath);
   if (size > 25 * 1024 * 1024) {
     throw new Error(

package/src/user.tsx

@@ -223,6 +223,8 @@ import { fetch } from "undici";
 import { getCloudflareApiBaseUrl } from "./cfetch";
 import { purgeConfigCaches } from "./config-cache";
 import { getEnvironmentVariableFactory } from "./environment-variables";
+import { generateAuthUrl } from "./generate-auth-url";
+import { generateRandomState } from "./generate-random-state";
 import { logger } from "./logger";
 import openInBrowser from "./open-in-browser";
 import { parseTOML, readFileSync } from "./parse";
@@ -591,7 +593,7 @@ const RECOMMENDED_STATE_LENGTH = 32;
 /**
  * Character set to generate code verifier defined in rfc7636.
  */
-const PKCE_CHARSET =
+export const PKCE_CHARSET =
   "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~";
 
 /**
@@ -641,17 +643,14 @@ export async function getAuthURL(scopes = ScopeKeys): Promise<string> {
     stateQueryParam,
   });
 
-  return (
-    AUTH_URL +
-    `?response_type=code&` +
-    `client_id=${encodeURIComponent(CLIENT_ID)}&` +
-    `redirect_uri=${encodeURIComponent(CALLBACK_URL)}&` +
-    // we add offline_access manually for every request
-    `scope=${encodeURIComponent([...scopes, "offline_access"].join(" "))}&` +
-    `state=${stateQueryParam}&` +
-    `code_challenge=${encodeURIComponent(codeChallenge)}&` +
-    `code_challenge_method=S256`
-  );
+  return generateAuthUrl({
+    authUrl: AUTH_URL,
+    clientId: CLIENT_ID,
+    callbackUrl: CALLBACK_URL,
+    scopes,
+    stateQueryParam,
+    codeChallenge,
+  });
 }
 
 type TokenResponse =
@@ -863,18 +862,6 @@ async function generatePKCECodes(): Promise<PKCECodes> {
   return { codeChallenge, codeVerifier };
 }
 
-/**
- * Generates random state to be passed for anti-csrf.
- */
-function generateRandomState(lengthOfState: number): string {
-  const output = new Uint32Array(lengthOfState);
-  // @ts-expect-error crypto's types aren't there yet
-  crypto.getRandomValues(output);
-  return Array.from(output)
-    .map((num: number) => PKCE_CHARSET[num % PKCE_CHARSET.length])
-    .join("");
-}
-
 /**
  * Writes a a wrangler config file (auth credentials) to disk,
  * and updates the user auth state with the new credentials.
@@ -1013,6 +1000,7 @@ export async function login(props?: LoginProps): Promise<boolean> {
     server.listen(8976);
   });
 
+  logger.log(`Opening a link in your default browser: ${urlToOpen}`);
   await openInBrowser(urlToOpen);
 
   return Promise.race([timerPromise, loginPromise]);

package/templates/static-asset-facade.js

@@ -1,13 +1,9 @@
 // DO NOT IMPORT THIS DIRECTLY
 import worker from "__ENTRY_POINT__";
-import { getAssetFromKV } from "@cloudflare/kv-asset-handler";
+import { getAssetFromKV } from "__KV_ASSET_HANDLER__";
 import manifest from "__STATIC_CONTENT_MANIFEST";
 const ASSET_MANIFEST = JSON.parse(manifest);
 
-// TODO: remove this
-globalThis.__STATIC_CONTENT = undefined;
-globalThis.__STATIC_CONTENT_MANIFEST = undefined;
-
 export default {
   async fetch(request, env, ctx) {
     let options = {
@@ -39,7 +35,7 @@ export default {
     } catch (e) {
       console.error(e);
       // if an error is thrown then serve from actual worker
-      return worker.fetch(request);
+      return worker.fetch(request, env, ctx);
       // TODO: throw here if worker is not available
       // (which implies it may be a service-worker)
     }