wrangler 2.0.8 → 2.0.12

package/src/publish.ts

@@ -4,13 +4,15 @@ import path from "node:path";
 import { URLSearchParams } from "node:url";
 import tmp from "tmp-promise";
 import { bundleWorker } from "./bundle";
-import { fetchResult } from "./cfetch";
+import { fetchListResult, fetchResult } from "./cfetch";
 import { printBindings } from "./config";
 import { createWorkerUploadForm } from "./create-worker-upload-form";
 import { confirm } from "./dialogs";
 import { getMigrationsToUpload } from "./durable";
 import { logger } from "./logger";
+import { ParseError } from "./parse";
 import { syncAssets } from "./sites";
+import { getZoneForRoute } from "./zones";
 import type { Config } from "./config";
 import type {
   Route,
@@ -260,7 +262,7 @@ export default async function publish(props: Props): Promise<void> {
   const nodeCompat = props.nodeCompat ?? config.node_compat;
   if (nodeCompat) {
     logger.warn(
-      "Enabling node.js compatibility mode for builtins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
+      "Enabling node.js compatibility mode for built-ins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
     );
   }
 
@@ -291,7 +293,7 @@ export default async function publish(props: Props): Promise<void> {
   const envName = props.env ?? "production";
 
   const start = Date.now();
-  const notProd = !props.legacyEnv && props.env;
+  const notProd = Boolean(!props.legacyEnv && props.env);
   const workerName = notProd ? `${scriptName} (${envName})` : scriptName;
   const workerUrl = notProd
     ? `/accounts/${accountId}/workers/services/${scriptName}/environments/${envName}`
@@ -428,7 +430,12 @@ export default async function publish(props: Props): Promise<void> {
             method: "PUT",
             body: createWorkerUploadForm(worker),
           },
-          new URLSearchParams({ include_subdomain_availability: "true" })
+          new URLSearchParams({
+            include_subdomain_availability: "true",
+            // pass excludeScript so the whole body of the
+            // script doesn't get included in the response
+            excludeScript: "true",
+          })
         )
       ).available_on_subdomain;
     }
@@ -497,18 +504,7 @@ export default async function publish(props: Props): Promise<void> {
   // Update routing table for the script.
   if (routesOnly.length > 0) {
     deployments.push(
-      fetchResult(`${workerUrl}/routes`, {
-        // Note: PUT will delete previous routes on this script.
-        method: "PUT",
-        body: JSON.stringify(
-          routesOnly.map((route) =>
-            typeof route !== "object" ? { pattern: route } : route
-          )
-        ),
-        headers: {
-          "Content-Type": "application/json",
-        },
-      }).then(() => {
+      publishRoutes(routesOnly, { workerUrl, scriptName, notProd }).then(() => {
         if (routesOnly.length > 10) {
           return routesOnly
             .slice(0, 9)
@@ -581,3 +577,145 @@ async function getSubdomain(accountId: string): Promise<string> {
     }
   }
 }
+
+/**
+ * Associate the newly deployed Worker with the given routes.
+ */
+async function publishRoutes(
+  routes: Route[],
+  {
+    workerUrl,
+    scriptName,
+    notProd,
+  }: { workerUrl: string; scriptName: string; notProd: boolean }
+): Promise<string[]> {
+  try {
+    return await fetchResult(`${workerUrl}/routes`, {
+      // Note: PUT will delete previous routes on this script.
+      method: "PUT",
+      body: JSON.stringify(
+        routes.map((route) =>
+          typeof route !== "object" ? { pattern: route } : route
+        )
+      ),
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
+  } catch (e) {
+    if (isAuthenticationError(e)) {
+      // An authentication error is probably due to a known issue,
+      // where the user is logged in via an API token that does not have "All Zones".
+      return await publishRoutesFallback(routes, { scriptName, notProd });
+    } else {
+      throw e;
+    }
+  }
+}
+
+/**
+ * Try updating routes for the Worker using a less optimal zone-based API.
+ *
+ * Compute match zones to the routes, then for each route attempt to connect it to the Worker via the zone.
+ */
+async function publishRoutesFallback(
+  routes: Route[],
+  { scriptName, notProd }: { scriptName: string; notProd: boolean }
+) {
+  if (notProd) {
+    throw new Error(
+      "Service environments combined with an API token that doesn't have 'All Zones' permissions is not supported.\n" +
+        "Either turn off service environments by setting `legacy_env = true`, creating an API token with 'All Zones' permissions, or logging in via OAuth"
+    );
+  }
+  logger.warn(
+    "The current authentication token does not have 'All Zones' permissions.\n" +
+      "Falling back to using the zone-based API endpoint to update each route individually.\n" +
+      "Note that there is no access to routes associated with zones that the API token does not have permission for.\n" +
+      "Existing routes for this Worker in such zones will not be deleted."
+  );
+
+  const deployedRoutes: string[] = [];
+
+  // Collect the routes (and their zones) that will be deployed.
+  const activeZones = new Map<string, string>();
+  const routesToDeploy = new Map<string, string>();
+  for (const route of routes) {
+    const zone = await getZoneForRoute(route);
+    if (zone) {
+      activeZones.set(zone.id, zone.host);
+      routesToDeploy.set(
+        typeof route === "string" ? route : route.pattern,
+        zone.id
+      );
+    }
+  }
+
+  // Collect the routes that are already deployed.
+  const allRoutes = new Map<string, string>();
+  const alreadyDeployedRoutes = new Set<string>();
+  for (const [zone, host] of activeZones) {
+    try {
+      for (const { pattern, script } of await fetchListResult<{
+        pattern: string;
+        script: string;
+      }>(`/zones/${zone}/workers/routes`)) {
+        allRoutes.set(pattern, script);
+        if (script === scriptName) {
+          alreadyDeployedRoutes.add(pattern);
+        }
+      }
+    } catch (e) {
+      if (isAuthenticationError(e)) {
+        e.notes.push({
+          text: `This could be because the API token being used does not have permission to access the zone "${host}" (${zone}).`,
+        });
+      }
+      throw e;
+    }
+  }
+
+  // Deploy each route that is not already deployed.
+  for (const [routePattern, zoneId] of routesToDeploy.entries()) {
+    if (allRoutes.has(routePattern)) {
+      const knownScript = allRoutes.get(routePattern);
+      if (knownScript === scriptName) {
+        // This route is already associated with this worker, so no need to hit the API.
+        alreadyDeployedRoutes.delete(routePattern);
+        continue;
+      } else {
+        throw new Error(
+          `The route with pattern "${routePattern}" is already associated with another worker called "${knownScript}".`
+        );
+      }
+    }
+
+    const { pattern } = await fetchResult(`/zones/${zoneId}/workers/routes`, {
+      method: "POST",
+      body: JSON.stringify({
+        pattern: routePattern,
+        script: scriptName,
+      }),
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
+
+    deployedRoutes.push(pattern);
+  }
+
+  if (alreadyDeployedRoutes.size) {
+    logger.warn(
+      "Previously deployed routes:\n" +
+        "The following routes were already associated with this worker, and have not been deleted:\n" +
+        [...alreadyDeployedRoutes.values()].map((route) => ` - "${route}"\n`) +
+        "If these routes are not wanted then you can remove them in the dashboard."
+    );
+  }
+
+  return deployedRoutes;
+}
+
+function isAuthenticationError(e: unknown): e is ParseError {
+  return e instanceof ParseError && (e as { code?: number }).code === 10000;
+}

package/src/sites.tsx

@@ -137,7 +137,13 @@ export async function syncAssets(
   const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
 
   const manifest: Record<string, string> = {};
-  const toUpload: KeyValue[] = [];
+
+  // A batch of uploads where each bucket has to be less than 100mb
+  const uploadBuckets: KeyValue[][] = [];
+  // The "live" bucket that we'll keep filling until it's just below 100mb
+  let uploadBucket: KeyValue[] = [];
+  // A size counter for the live bucket
+  let uploadBucketSize = 0;
 
   const include = createPatternMatcher(siteAssets.includePatterns, false);
   const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
@@ -148,7 +154,7 @@ export async function syncAssets(
     siteAssets.assetDirectory
   );
   for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
-    const assetFile = path.relative(siteAssets.baseDirectory, absAssetFile);
+    const assetFile = path.relative(assetDirectory, absAssetFile);
     if (!include(assetFile)) {
       continue;
     }
@@ -156,17 +162,32 @@ export async function syncAssets(
       continue;
     }
 
-    await validateAssetSize(absAssetFile, assetFile);
     logger.log(`Reading ${assetFile}...`);
     const content = await readFile(absAssetFile, "base64");
-
+    await validateAssetSize(absAssetFile, assetFile);
+    // while KV accepts files that are 25 MiB **before** b64 encoding
+    // the overall bucket size must be below 100 MiB **after** b64 encoding
+    const assetSize = Buffer.from(content).length;
     const assetKey = hashAsset(hasher, assetFile, content);
     validateAssetKey(assetKey);
 
     // now put each of the files into kv
     if (!namespaceKeys.has(assetKey)) {
       logger.log(`Uploading as ${assetKey}...`);
-      toUpload.push({
+
+      // Check if adding this asset to the bucket would
+      // push it over the 100 MiB limit KV bulk API limit
+      if (uploadBucketSize + assetSize > 100 * 1024 * 1024) {
+        // If so, move the current bucket into the batch,
+        // and reset the counter/bucket
+        uploadBuckets.push(uploadBucket);
+        uploadBucketSize = 0;
+        uploadBucket = [];
+      }
+
+      // Update the bucket and the size counter
+      uploadBucketSize += assetSize;
+      uploadBucket.push({
         key: assetKey,
         value: content,
         base64: true,
@@ -175,27 +196,31 @@ export async function syncAssets(
       logger.log(`Skipping - already uploaded.`);
     }
 
-    // remove the key from the set so we know what we've already uploaded
+    // Remove the key from the set so we know what we've already uploaded
     namespaceKeys.delete(assetKey);
 
-    // prevent causing different manifest keys on windows
-    const maifestKey = urlSafe(
-      path.relative(siteAssets.assetDirectory, absAssetFile)
-    );
-    manifest[maifestKey] = assetKey;
+    // Prevent different manifest keys on windows
+    const manifestKey = urlSafe(path.relative(assetDirectory, absAssetFile));
+    manifest[manifestKey] = assetKey;
   }
 
+  // Add the last (potentially only) bucket to the batch
+  uploadBuckets.push(uploadBucket);
+
   // keys now contains all the files we're deleting
   for (const key of namespaceKeys) {
     logger.log(`Deleting ${key} from the asset store...`);
   }
 
-  await Promise.all([
-    // upload all the new assets
-    putKVBulkKeyValue(accountId, namespace, toUpload),
-    // delete all the unused assets
-    deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys)),
-  ]);
+  // upload each bucket in parallel
+  const bucketsToPut = [];
+  for (const bucket of uploadBuckets) {
+    bucketsToPut.push(putKVBulkKeyValue(accountId, namespace, bucket));
+  }
+  await Promise.all(bucketsToPut);
+
+  // then delete all the assets that aren't used anymore
+  await deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys));
 
   logger.log("↗️  Done syncing assets");
 
@@ -214,10 +239,16 @@ function createPatternMatcher(
   }
 }
 
+/**
+ * validate that the passed-in file is below 25 MiB
+ * **PRIOR** to base64 encoding. 25 MiB is a KV limit
+ * @param absFilePath
+ * @param relativeFilePath
+ */
 async function validateAssetSize(
   absFilePath: string,
   relativeFilePath: string
-) {
+): Promise<void> {
   const { size } = await stat(absFilePath);
   if (size > 25 * 1024 * 1024) {
     throw new Error(

package/src/user.tsx

@@ -233,7 +233,7 @@ import type { Response } from "undici";
 /**
  * Try to read the API token from the environment.
  */
-const getCloudflareAPITokenFromEnv = getEnvironmentVariableFactory({
+export const getCloudflareAPITokenFromEnv = getEnvironmentVariableFactory({
   variableName: "CLOUDFLARE_API_TOKEN",
   deprecatedName: "CF_API_TOKEN",
 });
@@ -1201,3 +1201,14 @@ export async function requireAuth(config: {
 
   return accountId;
 }
+
+/**
+ * Throw an error if there is no API token available.
+ */
+export function requireApiToken(): string {
+  const authToken = getAPIToken();
+  if (!authToken) {
+    throw new Error("No API token found.");
+  }
+  return authToken;
+}

package/src/whoami.tsx

@@ -3,7 +3,7 @@ import Table from "ink-table";
 import React from "react";
 import { fetchListResult, fetchResult } from "./cfetch";
 import { logger } from "./logger";
-import { getAPIToken } from "./user";
+import { getAPIToken, getCloudflareAPITokenFromEnv } from "./user";
 
 export async function whoami() {
   logger.log("Getting User settings...");
@@ -48,10 +48,11 @@ export interface UserInfo {
 
 export async function getUserInfo(): Promise<UserInfo | undefined> {
   const apiToken = getAPIToken();
+  const apiTokenFromEnv = getCloudflareAPITokenFromEnv();
   return apiToken
     ? {
         apiToken,
-        authType: "OAuth",
+        authType: apiTokenFromEnv ? "API" : "OAuth",
         email: await getEmail(),
         accounts: await getAccounts(),
       }

package/src/worker.ts

@@ -176,5 +176,6 @@ export interface CfWorkerInit {
 export interface CfWorkerContext {
   env: string | undefined;
   legacyEnv: boolean | undefined;
-  zone: { id: string; host: string } | undefined;
+  zone: string | undefined;
+  host: string | undefined;
 }

package/src/zones.ts

@@ -0,0 +1,73 @@
+import { fetchListResult } from "./cfetch";
+import type { Route } from "./config/environment";
+
+/**
+ * An object holding information about a zone for publishing.
+ */
+export interface Zone {
+  id: string;
+  host: string;
+}
+
+/**
+ * Try to compute the a zone ID and host name for one or more routes.
+ *
+ * When we're given a route, we do 2 things:
+ * - We try to extract a host from it
+ * - We try to get a zone id from the host
+ */
+export async function getZoneForRoute(route: Route): Promise<Zone | undefined> {
+  const host =
+    typeof route === "string"
+      ? getHostFromUrl(route)
+      : typeof route === "object"
+      ? "zone_name" in route
+        ? getHostFromUrl(route.zone_name)
+        : getHostFromUrl(route.pattern)
+      : undefined;
+  const id =
+    typeof route === "object" && "zone_id" in route
+      ? route.zone_id
+      : host
+      ? await getZoneIdFromHost(host)
+      : undefined;
+  return id && host ? { id, host } : undefined;
+}
+
+/**
+ * Given something that resembles a URL, try to extract a host from it.
+ */
+function getHostFromUrl(urlLike: string): string | undefined {
+  // strip leading * / *.
+  urlLike = urlLike.replace(/^\*(\.)?/g, "");
+
+  if (!(urlLike.startsWith("http://") || urlLike.startsWith("https://"))) {
+    urlLike = "http://" + urlLike;
+  }
+  return new URL(urlLike).host;
+}
+
+/**
+ * Given something that resembles a host, try to infer a zone id from it.
+ *
+ * It's hard to get a 'valid' domain from a string, so we don't even try to validate TLDs, etc.
+ * For each domain-like part of the host (e.g. w.x.y.z) try to get a zone id for it by
+ * lopping off subdomains until we get a hit from the API.
+ */
+export async function getZoneIdFromHost(host: string): Promise<string> {
+  const hostPieces = host.split(".");
+
+  while (hostPieces.length > 1) {
+    const zones = await fetchListResult<{ id: string }>(
+      `/zones`,
+      {},
+      new URLSearchParams({ name: hostPieces.join(".") })
+    );
+    if (zones.length > 0) {
+      return zones[0].id;
+    }
+    hostPieces.shift();
+  }
+
+  throw new Error(`Could not find zone for ${host}`);
+}

package/templates/static-asset-facade.js

@@ -1,13 +1,9 @@
 // DO NOT IMPORT THIS DIRECTLY
 import worker from "__ENTRY_POINT__";
-import { getAssetFromKV } from "@cloudflare/kv-asset-handler";
+import { getAssetFromKV } from "__KV_ASSET_HANDLER__";
 import manifest from "__STATIC_CONTENT_MANIFEST";
 const ASSET_MANIFEST = JSON.parse(manifest);
 
-// TODO: remove this
-globalThis.__STATIC_CONTENT = undefined;
-globalThis.__STATIC_CONTENT_MANIFEST = undefined;
-
 export default {
   async fetch(request, env, ctx) {
     let options = {