wrangler 2.0.6 → 2.0.7

package/src/kv.ts

@@ -1,5 +1,6 @@
 import { URLSearchParams } from "node:url";
 import { fetchListResult, fetchResult, fetchKVGetValue } from "./cfetch";
+import { logger } from "./logger";
 import type { Config } from "./config";
 
 /** The largest number of kv items we can pass to the API in a single request. */
@@ -126,26 +127,51 @@ const KeyValueKeys = new Set([
   "base64",
 ]);
 
+/**
+ * The object has the specified property.
+ */
+function hasProperty<T extends object>(
+  obj: object,
+  property: keyof T
+): obj is T {
+  return property in obj;
+}
+
+/**
+ * The object has a required property of the specified type.
+ */
+function hasTypedProperty<T extends object>(
+  obj: object,
+  property: keyof T,
+  type: string
+): obj is T {
+  return hasProperty(obj, property) && typeof obj[property] === type;
+}
+
+/**
+ * The object an optional property, of the specified type.
+ */
+function hasOptionalTypedProperty<T extends object>(
+  obj: object,
+  property: keyof T,
+  type: string
+): obj is Omit<T, typeof property> | T {
+  return !hasProperty(obj, property) || typeof obj[property] === type;
+}
+
 /**
  * Is the given object a valid `KeyValue` type?
  */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export function isKVKeyValue(keyValue: any): keyValue is KeyValue {
-  // (keyValue could indeed be any-thing)
+export function isKVKeyValue(keyValue: unknown): keyValue is KeyValue {
   if (
+    keyValue === null ||
     typeof keyValue !== "object" ||
-    typeof keyValue.key !== "string" ||
-    typeof keyValue.value !== "string" ||
-    !(
-      keyValue.expiration === undefined ||
-      typeof keyValue.expiration === "number"
-    ) ||
-    !(
-      keyValue.expiration_ttl === undefined ||
-      typeof keyValue.expiration_ttl === "number"
-    ) ||
-    !(keyValue.base64 === undefined || typeof keyValue.base64 === "boolean") ||
-    !(keyValue.metadata === undefined || typeof keyValue.metadata === "object")
+    !hasTypedProperty(keyValue, "key", "string") ||
+    !hasTypedProperty(keyValue, "value", "string") ||
+    !hasOptionalTypedProperty(keyValue, "expiration", "number") ||
+    !hasOptionalTypedProperty(keyValue, "expiration_ttl", "number") ||
+    !hasOptionalTypedProperty(keyValue, "base64", "boolean") ||
+    !hasOptionalTypedProperty(keyValue, "metadata", "object")
   ) {
     return false;
   }
@@ -205,15 +231,37 @@ export async function deleteKVKeyValue(
   );
 }
 
+/**
+ * Formatter for converting e.g. 5328 --> 5,328
+ */
+const formatNumber = new Intl.NumberFormat("en-US", {
+  notation: "standard",
+}).format;
+
+/**
+ * Helper function for bulk requests, logs ongoing output to console.
+ */
+function logBulkProgress(
+  operation: "put" | "delete",
+  index: number,
+  total: number
+) {
+  logger.log(
+    `${operation === "put" ? "Uploaded" : "Deleted"} ${Math.floor(
+      (100 * index) / total
+    )}% (${formatNumber(index)} out of ${formatNumber(total)})`
+  );
+}
+
 export async function putKVBulkKeyValue(
   accountId: string,
   namespaceId: string,
   keyValues: KeyValue[],
-  progressCallback: (index: number, total: number) => void
+  quiet = false
 ) {
   for (let index = 0; index < keyValues.length; index += BATCH_KEY_MAX) {
-    if (progressCallback && keyValues.length > BATCH_KEY_MAX) {
-      progressCallback(index, keyValues.length);
+    if (!quiet && keyValues.length > BATCH_KEY_MAX) {
+      logBulkProgress("put", index, keyValues.length);
     }
 
     await fetchResult(
@@ -225,8 +273,9 @@ export async function putKVBulkKeyValue(
       }
     );
   }
-  if (progressCallback && keyValues.length > BATCH_KEY_MAX) {
-    progressCallback(keyValues.length, keyValues.length);
+
+  if (!quiet && keyValues.length > BATCH_KEY_MAX) {
+    logBulkProgress("put", keyValues.length, keyValues.length);
   }
 }
 
@@ -234,11 +283,11 @@ export async function deleteKVBulkKeyValue(
   accountId: string,
   namespaceId: string,
   keys: string[],
-  progressCallback: (index: number, total: number) => void
+  quiet = false
 ) {
   for (let index = 0; index < keys.length; index += BATCH_KEY_MAX) {
-    if (progressCallback && keys.length > BATCH_KEY_MAX) {
-      progressCallback(index, keys.length);
+    if (!quiet && keys.length > BATCH_KEY_MAX) {
+      logBulkProgress("delete", index, keys.length);
     }
 
     await fetchResult(
@@ -250,8 +299,8 @@ export async function deleteKVBulkKeyValue(
       }
     );
   }
-  if (progressCallback && keys.length > BATCH_KEY_MAX) {
-    progressCallback(keys.length, keys.length);
+  if (!quiet && keys.length > BATCH_KEY_MAX) {
+    logBulkProgress("delete", keys.length, keys.length);
   }
 }
 

package/src/open-in-browser.ts

@@ -8,19 +8,12 @@ import { logger } from "./logger";
  * StackBlitz, remote servers), it doesn't just crash the process.
  *
  * @param url the URL to point the browser at
- * @param options open a Chromium-based browser instead of the default
  */
-export default async function openInBrowser(
-  url: string,
-  { forceChromium }: { forceChromium: boolean } = { forceChromium: false }
-): Promise<void> {
-  const options: open.Options | undefined = forceChromium
-    ? {
-        app: [{ name: open.apps.chrome }, { name: open.apps.edge }],
-      }
-    : undefined;
-  const childProcess = await open(url, options);
+export default async function openInBrowser(url: string): Promise<void> {
+  const errorMessage = `Failed to open ${url} in a browser`;
+
+  const childProcess = await open(url);
   childProcess.on("error", () => {
-    logger.warn(`Failed to open ${url} in a browser.`);
+    logger.warn(errorMessage);
   });
 }

package/src/pages.tsx

@@ -14,6 +14,7 @@ import SelectInput from "ink-select-input";
 import Spinner from "ink-spinner";
 import Table from "ink-table";
 import { getType } from "mime";
+import PQueue from "p-queue";
 import prettyBytes from "pretty-bytes";
 import React from "react";
 import { format as timeagoFormat } from "timeago.js";
@@ -32,7 +33,11 @@ import openInBrowser from "./open-in-browser";
 import { toUrlPath } from "./paths";
 import { requireAuth } from "./user";
 import type { Config } from "../pages/functions/routes";
-import type { Headers, Request, fetch } from "@miniflare/core";
+import type {
+  Headers as MiniflareHeaders,
+  Request as MiniflareRequest,
+  fetch as miniflareFetch,
+} from "@miniflare/core";
 import type { BuildResult } from "esbuild";
 import type { MiniflareOptions } from "miniflare";
 import type { BuilderCallback, CommandModule } from "yargs";
@@ -68,12 +73,23 @@ export type Deployment = {
   project_name: string;
 };
 
+export type UploadPayloadFile = {
+  key: string;
+  value: string;
+  metadata: { contentType: string };
+  base64: boolean;
+};
+
 interface PagesConfigCache {
   account_id?: string;
   project_name?: string;
 }
 
 const PAGES_CONFIG_CACHE_FILENAME = "pages.json";
+const MAX_BUCKET_SIZE = 50 * 1024 * 1024;
+const MAX_BUCKET_FILE_COUNT = 5000;
+const BULK_UPLOAD_CONCURRENCY = 3;
+const MAX_UPLOAD_ATTEMPTS = 5;
 
 // Defer importing miniflare until we really need it. This takes ~0.5s
 // and also modifies some `stream/web` and `undici` prototypes, so we
@@ -284,7 +300,7 @@ function generateRulesMatcher<T>(
     T
   ][];
 
-  return ({ request }: { request: Request }) => {
+  return ({ request }: { request: MiniflareRequest }) => {
     const { pathname, host } = new URL(request.url);
 
     return compiledRules
@@ -357,7 +373,7 @@ function generateHeadersMatcher(headersFile: string) {
       )
     );
 
-    return (request: Request) => {
+    return (request: MiniflareRequest) => {
       const matches = rulesMatcher({
         request,
       });
@@ -406,7 +422,7 @@ function generateRedirectsMatcher(redirectsFile: string) {
       })
     );
 
-    return (request: Request) => {
+    return (request: MiniflareRequest) => {
       const match = rulesMatcher({
         request,
       })[0];
@@ -461,7 +477,9 @@ function hasFileExtension(pathname: string) {
   return /\/.+\.[a-z0-9]+$/i.test(pathname);
 }
 
-async function generateAssetsFetch(directory: string): Promise<typeof fetch> {
+async function generateAssetsFetch(
+  directory: string
+): Promise<typeof miniflareFetch> {
   // Defer importing miniflare until we really need it
   const { Headers, Request, Response } = await import("@miniflare/core");
 
@@ -510,12 +528,12 @@ async function generateAssetsFetch(directory: string): Promise<typeof fetch> {
     return readFileSync(file);
   };
 
-  const generateResponse = (request: Request) => {
+  const generateResponse = (request: MiniflareRequest) => {
     const url = new URL(request.url);
 
     const deconstructedResponse: {
       status: number;
-      headers: Headers;
+      headers: MiniflareHeaders;
       body?: Buffer;
     } = {
       status: 200,
@@ -667,8 +685,12 @@ async function generateAssetsFetch(directory: string): Promise<typeof fetch> {
   };
 
   const attachHeaders = (
-    request: Request,
-    deconstructedResponse: { status: number; headers: Headers; body?: Buffer }
+    request: MiniflareRequest,
+    deconstructedResponse: {
+      status: number;
+      headers: MiniflareHeaders;
+      body?: Buffer;
+    }
   ) => {
     const headers = deconstructedResponse.headers;
     const newHeaders = new Headers({});
@@ -722,6 +744,7 @@ async function buildFunctions({
   onEnd,
   plugin = false,
   buildOutputDirectory,
+  nodeCompat,
 }: {
   outfile: string;
   outputConfigPath?: string;
@@ -733,12 +756,13 @@ async function buildFunctions({
   onEnd?: () => void;
   plugin?: boolean;
   buildOutputDirectory?: string;
+  nodeCompat?: boolean;
 }) {
   RUNNING_BUILDERS.forEach(
     (runningBuilder) => runningBuilder.stop && runningBuilder.stop()
   );
 
-  const routesModule = join(tmpdir(), "./functionsRoutes.mjs");
+  const routesModule = join(tmpdir(), `./functionsRoutes-${Math.random()}.mjs`);
   const baseURL = toUrlPath("/");
 
   const config: Config = await generateConfigFromFileTree({
@@ -767,6 +791,7 @@ async function buildFunctions({
         minify,
         sourcemap,
         watch,
+        nodeCompat,
         onEnd,
       })
     );
@@ -781,6 +806,7 @@ async function buildFunctions({
         watch,
         onEnd,
         buildOutputDirectory,
+        nodeCompat,
       })
     );
   }
@@ -1001,7 +1027,7 @@ const createDeployment: CommandModule<
 
       if (isGitDirty && !commitDirty) {
         logger.warn(
-          `Warning: Your working directory is a git repo and has uncommitted changes\nTo silense this warning, pass in --commit-dirty=true`
+          `Warning: Your working directory is a git repo and has uncommitted changes\nTo silence this warning, pass in --commit-dirty=true`
         );
       }
 
@@ -1013,7 +1039,7 @@ const createDeployment: CommandModule<
     let builtFunctions: string | undefined = undefined;
     const functionsDirectory = join(cwd(), "functions");
     if (existsSync(functionsDirectory)) {
-      const outfile = join(tmpdir(), "./functionsWorker.js");
+      const outfile = join(tmpdir(), `./functionsWorker-${Math.random()}.js`);
 
       await new Promise((resolve) =>
         buildFunctions({
@@ -1027,12 +1053,9 @@ const createDeployment: CommandModule<
       builtFunctions = readFileSync(outfile, "utf-8");
     }
 
-    type File = {
-      content: Buffer;
-      metadata: Metadata;
-    };
-
-    type Metadata = {
+    type FileContainer = {
+      content: string;
+      contentType: string;
       sizeInBytes: number;
       hash: string;
     };
@@ -1047,7 +1070,7 @@ const createDeployment: CommandModule<
 
     const walk = async (
       dir: string,
-      fileMap: Map<string, File> = new Map(),
+      fileMap: Map<string, FileContainer> = new Map(),
       depth = 0
     ) => {
       const files = await readdir(dir);
@@ -1081,8 +1104,6 @@ const createDeployment: CommandModule<
             const base64Content = fileContent.toString("base64");
             const extension = extname(basename(name)).substring(1);
 
-            const content = base64Content + extension;
-
             if (filestat.size > 25 * 1024 * 1024) {
               throw new Error(
                 `Error: Pages only supports files up to ${prettyBytes(
@@ -1092,11 +1113,12 @@ const createDeployment: CommandModule<
             }
 
             fileMap.set(name, {
-              content: fileContent,
-              metadata: {
-                sizeInBytes: filestat.size,
-                hash: hash(content).toString("hex").slice(0, 32),
-              },
+              content: base64Content,
+              contentType: getType(name) || "application/octet-stream",
+              sizeInBytes: filestat.size,
+              hash: hash(base64Content + extension)
+                .toString("hex")
+                .slice(0, 32),
             });
           }
         })
@@ -1107,51 +1129,139 @@ const createDeployment: CommandModule<
 
     const fileMap = await walk(directory);
 
+    if (fileMap.size > 20000) {
+      throw new FatalError(
+        `Error: Pages only supports up to 20,000 files in a deployment. Ensure you have specified your build output directory correctly.`,
+        1
+      );
+    }
+
+    const files = [...fileMap.values()];
+
+    const { jwt } = await fetchResult<{ jwt: string }>(
+      `/accounts/${accountId}/pages/projects/${projectName}/upload-token`
+    );
+
     const start = Date.now();
 
-    const files: Array<Promise<void>> = [];
+    const missingHashes = await fetchResult<string[]>(
+      `/pages/assets/check-missing`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${jwt}`,
+        },
+        body: JSON.stringify({
+          hashes: files.map(({ hash }) => hash),
+        }),
+      }
+    );
 
-    if (fileMap.size > 1000) {
-      throw new Error(
-        `Error: Pages only supports up to 1,000 files in a deployment at the moment.\nTry a smaller project perhaps?`
-      );
-    }
+    const sortedFiles = files
+      .filter((file) => missingHashes.includes(file.hash))
+      .sort((a, b) => b.sizeInBytes - a.sizeInBytes);
+
+    // Start with a few buckets so small projects still get
+    // the benefit of multiple upload streams
+    const buckets: {
+      files: FileContainer[];
+      remainingSize: number;
+    }[] = new Array(BULK_UPLOAD_CONCURRENCY).fill(null).map(() => ({
+      files: [],
+      remainingSize: MAX_BUCKET_SIZE,
+    }));
+
+    let bucketOffset = 0;
+    for (const file of sortedFiles) {
+      let inserted = false;
+
+      for (let i = 0; i < buckets.length; i++) {
+        // Start at a different bucket for each new file
+        const bucket = buckets[(i + bucketOffset) % buckets.length];
+        if (
+          bucket.remainingSize >= file.sizeInBytes &&
+          bucket.files.length < MAX_BUCKET_FILE_COUNT
+        ) {
+          bucket.files.push(file);
+          bucket.remainingSize -= file.sizeInBytes;
+          inserted = true;
+          break;
+        }
+      }
 
-    let counter = 0;
+      if (!inserted) {
+        buckets.push({
+          files: [file],
+          remainingSize: MAX_BUCKET_SIZE - file.sizeInBytes,
+        });
+      }
+      bucketOffset++;
+    }
 
+    let counter = fileMap.size - sortedFiles.length;
     const { rerender, unmount } = render(
       <Progress done={counter} total={fileMap.size} />
     );
 
-    fileMap.forEach((file: File, name: string) => {
-      const form = new FormData();
-      form.append(
-        "file",
-        new File([new Uint8Array(file.content.buffer)], name)
-      );
+    const queue = new PQueue({ concurrency: BULK_UPLOAD_CONCURRENCY });
 
-      // TODO: Consider a retry
+    for (const bucket of buckets) {
+      // Don't upload empty buckets (can happen for tiny projects)
+      if (bucket.files.length === 0) continue;
 
-      const promise = fetchResult<{ id: string }>(
-        `/accounts/${accountId}/pages/projects/${projectName}/file`,
-        {
-          method: "POST",
-          body: form,
-        }
-      ).then((response) => {
-        counter++;
-        rerender(<Progress done={counter} total={fileMap.size} />);
-        if (response.id != file.metadata.hash) {
-          throw new Error(
-            `Looks like there was an issue uploading '${name}'. Try again perhaps?`
-          );
+      const payload: UploadPayloadFile[] = bucket.files.map((file) => ({
+        key: file.hash,
+        value: file.content,
+        metadata: {
+          contentType: file.contentType,
+        },
+        base64: true,
+      }));
+
+      let attempts = 0;
+      const doUpload = async (): Promise<void> => {
+        try {
+          return await fetchResult(`/pages/assets/upload`, {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              Authorization: `Bearer ${jwt}`,
+            },
+            body: JSON.stringify(payload),
+          });
+        } catch (e) {
+          if (attempts < MAX_UPLOAD_ATTEMPTS) {
+            // Linear backoff, 0 second first time, then 1 second etc.
+            await new Promise((resolve) =>
+              setTimeout(resolve, attempts++ * 1000)
+            );
+            return doUpload();
+          } else {
+            throw e;
+          }
         }
-      });
+      };
 
-      files.push(promise);
-    });
+      queue.add(() =>
+        doUpload().then(
+          () => {
+            counter += bucket.files.length;
+            rerender(<Progress done={counter} total={fileMap.size} />);
+          },
+          (error) => {
+            return Promise.reject(
+              new FatalError(
+                "Failed to upload files. Please try again.",
+                error.code || 1
+              )
+            );
+          }
+        )
+      );
+    }
 
-    await Promise.all(files);
+    await queue.onIdle();
 
     unmount();
 
@@ -1169,7 +1279,7 @@ const createDeployment: CommandModule<
         Object.fromEntries(
           [...fileMap.entries()].map(([fileName, file]) => [
             `/${fileName}`,
-            file.metadata.hash,
+            file.hash,
           ])
         )
       )
@@ -1299,6 +1409,12 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               default: false,
               description: "Auto reload HTML pages when change is detected",
             },
+            "node-compat": {
+              describe: "Enable node.js compatibility",
+              default: false,
+              type: "boolean",
+              hidden: true,
+            },
             // TODO: Miniflare user options
           })
           .epilogue(pagesBetaWarning);
@@ -1313,6 +1429,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
         kv: kvs = [],
         do: durableObjects = [],
         "live-reload": liveReload,
+        "node-compat": nodeCompat,
         _: [_pages, _dev, ...remaining],
       }) => {
         // Beta message for `wrangler pages <commands>` usage
@@ -1346,7 +1463,16 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
         );
 
         if (usingFunctions) {
-          const outfile = join(tmpdir(), "./functionsWorker.js");
+          const outfile = join(
+            tmpdir(),
+            `./functionsWorker-${Math.random()}.js`
+          );
+
+          if (nodeCompat) {
+            console.warn(
+              "Enabling node.js compatibility mode for builtins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
+            );
+          }
 
           logger.log(`Compiling worker to "${outfile}"...`);
 
@@ -1358,6 +1484,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               watch: true,
               onEnd: () => scriptReadyResolve(),
               buildOutputDirectory: directory,
+              nodeCompat,
             });
           } catch {}
 
@@ -1372,6 +1499,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               watch: true,
               onEnd: () => scriptReadyResolve(),
               buildOutputDirectory: directory,
+              nodeCompat,
             });
           });
 
@@ -1455,7 +1583,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
 
           // env.ASSETS.fetch
           serviceBindings: {
-            async ASSETS(request: Request) {
+            async ASSETS(request: MiniflareRequest) {
               if (proxyPort) {
                 try {
                   const url = new URL(request.url);
@@ -1579,6 +1707,12 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
                 type: "string",
                 description: "The directory to output static assets to",
               },
+              "node-compat": {
+                describe: "Enable node.js compatibility",
+                default: false,
+                type: "boolean",
+                hidden: true,
+              },
             })
             .epilogue(pagesBetaWarning),
         async ({
@@ -1591,12 +1725,19 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
           watch,
           plugin,
           "build-output-directory": buildOutputDirectory,
+          "node-compat": nodeCompat,
         }) => {
           if (!isInPagesCI) {
             // Beta message for `wrangler pages <commands>` usage
             logger.log(pagesBetaWarning);
           }
 
+          if (nodeCompat) {
+            console.warn(
+              "Enabling node.js compatibility mode for builtins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
+            );
+          }
+
           buildOutputDirectory ??= dirname(outfile);
 
           await buildFunctions({
@@ -1609,6 +1750,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
             watch,
             plugin,
             buildOutputDirectory,
+            nodeCompat,
           });
         }
       )
@@ -1833,7 +1975,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
     } as CommandModule);
 };
 
-const invalidAssetsFetch: typeof fetch = () => {
+const invalidAssetsFetch: typeof miniflareFetch = () => {
   throw new Error(
     "Trying to fetch assets directly when there is no `directory` option specified, and not in `local` mode."
   );