wrangler 2.0.6 → 2.0.9

package/src/pages.tsx

@@ -14,6 +14,7 @@ import SelectInput from "ink-select-input";
 import Spinner from "ink-spinner";
 import Table from "ink-table";
 import { getType } from "mime";
+import PQueue from "p-queue";
 import prettyBytes from "pretty-bytes";
 import React from "react";
 import { format as timeagoFormat } from "timeago.js";
@@ -32,7 +33,11 @@ import openInBrowser from "./open-in-browser";
 import { toUrlPath } from "./paths";
 import { requireAuth } from "./user";
 import type { Config } from "../pages/functions/routes";
-import type { Headers, Request, fetch } from "@miniflare/core";
+import type {
+  Headers as MiniflareHeaders,
+  Request as MiniflareRequest,
+  fetch as miniflareFetch,
+} from "@miniflare/core";
 import type { BuildResult } from "esbuild";
 import type { MiniflareOptions } from "miniflare";
 import type { BuilderCallback, CommandModule } from "yargs";
@@ -68,12 +73,23 @@ export type Deployment = {
   project_name: string;
 };
 
+export type UploadPayloadFile = {
+  key: string;
+  value: string;
+  metadata: { contentType: string };
+  base64: boolean;
+};
+
 interface PagesConfigCache {
   account_id?: string;
   project_name?: string;
 }
 
 const PAGES_CONFIG_CACHE_FILENAME = "pages.json";
+const MAX_BUCKET_SIZE = 50 * 1024 * 1024;
+const MAX_BUCKET_FILE_COUNT = 5000;
+const BULK_UPLOAD_CONCURRENCY = 3;
+const MAX_UPLOAD_ATTEMPTS = 5;
 
 // Defer importing miniflare until we really need it. This takes ~0.5s
 // and also modifies some `stream/web` and `undici` prototypes, so we
@@ -284,7 +300,7 @@ function generateRulesMatcher<T>(
     T
   ][];
 
-  return ({ request }: { request: Request }) => {
+  return ({ request }: { request: MiniflareRequest }) => {
     const { pathname, host } = new URL(request.url);
 
     return compiledRules
@@ -357,7 +373,7 @@ function generateHeadersMatcher(headersFile: string) {
       )
     );
 
-    return (request: Request) => {
+    return (request: MiniflareRequest) => {
       const matches = rulesMatcher({
         request,
       });
@@ -406,7 +422,7 @@ function generateRedirectsMatcher(redirectsFile: string) {
       })
     );
 
-    return (request: Request) => {
+    return (request: MiniflareRequest) => {
       const match = rulesMatcher({
         request,
       })[0];
@@ -461,7 +477,9 @@ function hasFileExtension(pathname: string) {
   return /\/.+\.[a-z0-9]+$/i.test(pathname);
 }
 
-async function generateAssetsFetch(directory: string): Promise<typeof fetch> {
+async function generateAssetsFetch(
+  directory: string
+): Promise<typeof miniflareFetch> {
   // Defer importing miniflare until we really need it
   const { Headers, Request, Response } = await import("@miniflare/core");
 
@@ -510,12 +528,12 @@ async function generateAssetsFetch(directory: string): Promise<typeof fetch> {
     return readFileSync(file);
   };
 
-  const generateResponse = (request: Request) => {
+  const generateResponse = (request: MiniflareRequest) => {
     const url = new URL(request.url);
 
     const deconstructedResponse: {
       status: number;
-      headers: Headers;
+      headers: MiniflareHeaders;
       body?: Buffer;
     } = {
       status: 200,
@@ -667,8 +685,12 @@ async function generateAssetsFetch(directory: string): Promise<typeof fetch> {
   };
 
   const attachHeaders = (
-    request: Request,
-    deconstructedResponse: { status: number; headers: Headers; body?: Buffer }
+    request: MiniflareRequest,
+    deconstructedResponse: {
+      status: number;
+      headers: MiniflareHeaders;
+      body?: Buffer;
+    }
   ) => {
     const headers = deconstructedResponse.headers;
     const newHeaders = new Headers({});
@@ -722,6 +744,7 @@ async function buildFunctions({
   onEnd,
   plugin = false,
   buildOutputDirectory,
+  nodeCompat,
 }: {
   outfile: string;
   outputConfigPath?: string;
@@ -733,12 +756,13 @@ async function buildFunctions({
   onEnd?: () => void;
   plugin?: boolean;
   buildOutputDirectory?: string;
+  nodeCompat?: boolean;
 }) {
   RUNNING_BUILDERS.forEach(
     (runningBuilder) => runningBuilder.stop && runningBuilder.stop()
   );
 
-  const routesModule = join(tmpdir(), "./functionsRoutes.mjs");
+  const routesModule = join(tmpdir(), `./functionsRoutes-${Math.random()}.mjs`);
   const baseURL = toUrlPath("/");
 
   const config: Config = await generateConfigFromFileTree({
@@ -767,6 +791,7 @@ async function buildFunctions({
         minify,
         sourcemap,
         watch,
+        nodeCompat,
         onEnd,
       })
     );
@@ -781,6 +806,7 @@ async function buildFunctions({
         watch,
         onEnd,
         buildOutputDirectory,
+        nodeCompat,
       })
     );
   }
@@ -1001,7 +1027,7 @@ const createDeployment: CommandModule<
 
       if (isGitDirty && !commitDirty) {
         logger.warn(
-          `Warning: Your working directory is a git repo and has uncommitted changes\nTo silense this warning, pass in --commit-dirty=true`
+          `Warning: Your working directory is a git repo and has uncommitted changes\nTo silence this warning, pass in --commit-dirty=true`
         );
       }
 
@@ -1013,7 +1039,7 @@ const createDeployment: CommandModule<
     let builtFunctions: string | undefined = undefined;
     const functionsDirectory = join(cwd(), "functions");
     if (existsSync(functionsDirectory)) {
-      const outfile = join(tmpdir(), "./functionsWorker.js");
+      const outfile = join(tmpdir(), `./functionsWorker-${Math.random()}.js`);
 
       await new Promise((resolve) =>
         buildFunctions({
@@ -1027,12 +1053,9 @@ const createDeployment: CommandModule<
       builtFunctions = readFileSync(outfile, "utf-8");
     }
 
-    type File = {
-      content: Buffer;
-      metadata: Metadata;
-    };
-
-    type Metadata = {
+    type FileContainer = {
+      content: string;
+      contentType: string;
       sizeInBytes: number;
       hash: string;
     };
@@ -1043,11 +1066,12 @@ const createDeployment: CommandModule<
       "_headers",
       ".DS_Store",
       "node_modules",
+      ".git",
     ];
 
     const walk = async (
       dir: string,
-      fileMap: Map<string, File> = new Map(),
+      fileMap: Map<string, FileContainer> = new Map(),
       depth = 0
     ) => {
       const files = await readdir(dir);
@@ -1081,8 +1105,6 @@ const createDeployment: CommandModule<
             const base64Content = fileContent.toString("base64");
             const extension = extname(basename(name)).substring(1);
 
-            const content = base64Content + extension;
-
             if (filestat.size > 25 * 1024 * 1024) {
               throw new Error(
                 `Error: Pages only supports files up to ${prettyBytes(
@@ -1092,11 +1114,12 @@ const createDeployment: CommandModule<
             }
 
             fileMap.set(name, {
-              content: fileContent,
-              metadata: {
-                sizeInBytes: filestat.size,
-                hash: hash(content).toString("hex").slice(0, 32),
-              },
+              content: base64Content,
+              contentType: getType(name) || "application/octet-stream",
+              sizeInBytes: filestat.size,
+              hash: hash(base64Content + extension)
+                .toString("hex")
+                .slice(0, 32),
             });
           }
         })
@@ -1107,51 +1130,150 @@ const createDeployment: CommandModule<
 
     const fileMap = await walk(directory);
 
-    const start = Date.now();
+    if (fileMap.size > 20000) {
+      throw new FatalError(
+        `Error: Pages only supports up to 20,000 files in a deployment. Ensure you have specified your build output directory correctly.`,
+        1
+      );
+    }
 
-    const files: Array<Promise<void>> = [];
+    const files = [...fileMap.values()];
 
-    if (fileMap.size > 1000) {
-      throw new Error(
-        `Error: Pages only supports up to 1,000 files in a deployment at the moment.\nTry a smaller project perhaps?`
-      );
+    async function fetchJwt(): Promise<string> {
+      return (
+        await fetchResult<{ jwt: string }>(
+          `/accounts/${accountId}/pages/projects/${projectName}/upload-token`
+        )
+      ).jwt;
     }
 
-    let counter = 0;
+    let jwt = await fetchJwt();
+
+    const start = Date.now();
 
+    const missingHashes = await fetchResult<string[]>(
+      `/pages/assets/check-missing`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${jwt}`,
+        },
+        body: JSON.stringify({
+          hashes: files.map(({ hash }) => hash),
+        }),
+      }
+    );
+
+    const sortedFiles = files
+      .filter((file) => missingHashes.includes(file.hash))
+      .sort((a, b) => b.sizeInBytes - a.sizeInBytes);
+
+    // Start with a few buckets so small projects still get
+    // the benefit of multiple upload streams
+    const buckets: {
+      files: FileContainer[];
+      remainingSize: number;
+    }[] = new Array(BULK_UPLOAD_CONCURRENCY).fill(null).map(() => ({
+      files: [],
+      remainingSize: MAX_BUCKET_SIZE,
+    }));
+
+    let bucketOffset = 0;
+    for (const file of sortedFiles) {
+      let inserted = false;
+
+      for (let i = 0; i < buckets.length; i++) {
+        // Start at a different bucket for each new file
+        const bucket = buckets[(i + bucketOffset) % buckets.length];
+        if (
+          bucket.remainingSize >= file.sizeInBytes &&
+          bucket.files.length < MAX_BUCKET_FILE_COUNT
+        ) {
+          bucket.files.push(file);
+          bucket.remainingSize -= file.sizeInBytes;
+          inserted = true;
+          break;
+        }
+      }
+
+      if (!inserted) {
+        buckets.push({
+          files: [file],
+          remainingSize: MAX_BUCKET_SIZE - file.sizeInBytes,
+        });
+      }
+      bucketOffset++;
+    }
+
+    let counter = fileMap.size - sortedFiles.length;
     const { rerender, unmount } = render(
       <Progress done={counter} total={fileMap.size} />
     );
 
-    fileMap.forEach((file: File, name: string) => {
-      const form = new FormData();
-      form.append(
-        "file",
-        new File([new Uint8Array(file.content.buffer)], name)
-      );
+    const queue = new PQueue({ concurrency: BULK_UPLOAD_CONCURRENCY });
 
-      // TODO: Consider a retry
+    for (const bucket of buckets) {
+      // Don't upload empty buckets (can happen for tiny projects)
+      if (bucket.files.length === 0) continue;
 
-      const promise = fetchResult<{ id: string }>(
-        `/accounts/${accountId}/pages/projects/${projectName}/file`,
-        {
-          method: "POST",
-          body: form,
-        }
-      ).then((response) => {
-        counter++;
-        rerender(<Progress done={counter} total={fileMap.size} />);
-        if (response.id != file.metadata.hash) {
-          throw new Error(
-            `Looks like there was an issue uploading '${name}'. Try again perhaps?`
-          );
+      const payload: UploadPayloadFile[] = bucket.files.map((file) => ({
+        key: file.hash,
+        value: file.content,
+        metadata: {
+          contentType: file.contentType,
+        },
+        base64: true,
+      }));
+
+      let attempts = 0;
+      const doUpload = async (): Promise<void> => {
+        try {
+          return await fetchResult(`/pages/assets/upload`, {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              Authorization: `Bearer ${jwt}`,
+            },
+            body: JSON.stringify(payload),
+          });
+        } catch (e) {
+          if (attempts < MAX_UPLOAD_ATTEMPTS) {
+            // Linear backoff, 0 second first time, then 1 second etc.
+            await new Promise((resolve) =>
+              setTimeout(resolve, attempts++ * 1000)
+            );
+
+            if ((e as { code: number }).code === 8000013) {
+              // Looks like the JWT expired, fetch another one
+              jwt = await fetchJwt();
+            }
+            return doUpload();
+          } else {
+            throw e;
+          }
         }
-      });
+      };
 
-      files.push(promise);
-    });
+      queue.add(() =>
+        doUpload().then(
+          () => {
+            counter += bucket.files.length;
+            rerender(<Progress done={counter} total={fileMap.size} />);
+          },
+          (error) => {
+            return Promise.reject(
+              new FatalError(
+                "Failed to upload files. Please try again.",
+                error.code || 1
+              )
+            );
+          }
+        )
+      );
+    }
 
-    await Promise.all(files);
+    await queue.onIdle();
 
     unmount();
 
@@ -1169,7 +1291,7 @@ const createDeployment: CommandModule<
         Object.fromEntries(
           [...fileMap.entries()].map(([fileName, file]) => [
             `/${fileName}`,
-            file.metadata.hash,
+            file.hash,
           ])
         )
       )
@@ -1299,6 +1421,12 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               default: false,
               description: "Auto reload HTML pages when change is detected",
             },
+            "node-compat": {
+              describe: "Enable node.js compatibility",
+              default: false,
+              type: "boolean",
+              hidden: true,
+            },
             // TODO: Miniflare user options
           })
           .epilogue(pagesBetaWarning);
@@ -1313,6 +1441,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
         kv: kvs = [],
         do: durableObjects = [],
         "live-reload": liveReload,
+        "node-compat": nodeCompat,
         _: [_pages, _dev, ...remaining],
       }) => {
         // Beta message for `wrangler pages <commands>` usage
@@ -1346,7 +1475,16 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
         );
 
         if (usingFunctions) {
-          const outfile = join(tmpdir(), "./functionsWorker.js");
+          const outfile = join(
+            tmpdir(),
+            `./functionsWorker-${Math.random()}.js`
+          );
+
+          if (nodeCompat) {
+            console.warn(
+              "Enabling node.js compatibility mode for builtins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
+            );
+          }
 
           logger.log(`Compiling worker to "${outfile}"...`);
 
@@ -1358,6 +1496,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               watch: true,
               onEnd: () => scriptReadyResolve(),
               buildOutputDirectory: directory,
+              nodeCompat,
             });
           } catch {}
 
@@ -1372,6 +1511,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               watch: true,
               onEnd: () => scriptReadyResolve(),
               buildOutputDirectory: directory,
+              nodeCompat,
             });
           });
 
@@ -1394,6 +1534,9 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
           } else {
             logger.log("No functions. Shimming...");
             miniflareArgs = {
+              // cfFetch sets the `cf` object that a function could expect
+              // If there are no functions, there's no reason to set this up (and not make that network call)
+              cfFetch: false,
               // TODO: The fact that these request/response hacks are necessary is ridiculous.
               // We need to eliminate them from env.ASSETS.fetch (not sure if just local or prod as well)
               script: `
@@ -1455,7 +1598,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
 
           // env.ASSETS.fetch
           serviceBindings: {
-            async ASSETS(request: Request) {
+            async ASSETS(request: MiniflareRequest) {
               if (proxyPort) {
                 try {
                   const url = new URL(request.url);
@@ -1579,6 +1722,12 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
                 type: "string",
                 description: "The directory to output static assets to",
               },
+              "node-compat": {
+                describe: "Enable node.js compatibility",
+                default: false,
+                type: "boolean",
+                hidden: true,
+              },
             })
             .epilogue(pagesBetaWarning),
         async ({
@@ -1591,12 +1740,19 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
           watch,
           plugin,
           "build-output-directory": buildOutputDirectory,
+          "node-compat": nodeCompat,
         }) => {
           if (!isInPagesCI) {
             // Beta message for `wrangler pages <commands>` usage
             logger.log(pagesBetaWarning);
           }
 
+          if (nodeCompat) {
+            console.warn(
+              "Enabling node.js compatibility mode for builtins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
+            );
+          }
+
           buildOutputDirectory ??= dirname(outfile);
 
           await buildFunctions({
@@ -1609,6 +1765,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
             watch,
             plugin,
             buildOutputDirectory,
+            nodeCompat,
           });
         }
       )
@@ -1833,7 +1990,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
     } as CommandModule);
 };
 
-const invalidAssetsFetch: typeof fetch = () => {
+const invalidAssetsFetch: typeof miniflareFetch = () => {
   throw new Error(
     "Trying to fetch assets directly when there is no `directory` option specified, and not in `local` mode."
   );

package/src/parse.ts

@@ -62,8 +62,6 @@ export class ParseError extends Error implements Message {
   }
 }
 
-/* eslint-disable @typescript-eslint/no-explicit-any */
-
 const TOML_ERROR_NAME = "TomlError";
 const TOML_ERROR_SUFFIX = " at row ";
 
@@ -78,7 +76,7 @@ type TomlError = Error & {
 export function parseTOML(input: string, file?: string): TOML.JsonMap | never {
   try {
     // Normalize CRLF to LF to avoid hitting https://github.com/iarna/iarna-toml/issues/33.
-    const normalizedInput = input.replace(/\r\n$/g, "\n");
+    const normalizedInput = input.replace(/\r\n/g, "\n");
     return TOML.parse(normalizedInput);
   } catch (err) {
     const { name, message, line, col } = err as TomlError;
@@ -100,10 +98,29 @@ export function parseTOML(input: string, file?: string): TOML.JsonMap | never {
 
 const JSON_ERROR_SUFFIX = " in JSON at position ";
 
+/**
+ * A minimal type describing a package.json file.
+ */
+export type PackageJSON = {
+  devDependencies?: Record<string, unknown>;
+  dependencies?: Record<string, unknown>;
+  scripts?: Record<string, unknown>;
+};
+
+/**
+ * A typed version of `parseJSON()`.
+ */
+export function parsePackageJSON<T extends PackageJSON = PackageJSON>(
+  input: string,
+  file?: string
+): T {
+  return parseJSON<T>(input, file);
+}
+
 /**
  * A wrapper around `JSON.parse` that throws a `ParseError`.
  */
-export function parseJSON(input: string, file?: string): any {
+export function parseJSON<T>(input: string, file?: string): T {
   try {
     return JSON.parse(input);
   } catch (err) {