wrangler 0.0.0-e95e5a0 → 0.0.0-e978986

package/src/pages.tsx

@@ -4,7 +4,7 @@ import { execSync, spawn } from "node:child_process";
 import { existsSync, lstatSync, readFileSync, writeFileSync } from "node:fs";
 import { readdir, readFile, stat } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { dirname, join, sep } from "node:path";
+import { dirname, join, sep, extname, basename } from "node:path";
 import { cwd } from "node:process";
 import { URL } from "node:url";
 import { hash } from "blake3-wasm";
@@ -14,6 +14,7 @@ import SelectInput from "ink-select-input";
 import Spinner from "ink-spinner";
 import Table from "ink-table";
 import { getType } from "mime";
+import PQueue from "p-queue";
 import prettyBytes from "pretty-bytes";
 import React from "react";
 import { format as timeagoFormat } from "timeago.js";
@@ -32,7 +33,11 @@ import openInBrowser from "./open-in-browser";
 import { toUrlPath } from "./paths";
 import { requireAuth } from "./user";
 import type { Config } from "../pages/functions/routes";
-import type { Headers, Request, fetch } from "@miniflare/core";
+import type {
+  Headers as MiniflareHeaders,
+  Request as MiniflareRequest,
+  fetch as miniflareFetch,
+} from "@miniflare/core";
 import type { BuildResult } from "esbuild";
 import type { MiniflareOptions } from "miniflare";
 import type { BuilderCallback, CommandModule } from "yargs";
@@ -68,12 +73,23 @@ export type Deployment = {
   project_name: string;
 };
 
+export type UploadPayloadFile = {
+  key: string;
+  value: string;
+  metadata: { contentType: string };
+  base64: boolean;
+};
+
 interface PagesConfigCache {
   account_id?: string;
   project_name?: string;
 }
 
 const PAGES_CONFIG_CACHE_FILENAME = "pages.json";
+const MAX_BUCKET_SIZE = 50 * 1024 * 1024;
+const MAX_BUCKET_FILE_COUNT = 5000;
+const BULK_UPLOAD_CONCURRENCY = 3;
+const MAX_UPLOAD_ATTEMPTS = 5;
 
 // Defer importing miniflare until we really need it. This takes ~0.5s
 // and also modifies some `stream/web` and `undici` prototypes, so we
@@ -82,6 +98,8 @@ const PAGES_CONFIG_CACHE_FILENAME = "pages.json";
 export const pagesBetaWarning =
   "🚧 'wrangler pages <command>' is a beta command. Please report any issues to https://github.com/cloudflare/wrangler2/issues/new/choose";
 
+const isInPagesCI = !!process.env.CF_PAGES;
+
 const CLEANUP_CALLBACKS: (() => void)[] = [];
 const CLEANUP = () => {
   CLEANUP_CALLBACKS.forEach((callback) => callback());
@@ -282,7 +300,7 @@ function generateRulesMatcher<T>(
     T
   ][];
 
-  return ({ request }: { request: Request }) => {
+  return ({ request }: { request: MiniflareRequest }) => {
     const { pathname, host } = new URL(request.url);
 
     return compiledRules
@@ -355,7 +373,7 @@ function generateHeadersMatcher(headersFile: string) {
       )
     );
 
-    return (request: Request) => {
+    return (request: MiniflareRequest) => {
       const matches = rulesMatcher({
         request,
       });
@@ -404,7 +422,7 @@ function generateRedirectsMatcher(redirectsFile: string) {
       })
     );
 
-    return (request: Request) => {
+    return (request: MiniflareRequest) => {
       const match = rulesMatcher({
         request,
       })[0];
@@ -459,7 +477,9 @@ function hasFileExtension(pathname: string) {
   return /\/.+\.[a-z0-9]+$/i.test(pathname);
 }
 
-async function generateAssetsFetch(directory: string): Promise<typeof fetch> {
+async function generateAssetsFetch(
+  directory: string
+): Promise<typeof miniflareFetch> {
   // Defer importing miniflare until we really need it
   const { Headers, Request, Response } = await import("@miniflare/core");
 
@@ -508,12 +528,12 @@ async function generateAssetsFetch(directory: string): Promise<typeof fetch> {
     return readFileSync(file);
   };
 
-  const generateResponse = (request: Request) => {
+  const generateResponse = (request: MiniflareRequest) => {
     const url = new URL(request.url);
 
     const deconstructedResponse: {
       status: number;
-      headers: Headers;
+      headers: MiniflareHeaders;
       body?: Buffer;
     } = {
       status: 200,
@@ -665,8 +685,12 @@ async function generateAssetsFetch(directory: string): Promise<typeof fetch> {
   };
 
   const attachHeaders = (
-    request: Request,
-    deconstructedResponse: { status: number; headers: Headers; body?: Buffer }
+    request: MiniflareRequest,
+    deconstructedResponse: {
+      status: number;
+      headers: MiniflareHeaders;
+      body?: Buffer;
+    }
   ) => {
     const headers = deconstructedResponse.headers;
     const newHeaders = new Headers({});
@@ -720,6 +744,7 @@ async function buildFunctions({
   onEnd,
   plugin = false,
   buildOutputDirectory,
+  nodeCompat,
 }: {
   outfile: string;
   outputConfigPath?: string;
@@ -731,12 +756,13 @@ async function buildFunctions({
   onEnd?: () => void;
   plugin?: boolean;
   buildOutputDirectory?: string;
+  nodeCompat?: boolean;
 }) {
   RUNNING_BUILDERS.forEach(
     (runningBuilder) => runningBuilder.stop && runningBuilder.stop()
   );
 
-  const routesModule = join(tmpdir(), "./functionsRoutes.mjs");
+  const routesModule = join(tmpdir(), `./functionsRoutes-${Math.random()}.mjs`);
   const baseURL = toUrlPath("/");
 
   const config: Config = await generateConfigFromFileTree({
@@ -765,6 +791,7 @@ async function buildFunctions({
         minify,
         sourcemap,
         watch,
+        nodeCompat,
         onEnd,
       })
     );
@@ -779,6 +806,7 @@ async function buildFunctions({
         watch,
         onEnd,
         buildOutputDirectory,
+        nodeCompat,
       })
     );
   }
@@ -802,34 +830,30 @@ const createDeployment: CommandModule<
     return yargs
       .positional("directory", {
         type: "string",
-        default: "functions",
-        description: "The directory of Pages Functions",
+        demandOption: true,
+        description: "The directory of static files to upload",
       })
       .options({
         "project-name": {
           type: "string",
-          description:
-            "The name of the project you want to list deployments for",
+          description: "The name of the project you want to deploy to",
         },
         branch: {
           type: "string",
-          description:
-            "The branch of the project you want to list deployments for",
+          description: "The name of the branch you want to deploy to",
         },
         "commit-hash": {
           type: "string",
-          description:
-            "The branch of the project you want to list deployments for",
+          description: "The SHA to attach to this deployment",
         },
         "commit-message": {
           type: "string",
-          description:
-            "The branch of the project you want to list deployments for",
+          description: "The commit message to attach to this deployment",
         },
         "commit-dirty": {
           type: "boolean",
           description:
-            "The branch of the project you want to list deployments for",
+            "Whether or not the workspace should be considered dirty for this deployment",
         },
       })
       .epilogue(pagesBetaWarning);
@@ -842,6 +866,10 @@ const createDeployment: CommandModule<
     commitMessage,
     commitDirty,
   }) => {
+    if (!directory) {
+      throw new FatalError("Must specify a directory.", 1);
+    }
+
     const config = getConfigCache<PagesConfigCache>(
       PAGES_CONFIG_CACHE_FILENAME
     );
@@ -851,40 +879,45 @@ const createDeployment: CommandModule<
 
     const isInteractive = process.stdin.isTTY;
     if (!projectName && isInteractive) {
-      const existingOrNew = await new Promise<"new" | "existing">((resolve) => {
-        const { unmount } = render(
-          <>
-            <Text>
-              No project selected. Would you like to create one or use an
-              existing project?
-            </Text>
-            <SelectInput
-              items={[
-                {
-                  key: "new",
-                  label: "Create a new project",
-                  value: "new",
-                },
-                {
-                  key: "existing",
-                  label: "Use an existing project",
-                  value: "existing",
-                },
-              ]}
-              onSelect={async (selected) => {
-                resolve(selected.value as "new" | "existing");
-                unmount();
-              }}
-            />
-          </>
-        );
-      });
+      const projects = (await listProjects({ accountId })).filter(
+        (project) => !project.source
+      );
+
+      let existingOrNew: "existing" | "new" = "new";
+
+      if (projects.length > 0) {
+        existingOrNew = await new Promise<"new" | "existing">((resolve) => {
+          const { unmount } = render(
+            <>
+              <Text>
+                No project selected. Would you like to create one or use an
+                existing project?
+              </Text>
+              <SelectInput
+                items={[
+                  {
+                    key: "new",
+                    label: "Create a new project",
+                    value: "new",
+                  },
+                  {
+                    key: "existing",
+                    label: "Use an existing project",
+                    value: "existing",
+                  },
+                ]}
+                onSelect={async (selected) => {
+                  resolve(selected.value as "new" | "existing");
+                  unmount();
+                }}
+              />
+            </>
+          );
+        });
+      }
 
       switch (existingOrNew) {
         case "existing": {
-          const projects = (await listProjects({ accountId })).filter(
-            (project) => !project.source
-          );
           projectName = await new Promise((resolve) => {
             const { unmount } = render(
               <>
@@ -994,7 +1027,7 @@ const createDeployment: CommandModule<
 
       if (isGitDirty && !commitDirty) {
         logger.warn(
-          `Warning: Your working directory is a git repo and has uncommitted changes\nTo silense this warning, pass in --commit-dirty=true`
+          `Warning: Your working directory is a git repo and has uncommitted changes\nTo silence this warning, pass in --commit-dirty=true`
         );
       }
 
@@ -1006,7 +1039,7 @@ const createDeployment: CommandModule<
     let builtFunctions: string | undefined = undefined;
     const functionsDirectory = join(cwd(), "functions");
     if (existsSync(functionsDirectory)) {
-      const outfile = join(tmpdir(), "./functionsWorker.js");
+      const outfile = join(tmpdir(), `./functionsWorker-${Math.random()}.js`);
 
       await new Promise((resolve) =>
         buildFunctions({
@@ -1020,12 +1053,9 @@ const createDeployment: CommandModule<
       builtFunctions = readFileSync(outfile, "utf-8");
     }
 
-    type File = {
-      content: Buffer;
-      metadata: Metadata;
-    };
-
-    type Metadata = {
+    type FileContainer = {
+      content: string;
+      contentType: string;
       sizeInBytes: number;
       hash: string;
     };
@@ -1036,11 +1066,12 @@ const createDeployment: CommandModule<
       "_headers",
       ".DS_Store",
       "node_modules",
+      ".git",
     ];
 
     const walk = async (
       dir: string,
-      fileMap: Map<string, File> = new Map(),
+      fileMap: Map<string, FileContainer> = new Map(),
       depth = 0
     ) => {
       const files = await readdir(dir);
@@ -1072,10 +1103,7 @@ const createDeployment: CommandModule<
             const fileContent = await readFile(filepath);
 
             const base64Content = fileContent.toString("base64");
-            const extension =
-              name.split(".").length > 1 ? name.split(".").at(-1) || "" : "";
-
-            const content = base64Content + extension;
+            const extension = extname(basename(name)).substring(1);
 
             if (filestat.size > 25 * 1024 * 1024) {
               throw new Error(
@@ -1086,11 +1114,12 @@ const createDeployment: CommandModule<
             }
 
             fileMap.set(name, {
-              content: fileContent,
-              metadata: {
-                sizeInBytes: filestat.size,
-                hash: hash(content).toString("hex").slice(0, 32),
-              },
+              content: base64Content,
+              contentType: getType(name) || "application/octet-stream",
+              sizeInBytes: filestat.size,
+              hash: hash(base64Content + extension)
+                .toString("hex")
+                .slice(0, 32),
             });
           }
         })
@@ -1101,51 +1130,139 @@ const createDeployment: CommandModule<
 
     const fileMap = await walk(directory);
 
+    if (fileMap.size > 20000) {
+      throw new FatalError(
+        `Error: Pages only supports up to 20,000 files in a deployment. Ensure you have specified your build output directory correctly.`,
+        1
+      );
+    }
+
+    const files = [...fileMap.values()];
+
+    const { jwt } = await fetchResult<{ jwt: string }>(
+      `/accounts/${accountId}/pages/projects/${projectName}/upload-token`
+    );
+
     const start = Date.now();
 
-    const files: Array<Promise<void>> = [];
+    const missingHashes = await fetchResult<string[]>(
+      `/pages/assets/check-missing`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${jwt}`,
+        },
+        body: JSON.stringify({
+          hashes: files.map(({ hash }) => hash),
+        }),
+      }
+    );
 
-    if (fileMap.size > 1000) {
-      throw new Error(
-        `Error: Pages only supports up to 1,000 files in a deployment at the moment.\nTry a smaller project perhaps?`
-      );
-    }
+    const sortedFiles = files
+      .filter((file) => missingHashes.includes(file.hash))
+      .sort((a, b) => b.sizeInBytes - a.sizeInBytes);
+
+    // Start with a few buckets so small projects still get
+    // the benefit of multiple upload streams
+    const buckets: {
+      files: FileContainer[];
+      remainingSize: number;
+    }[] = new Array(BULK_UPLOAD_CONCURRENCY).fill(null).map(() => ({
+      files: [],
+      remainingSize: MAX_BUCKET_SIZE,
+    }));
+
+    let bucketOffset = 0;
+    for (const file of sortedFiles) {
+      let inserted = false;
+
+      for (let i = 0; i < buckets.length; i++) {
+        // Start at a different bucket for each new file
+        const bucket = buckets[(i + bucketOffset) % buckets.length];
+        if (
+          bucket.remainingSize >= file.sizeInBytes &&
+          bucket.files.length < MAX_BUCKET_FILE_COUNT
+        ) {
+          bucket.files.push(file);
+          bucket.remainingSize -= file.sizeInBytes;
+          inserted = true;
+          break;
+        }
+      }
 
-    let counter = 0;
+      if (!inserted) {
+        buckets.push({
+          files: [file],
+          remainingSize: MAX_BUCKET_SIZE - file.sizeInBytes,
+        });
+      }
+      bucketOffset++;
+    }
 
+    let counter = fileMap.size - sortedFiles.length;
     const { rerender, unmount } = render(
       <Progress done={counter} total={fileMap.size} />
     );
 
-    fileMap.forEach((file: File, name: string) => {
-      const form = new FormData();
-      form.append(
-        "file",
-        new File([new Uint8Array(file.content.buffer)], name)
-      );
+    const queue = new PQueue({ concurrency: BULK_UPLOAD_CONCURRENCY });
 
-      // TODO: Consider a retry
+    for (const bucket of buckets) {
+      // Don't upload empty buckets (can happen for tiny projects)
+      if (bucket.files.length === 0) continue;
 
-      const promise = fetchResult<{ id: string }>(
-        `/accounts/${accountId}/pages/projects/${projectName}/file`,
-        {
-          method: "POST",
-          body: form,
-        }
-      ).then((response) => {
-        counter++;
-        rerender(<Progress done={counter} total={fileMap.size} />);
-        if (response.id != file.metadata.hash) {
-          throw new Error(
-            `Looks like there was an issue uploading '${name}'. Try again perhaps?`
-          );
+      const payload: UploadPayloadFile[] = bucket.files.map((file) => ({
+        key: file.hash,
+        value: file.content,
+        metadata: {
+          contentType: file.contentType,
+        },
+        base64: true,
+      }));
+
+      let attempts = 0;
+      const doUpload = async (): Promise<void> => {
+        try {
+          return await fetchResult(`/pages/assets/upload`, {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              Authorization: `Bearer ${jwt}`,
+            },
+            body: JSON.stringify(payload),
+          });
+        } catch (e) {
+          if (attempts < MAX_UPLOAD_ATTEMPTS) {
+            // Linear backoff, 0 second first time, then 1 second etc.
+            await new Promise((resolve) =>
+              setTimeout(resolve, attempts++ * 1000)
+            );
+            return doUpload();
+          } else {
+            throw e;
+          }
         }
-      });
+      };
 
-      files.push(promise);
-    });
+      queue.add(() =>
+        doUpload().then(
+          () => {
+            counter += bucket.files.length;
+            rerender(<Progress done={counter} total={fileMap.size} />);
+          },
+          (error) => {
+            return Promise.reject(
+              new FatalError(
+                "Failed to upload files. Please try again.",
+                error.code || 1
+              )
+            );
+          }
+        )
+      );
+    }
 
-    await Promise.all(files);
+    await queue.onIdle();
 
     unmount();
 
@@ -1163,7 +1280,7 @@ const createDeployment: CommandModule<
         Object.fromEntries(
           [...fileMap.entries()].map(([fileName, file]) => [
             `/${fileName}`,
-            file.metadata.hash,
+            file.hash,
           ])
         )
       )
@@ -1237,7 +1354,7 @@ const createDeployment: CommandModule<
 export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
   return yargs
     .command(
-      "dev [directory] [-- command]",
+      "dev [directory] [-- command..]",
       "🧑‍💻 Develop your full-stack Pages application locally",
       (yargs) => {
         return yargs
@@ -1293,6 +1410,12 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               default: false,
               description: "Auto reload HTML pages when change is detected",
             },
+            "node-compat": {
+              describe: "Enable node.js compatibility",
+              default: false,
+              type: "boolean",
+              hidden: true,
+            },
             // TODO: Miniflare user options
           })
           .epilogue(pagesBetaWarning);
@@ -1307,6 +1430,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
         kv: kvs = [],
         do: durableObjects = [],
         "live-reload": liveReload,
+        "node-compat": nodeCompat,
         _: [_pages, _dev, ...remaining],
       }) => {
         // Beta message for `wrangler pages <commands>` usage
@@ -1340,7 +1464,16 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
         );
 
         if (usingFunctions) {
-          const outfile = join(tmpdir(), "./functionsWorker.js");
+          const outfile = join(
+            tmpdir(),
+            `./functionsWorker-${Math.random()}.js`
+          );
+
+          if (nodeCompat) {
+            console.warn(
+              "Enabling node.js compatibility mode for builtins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
+            );
+          }
 
           logger.log(`Compiling worker to "${outfile}"...`);
 
@@ -1352,6 +1485,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               watch: true,
               onEnd: () => scriptReadyResolve(),
               buildOutputDirectory: directory,
+              nodeCompat,
             });
           } catch {}
 
@@ -1366,6 +1500,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               watch: true,
               onEnd: () => scriptReadyResolve(),
               buildOutputDirectory: directory,
+              nodeCompat,
             });
           });
 
@@ -1388,6 +1523,9 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
           } else {
             logger.log("No functions. Shimming...");
             miniflareArgs = {
+              // cfFetch sets the `cf` object that a function could expect
+              // If there are no functions, there's no reason to set this up (and not make that network call)
+              cfFetch: false,
               // TODO: The fact that these request/response hacks are necessary is ridiculous.
               // We need to eliminate them from env.ASSETS.fetch (not sure if just local or prod as well)
               script: `
@@ -1449,7 +1587,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
 
           // env.ASSETS.fetch
           serviceBindings: {
-            async ASSETS(request: Request) {
+            async ASSETS(request: MiniflareRequest) {
               if (proxyPort) {
                 try {
                   const url = new URL(request.url);
@@ -1569,6 +1707,16 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
                 default: false,
                 description: "Build a plugin rather than a Worker script",
               },
+              "build-output-directory": {
+                type: "string",
+                description: "The directory to output static assets to",
+              },
+              "node-compat": {
+                describe: "Enable node.js compatibility",
+                default: false,
+                type: "boolean",
+                hidden: true,
+              },
             })
             .epilogue(pagesBetaWarning),
         async ({
@@ -1580,9 +1728,21 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
           fallbackService,
           watch,
           plugin,
+          "build-output-directory": buildOutputDirectory,
+          "node-compat": nodeCompat,
         }) => {
-          // Beta message for `wrangler pages <commands>` usage
-          logger.log(pagesBetaWarning);
+          if (!isInPagesCI) {
+            // Beta message for `wrangler pages <commands>` usage
+            logger.log(pagesBetaWarning);
+          }
+
+          if (nodeCompat) {
+            console.warn(
+              "Enabling node.js compatibility mode for builtins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
+            );
+          }
+
+          buildOutputDirectory ??= dirname(outfile);
 
           await buildFunctions({
             outfile,
@@ -1593,7 +1753,8 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
             fallbackService,
             watch,
             plugin,
-            buildOutputDirectory: dirname(outfile),
+            buildOutputDirectory,
+            nodeCompat,
           });
         }
       )
@@ -1818,7 +1979,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
     } as CommandModule);
 };
 
-const invalidAssetsFetch: typeof fetch = () => {
+const invalidAssetsFetch: typeof miniflareFetch = () => {
   throw new Error(
     "Trying to fetch assets directly when there is no `directory` option specified, and not in `local` mode."
   );