wrangler 0.0.0-e1c615f → 0.0.0-e5d3690

package/src/pages.tsx

@@ -1,10 +1,9 @@
 /* eslint-disable no-shadow */
 
-import assert from "assert";
 import type { BuilderCallback } from "yargs";
 import { join } from "path";
 import { tmpdir } from "os";
-import { existsSync, lstatSync, readFileSync } from "fs";
+import { existsSync, lstatSync, readFileSync, writeFileSync } from "fs";
 import { execSync, spawn } from "child_process";
 import { URL } from "url";
 import { getType } from "mime";
@@ -22,7 +21,7 @@ import { generateConfigFromFileTree } from "../pages/functions/filepath-routing"
 import type { Headers, Request, fetch } from "@miniflare/core";
 import type { MiniflareOptions } from "miniflare";
 
-const EXIT_CALLBACKS = [];
+const EXIT_CALLBACKS: (() => void)[] = [];
 const EXIT = (message?: string, code?: number) => {
   if (message) console.log(message);
   if (code) process.exitCode = code;
@@ -122,7 +121,9 @@ async function spawnProxyProcess({
       },
     }
   );
-  EXIT_CALLBACKS.push(() => proxy.kill());
+  EXIT_CALLBACKS.push(() => {
+    proxy.kill();
+  });
 
   proxy.stdout.on("data", (data) => {
     console.log(`[proxy]: ${data}`);
@@ -642,16 +643,20 @@ const RUNNING_BUILDERS: BuildResult[] = [];
 
 async function buildFunctions({
   scriptPath,
+  outputConfigPath,
   functionsDirectory,
   minify = false,
   sourcemap = false,
+  fallbackService = "ASSETS",
   watch = false,
   onEnd,
 }: {
   scriptPath: string;
+  outputConfigPath?: string;
   functionsDirectory: string;
   minify?: boolean;
   sourcemap?: boolean;
+  fallbackService?: string;
   watch?: boolean;
   onEnd?: () => void;
 }) {
@@ -666,6 +671,13 @@ async function buildFunctions({
     baseURL: "/",
   });
 
+  if (outputConfigPath) {
+    writeFileSync(
+      outputConfigPath,
+      JSON.stringify({ ...config, baseURL: "/" }, null, 2)
+    );
+  }
+
   await writeRoutesModule({
     config,
     srcDir: functionsDirectory,
@@ -678,6 +690,7 @@ async function buildFunctions({
       outfile: scriptPath,
       minify,
       sourcemap,
+      fallbackService,
       watch,
       onEnd,
     })
@@ -738,6 +751,11 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               description: "Durable Object to bind (NAME=CLASS)",
               alias: "o",
             },
+            "live-reload": {
+              type: "boolean",
+              default: false,
+              description: "Auto reload HTML pages when change is detected",
+            },
             // TODO: Miniflare user options
           });
       },
@@ -750,6 +768,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
         binding: bindings = [],
         kv: kvs = [],
         do: durableObjects = [],
+        "live-reload": liveReload,
         "--": remaining = [],
       }) => {
         if (!local) {
@@ -774,7 +793,10 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
 
         let miniflareArgs: MiniflareOptions = {};
 
-        let scriptReady = true;
+        let scriptReadyResolve;
+        const scriptReadyPromise = new Promise(
+          (resolve) => (scriptReadyResolve = resolve)
+        );
 
         if (usingFunctions) {
           const scriptPath = join(tmpdir(), "./functionsWorker.js");
@@ -786,22 +808,19 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
             functionsDirectory,
             sourcemap: true,
             watch: true,
-            onEnd: () => {
-              scriptReady = true;
-            },
+            onEnd: () => scriptReadyResolve(),
           });
 
           watch([functionsDirectory], {
             persistent: true,
+            ignoreInitial: true,
           }).on("all", async () => {
             await buildFunctions({
               scriptPath,
               functionsDirectory,
               sourcemap: true,
               watch: true,
-              onEnd: () => {
-                scriptReady = true;
-              },
+              onEnd: () => scriptReadyResolve(),
             });
           });
 
@@ -838,11 +857,19 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
         const { Miniflare, Log, LogLevel } = await import("miniflare");
         const { Response, fetch } = await import("@miniflare/core");
 
-        // Should only be called if no proxyPort, using `assert.fail()` here
-        // means the type of `assetsFetch` is still `typeof fetch`
-        const assetsFetch = proxyPort
-          ? () => assert.fail()
-          : await generateAssetsFetch(directory);
+        // Wait for esbuild to finish building before starting Miniflare.
+        // This must be before the call to `new Miniflare`, as that will
+        // asynchronously start loading the script. `await startServer()`
+        // internally just waits for that promise to resolve.
+        await scriptReadyPromise;
+
+        // `assetsFetch()` will only be called if there is `proxyPort` defined.
+        // We only define `proxyPort`, above, when there is no `directory` defined.
+        const assetsFetch =
+          directory !== undefined
+            ? await generateAssetsFetch(directory)
+            : invalidAssetsFetch;
+
         const miniflare = new Miniflare({
           port,
           watch: true,
@@ -873,7 +900,7 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
               if (proxyPort) {
                 try {
                   const url = new URL(request.url);
-                  url.host = `127.0.0.1:${proxyPort}`;
+                  url.host = `localhost:${proxyPort}`;
                   return await fetch(url, request);
                 } catch (thrown) {
                   console.error(`Could not proxy request: ${thrown}`);
@@ -903,22 +930,29 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
           kvPersist: true,
           durableObjectsPersist: true,
           cachePersist: true,
+          liveReload,
 
           ...miniflareArgs,
         });
 
-        // Wait for esbuild to finish building the script
-        while (!scriptReady) {}
-
         try {
           // `startServer` might throw if user code contains errors
           const server = await miniflare.startServer();
-          console.log(`Serving at http://127.0.0.1:${port}/`);
+          console.log(`Serving at http://localhost:${port}/`);
 
           if (process.env.BROWSER !== "none") {
-            try {
-              await open(`http://127.0.0.1:${port}/`);
-            } catch {}
+            const childProcess = await open(`http://localhost:${port}/`);
+            // fail silently if the open command doesn't work (e.g. in GitHub Codespaces)
+            childProcess.on("error", (_err) => {});
+          }
+
+          if (directory !== undefined && liveReload) {
+            watch([directory], {
+              persistent: true,
+              ignoreInitial: true,
+            }).on("all", async () => {
+              await miniflare.reload();
+            });
           }
 
           EXIT_CALLBACKS.push(() => {
@@ -933,36 +967,74 @@ export const pages: BuilderCallback<unknown, unknown> = (yargs) => {
     )
     .command("functions", "Cloudflare Pages Functions", (yargs) =>
       yargs.command(
-        "build",
+        "build [directory]",
         "Compile a folder of Cloudflare Pages Functions into a single Worker",
         (yargs) =>
-          yargs.options({
-            "script-path": {
+          yargs
+            .positional("directory", {
               type: "string",
-              default: "_worker.js",
-              description: "The location of the output Worker script",
-            },
-            minify: {
-              type: "boolean",
-              default: false,
-              description: "Minify the output Worker script",
-            },
-            sourcemap: {
-              type: "boolean",
-              default: false,
-              description: "Generate a sourcemap for the output Worker script",
-            },
-          }),
-        async ({ "script-path": scriptPath, minify, sourcemap }) => {
-          const functionsDirectory = "./functions";
-
+              default: "functions",
+              description: "The directory of Pages Functions",
+            })
+            .options({
+              "script-path": {
+                type: "string",
+                default: "_worker.js",
+                description: "The location of the output Worker script",
+              },
+              "output-config-path": {
+                type: "string",
+                description: "The location for the output config file",
+              },
+              minify: {
+                type: "boolean",
+                default: false,
+                description: "Minify the output Worker script",
+              },
+              sourcemap: {
+                type: "boolean",
+                default: false,
+                description:
+                  "Generate a sourcemap for the output Worker script",
+              },
+              "fallback-service": {
+                type: "string",
+                default: "ASSETS",
+                description:
+                  "The service to fallback to at the end of the `next` chain. Setting to '' will fallback to the global `fetch`.",
+              },
+              watch: {
+                type: "boolean",
+                default: false,
+                description:
+                  "Watch for changes to the functions and automatically rebuild the Worker script",
+              },
+            }),
+        async ({
+          directory,
+          "script-path": scriptPath,
+          "output-config-path": outputConfigPath,
+          minify,
+          sourcemap,
+          fallbackService,
+          watch,
+        }) => {
           await buildFunctions({
             scriptPath,
-            functionsDirectory,
+            outputConfigPath,
+            functionsDirectory: directory,
             minify,
             sourcemap,
+            fallbackService,
+            watch,
           });
         }
       )
     );
 };
+
+const invalidAssetsFetch: typeof fetch = () => {
+  throw new Error(
+    "Trying to fetch assets directly when there is no `directory` option specified, and not in `local` mode."
+  );
+};

package/src/proxy.ts

@@ -1,97 +1,245 @@
 import { connect } from "node:http2";
+import type { ClientHttp2Session, ServerHttp2Stream } from "node:http2";
 import { createServer } from "node:http";
-import type { Server, IncomingHttpHeaders, RequestListener } from "node:http";
+import type {
+  IncomingHttpHeaders,
+  RequestListener,
+  IncomingMessage,
+  ServerResponse,
+  Server,
+} from "node:http";
 import WebSocket from "faye-websocket";
 import serveStatic from "serve-static";
+import type { CfPreviewToken } from "./api/preview";
+import { useEffect, useRef } from "react";
 
-export interface HttpProxyInit {
-  host: string;
-  assetPath?: string | null;
-  onRequest?: (headers: IncomingHttpHeaders) => void;
-  onResponse?: (headers: IncomingHttpHeaders) => void;
+/**
+ * `usePreviewServer` is a React hook that creates a local development
+ * server that can be used to develop a Worker.
+ *
+ * When we run `wrangler dev`, we start by uploading the compiled worker
+ * to the preview service, which responds with a preview token.
+ * (see `useWorker`/`createWorker` for details.)
+ * We can then use that token to connect to the preview server for a
+ * great local development experience. Further, as we change the worker,
+ * we can update the preview token transparently without having to restart
+ * the development server.
+ */
+
+/** Rewrite request headers to add the preview token. */
+function addCfPreviewTokenHeader(
+  headers: IncomingHttpHeaders,
+  previewTokenValue: string
+) {
+  headers["cf-workers-preview-token"] = previewTokenValue;
 }
 
 /**
- * Creates a HTTP/1 proxy that sends requests over HTTP/2.
+ * Rewrite references in request headers
+ * from the preview host to the local host.
  */
-export function createHttpProxy(init: HttpProxyInit): Server {
-  const { host, assetPath, onRequest = () => {}, onResponse = () => {} } = init;
-  const remote = connect(`https://${host}`);
-  const local = createServer();
-  // HTTP/2 -> HTTP/2
-  local.on("stream", (stream, headers: IncomingHttpHeaders) => {
-    onRequest(headers);
-    headers[":authority"] = host;
-    const request = stream.pipe(remote.request(headers));
-    request.on("response", (responseHeaders: IncomingHttpHeaders) => {
-      onResponse(responseHeaders);
-      stream.respond(responseHeaders);
-      request.pipe(stream, { end: true });
-    });
-  });
-  // HTTP/1 -> HTTP/2
-  const handleRequest: RequestListener = (message, response) => {
-    const { httpVersionMajor, headers, method, url } = message;
-    if (httpVersionMajor >= 2) {
-      return; // Already handled by the "stream" event.
+function rewriteRemoteHostToLocalHostInHeaders(
+  headers: IncomingHttpHeaders,
+  remoteHost: string,
+  localPort: number
+) {
+  for (const [name, value] of Object.entries(headers)) {
+    // Rewrite the remote host to the local host.
+    if (typeof value === "string" && value.includes(remoteHost)) {
+      headers[name] = value
+        .replaceAll(`https://${remoteHost}`, `http://localhost:${localPort}`)
+        .replaceAll(remoteHost, `localhost:${localPort}`);
     }
-    onRequest(headers);
-    headers[":method"] = method;
-    headers[":path"] = url;
-    headers[":authority"] = host;
-    headers[":scheme"] = "https";
-    for (const name of Object.keys(headers)) {
-      if (HTTP1_HEADERS.has(name.toLowerCase())) {
-        delete headers[name];
-      }
+  }
+}
+
+export function usePreviewServer({
+  previewToken,
+  publicRoot,
+  port,
+}: {
+  previewToken: CfPreviewToken | undefined;
+  publicRoot: undefined | string;
+  port: number;
+}) {
+  /** Creates an HTTP/1 proxy that sends requests over HTTP/2. */
+  const proxyServer = useRef<Server>();
+  const proxy = (proxyServer.current ??= createProxyServer());
+
+  /**
+   * When we're not connected / getting a fresh token on changes,
+   * we'd like to buffer streams/requests until we're connected.
+   * Once connected, we can flush the buffered streams/requests.
+   * streamBufferRef is used to buffer http/2 streams, while
+   * requestResponseBufferRef is used to buffer http/1 requests.
+   */
+  const streamBufferRef = useRef<
+    { stream: ServerHttp2Stream; headers: IncomingHttpHeaders }[]
+  >([]);
+  const requestResponseBufferRef = useRef<
+    { request: IncomingMessage; response: ServerResponse }[]
+  >([]);
+
+  useEffect(() => {
+    // If we don't have a token, that means either we're just starting up,
+    // or we're refreshing the token.
+    if (!previewToken) {
+      const cleanupListeners: (() => void)[] = [];
+      const bufferStream = (
+        stream: ServerHttp2Stream,
+        headers: IncomingHttpHeaders
+      ) => {
+        // store the stream in a buffer so we can replay it later
+        streamBufferRef.current.push({ stream, headers });
+      };
+      proxy.on("stream", bufferStream);
+      cleanupListeners.push(() => proxy.off("stream", bufferStream));
+
+      const bufferRequestResponse = (
+        request: IncomingMessage,
+        response: ServerResponse
+      ) => {
+        // store the request and response in a buffer so we can replay it later
+        requestResponseBufferRef.current.push({ request, response });
+      };
+
+      proxy.on("request", bufferRequestResponse);
+      cleanupListeners.push(() => proxy.off("request", bufferRequestResponse));
+      return () => {
+        cleanupListeners.forEach((cleanup) => cleanup());
+      };
     }
-    const request = message.pipe(remote.request(headers));
-    request.on("response", (responseHeaders) => {
-      const status = responseHeaders[":status"];
-      onResponse(responseHeaders);
-      for (const name of Object.keys(responseHeaders)) {
-        if (name.startsWith(":")) {
-          delete responseHeaders[name];
+
+    // We have a token. Let's proxy requests to the preview end point.
+    const cleanupListeners: (() => void)[] = [];
+
+    const assetPath = typeof publicRoot === "string" ? publicRoot : null;
+
+    // create a ClientHttp2Session
+    const remote = connect(`https://${previewToken.host}`);
+    cleanupListeners.push(() => remote.destroy());
+
+    /** HTTP/2 -> HTTP/2  */
+    const handleStream = createStreamHandler(previewToken, remote, port);
+    proxy.on("stream", handleStream);
+    cleanupListeners.push(() => proxy.off("stream", handleStream));
+
+    // flush and replay buffered streams
+    streamBufferRef.current.forEach((buffer) =>
+      handleStream(buffer.stream, buffer.headers)
+    );
+    streamBufferRef.current = [];
+
+    /** HTTP/1 -> HTTP/2  */
+    const handleRequest: RequestListener = (
+      message: IncomingMessage,
+      response: ServerResponse
+    ) => {
+      const { httpVersionMajor, headers, method, url } = message;
+      if (httpVersionMajor >= 2) {
+        return; // Already handled by the "stream" event.
+      }
+      addCfPreviewTokenHeader(headers, previewToken.value);
+      headers[":method"] = method;
+      headers[":path"] = url;
+      headers[":authority"] = previewToken.host;
+      headers[":scheme"] = "https";
+      for (const name of Object.keys(headers)) {
+        if (HTTP1_HEADERS.has(name.toLowerCase())) {
+          delete headers[name];
         }
       }
-      response.writeHead(status, responseHeaders);
-      request.pipe(response, { end: true });
-    });
-  };
-  // If an asset path is defined, check the file system
-  // for a file first and serve if it exists.
-  if (assetPath) {
-    const handleAsset = serveStatic(assetPath, {
-      cacheControl: false,
-    });
-    local.on("request", (request, response) => {
-      handleAsset(request, response, () => {
-        handleRequest(request, response);
+      const request = message.pipe(remote.request(headers));
+      request.on("response", (responseHeaders) => {
+        const status = responseHeaders[":status"] ?? 500;
+        rewriteRemoteHostToLocalHostInHeaders(
+          responseHeaders,
+          previewToken.host,
+          port
+        );
+        for (const name of Object.keys(responseHeaders)) {
+          if (name.startsWith(":")) {
+            delete responseHeaders[name];
+          }
+        }
+        response.writeHead(status, responseHeaders);
+        request.pipe(response, { end: true });
       });
-    });
-  } else {
-    local.on("request", handleRequest);
-  }
-  // HTTP/1 -> WebSocket (over HTTP/1)
-  local.on("upgrade", (message, socket, body) => {
-    const { headers, url } = message;
-    onRequest(headers);
-    headers["host"] = host;
-    const localWebsocket = new WebSocket(message, socket, body);
-    // TODO(soon): Custom WebSocket protocol is not working?
-    const remoteWebsocketClient = new WebSocket.Client(
-      `wss://${host}${url}`,
-      [],
-      { headers }
+    };
+
+    // If an asset path is defined, check the file system
+    // for a file first and serve if it exists.
+    const actualHandleRequest = assetPath
+      ? createHandleAssetsRequest(assetPath, handleRequest)
+      : handleRequest;
+
+    proxy.on("request", actualHandleRequest);
+    cleanupListeners.push(() => proxy.off("request", actualHandleRequest));
+
+    // flush and replay buffered requests
+    requestResponseBufferRef.current.forEach(({ request, response }) =>
+      actualHandleRequest(request, response)
     );
-    localWebsocket.pipe(remoteWebsocketClient).pipe(localWebsocket);
-  });
-  remote.on("close", () => {
-    local.close();
+    requestResponseBufferRef.current = [];
+
+    /** HTTP/1 -> WebSocket (over HTTP/1)  */
+    const handleUpgrade = (
+      message: IncomingMessage,
+      socket: WebSocket,
+      body: Buffer
+    ) => {
+      const { headers, url } = message;
+      addCfPreviewTokenHeader(headers, previewToken.value);
+      headers["host"] = previewToken.host;
+      const localWebsocket = new WebSocket(message, socket, body);
+      // TODO(soon): Custom WebSocket protocol is not working?
+      const remoteWebsocketClient = new WebSocket.Client(
+        `wss://${previewToken.host}${url}`,
+        [],
+        { headers }
+      );
+      localWebsocket.pipe(remoteWebsocketClient).pipe(localWebsocket);
+      // We close down websockets whenever we refresh the token.
+      cleanupListeners.push(() => {
+        localWebsocket.destroy();
+        remoteWebsocketClient.destroy();
+      });
+    };
+    proxy.on("upgrade", handleUpgrade);
+    cleanupListeners.push(() => proxy.off("upgrade", handleUpgrade));
+
+    return () => {
+      cleanupListeners.forEach((d) => d());
+    };
+  }, [previewToken, publicRoot, port, proxy]);
+
+  // Start/stop the server whenever the
+  // containing component is mounted/unmounted.
+  useEffect(() => {
+    proxy.listen(port);
+    console.log(`⬣ Listening at http://localhost:${port}`);
+
+    return () => {
+      proxy.close();
+    };
+  }, [port, proxy]);
+}
+
+function createHandleAssetsRequest(
+  assetPath: string,
+  handleRequest: RequestListener
+) {
+  const handleAsset = serveStatic(assetPath, {
+    cacheControl: false,
   });
-  return local;
+  return (request: IncomingMessage, response: ServerResponse) => {
+    handleAsset(request, response, () => {
+      handleRequest(request, response);
+    });
+  };
 }
 
+/** A Set of headers we want to remove from HTTP/1 requests. */
 const HTTP1_HEADERS = new Set([
   "host",
   "connection",
@@ -101,3 +249,54 @@ const HTTP1_HEADERS = new Set([
   "transfer-encoding",
   "http2-settings",
 ]);
+
+function createProxyServer() {
+  return createServer()
+    .on("request", function (req, res) {
+      // log all requests
+      console.log(
+        new Date().toLocaleTimeString(),
+        req.method,
+        req.url,
+        res.statusCode
+      );
+    })
+    .on("upgrade", (req) => {
+      // log all websocket connections
+      console.log(
+        new Date().toLocaleTimeString(),
+        req.method,
+        req.url,
+        101,
+        "(WebSocket)"
+      );
+    })
+    .on("error", (err) => {
+      // log all connection errors
+      console.error(new Date().toLocaleTimeString(), err);
+    });
+}
+
+function createStreamHandler(
+  previewToken: CfPreviewToken,
+  remote: ClientHttp2Session,
+  port: number
+) {
+  return function handleStream(
+    stream: ServerHttp2Stream,
+    headers: IncomingHttpHeaders
+  ) {
+    addCfPreviewTokenHeader(headers, previewToken.value);
+    headers[":authority"] = previewToken.host;
+    const request = stream.pipe(remote.request(headers));
+    request.on("response", (responseHeaders: IncomingHttpHeaders) => {
+      rewriteRemoteHostToLocalHostInHeaders(
+        responseHeaders,
+        previewToken.host,
+        port
+      );
+      stream.respond(responseHeaders);
+      request.pipe(stream, { end: true });
+    });
+  };
+}