wrangler 0.0.0-e9a1820 → 0.0.0-e9ab55a

package/src/proxy.ts

@@ -1,97 +1,245 @@
 import { connect } from "node:http2";
+import type { ClientHttp2Session, ServerHttp2Stream } from "node:http2";
 import { createServer } from "node:http";
-import type { Server, IncomingHttpHeaders, RequestListener } from "node:http";
+import type {
+  IncomingHttpHeaders,
+  RequestListener,
+  IncomingMessage,
+  ServerResponse,
+  Server,
+} from "node:http";
 import WebSocket from "faye-websocket";
 import serveStatic from "serve-static";
+import type { CfPreviewToken } from "./api/preview";
+import { useEffect, useRef } from "react";
 
-export interface HttpProxyInit {
-  host: string;
-  assetPath?: string | null;
-  onRequest?: (headers: IncomingHttpHeaders) => void;
-  onResponse?: (headers: IncomingHttpHeaders) => void;
+/**
+ * `usePreviewServer` is a React hook that creates a local development
+ * server that can be used to develop a Worker.
+ *
+ * When we run `wrangler dev`, we start by uploading the compiled worker
+ * to the preview service, which responds with a preview token.
+ * (see `useWorker`/`createWorker` for details.)
+ * We can then use that token to connect to the preview server for a
+ * great local development experience. Further, as we change the worker,
+ * we can update the preview token transparently without having to restart
+ * the development server.
+ */
+
+/** Rewrite request headers to add the preview token. */
+function addCfPreviewTokenHeader(
+  headers: IncomingHttpHeaders,
+  previewTokenValue: string
+) {
+  headers["cf-workers-preview-token"] = previewTokenValue;
 }
 
 /**
- * Creates a HTTP/1 proxy that sends requests over HTTP/2.
+ * Rewrite references in request headers
+ * from the preview host to the local host.
  */
-export function createHttpProxy(init: HttpProxyInit): Server {
-  const { host, assetPath, onRequest = () => {}, onResponse = () => {} } = init;
-  const remote = connect(`https://${host}`);
-  const local = createServer();
-  // HTTP/2 -> HTTP/2
-  local.on("stream", (stream, headers: IncomingHttpHeaders) => {
-    onRequest(headers);
-    headers[":authority"] = host;
-    const request = stream.pipe(remote.request(headers));
-    request.on("response", (responseHeaders: IncomingHttpHeaders) => {
-      onResponse(responseHeaders);
-      stream.respond(responseHeaders);
-      request.pipe(stream, { end: true });
-    });
-  });
-  // HTTP/1 -> HTTP/2
-  const handleRequest: RequestListener = (message, response) => {
-    const { httpVersionMajor, headers, method, url } = message;
-    if (httpVersionMajor >= 2) {
-      return; // Already handled by the "stream" event.
+function rewriteRemoteHostToLocalHostInHeaders(
+  headers: IncomingHttpHeaders,
+  remoteHost: string,
+  localPort: number
+) {
+  for (const [name, value] of Object.entries(headers)) {
+    // Rewrite the remote host to the local host.
+    if (typeof value === "string" && value.includes(remoteHost)) {
+      headers[name] = value
+        .replaceAll(`https://${remoteHost}`, `http://localhost:${localPort}`)
+        .replaceAll(remoteHost, `localhost:${localPort}`);
     }
-    onRequest(headers);
-    headers[":method"] = method;
-    headers[":path"] = url;
-    headers[":authority"] = host;
-    headers[":scheme"] = "https";
-    for (const name of Object.keys(headers)) {
-      if (HTTP1_HEADERS.has(name.toLowerCase())) {
-        delete headers[name];
-      }
+  }
+}
+
+export function usePreviewServer({
+  previewToken,
+  publicRoot,
+  port,
+}: {
+  previewToken: CfPreviewToken | undefined;
+  publicRoot: undefined | string;
+  port: number;
+}) {
+  /** Creates an HTTP/1 proxy that sends requests over HTTP/2. */
+  const proxyServer = useRef<Server>();
+  const proxy = (proxyServer.current ??= createProxyServer());
+
+  /**
+   * When we're not connected / getting a fresh token on changes,
+   * we'd like to buffer streams/requests until we're connected.
+   * Once connected, we can flush the buffered streams/requests.
+   * streamBufferRef is used to buffer http/2 streams, while
+   * requestResponseBufferRef is used to buffer http/1 requests.
+   */
+  const streamBufferRef = useRef<
+    { stream: ServerHttp2Stream; headers: IncomingHttpHeaders }[]
+  >([]);
+  const requestResponseBufferRef = useRef<
+    { request: IncomingMessage; response: ServerResponse }[]
+  >([]);
+
+  useEffect(() => {
+    // If we don't have a token, that means either we're just starting up,
+    // or we're refreshing the token.
+    if (!previewToken) {
+      const cleanupListeners: (() => void)[] = [];
+      const bufferStream = (
+        stream: ServerHttp2Stream,
+        headers: IncomingHttpHeaders
+      ) => {
+        // store the stream in a buffer so we can replay it later
+        streamBufferRef.current.push({ stream, headers });
+      };
+      proxy.on("stream", bufferStream);
+      cleanupListeners.push(() => proxy.off("stream", bufferStream));
+
+      const bufferRequestResponse = (
+        request: IncomingMessage,
+        response: ServerResponse
+      ) => {
+        // store the request and response in a buffer so we can replay it later
+        requestResponseBufferRef.current.push({ request, response });
+      };
+
+      proxy.on("request", bufferRequestResponse);
+      cleanupListeners.push(() => proxy.off("request", bufferRequestResponse));
+      return () => {
+        cleanupListeners.forEach((cleanup) => cleanup());
+      };
     }
-    const request = message.pipe(remote.request(headers));
-    request.on("response", (responseHeaders) => {
-      const status = responseHeaders[":status"];
-      onResponse(responseHeaders);
-      for (const name of Object.keys(responseHeaders)) {
-        if (name.startsWith(":")) {
-          delete responseHeaders[name];
+
+    // We have a token. Let's proxy requests to the preview end point.
+    const cleanupListeners: (() => void)[] = [];
+
+    const assetPath = typeof publicRoot === "string" ? publicRoot : null;
+
+    // create a ClientHttp2Session
+    const remote = connect(`https://${previewToken.host}`);
+    cleanupListeners.push(() => remote.destroy());
+
+    /** HTTP/2 -> HTTP/2  */
+    const handleStream = createStreamHandler(previewToken, remote, port);
+    proxy.on("stream", handleStream);
+    cleanupListeners.push(() => proxy.off("stream", handleStream));
+
+    // flush and replay buffered streams
+    streamBufferRef.current.forEach((buffer) =>
+      handleStream(buffer.stream, buffer.headers)
+    );
+    streamBufferRef.current = [];
+
+    /** HTTP/1 -> HTTP/2  */
+    const handleRequest: RequestListener = (
+      message: IncomingMessage,
+      response: ServerResponse
+    ) => {
+      const { httpVersionMajor, headers, method, url } = message;
+      if (httpVersionMajor >= 2) {
+        return; // Already handled by the "stream" event.
+      }
+      addCfPreviewTokenHeader(headers, previewToken.value);
+      headers[":method"] = method;
+      headers[":path"] = url;
+      headers[":authority"] = previewToken.host;
+      headers[":scheme"] = "https";
+      for (const name of Object.keys(headers)) {
+        if (HTTP1_HEADERS.has(name.toLowerCase())) {
+          delete headers[name];
         }
       }
-      response.writeHead(status, responseHeaders);
-      request.pipe(response, { end: true });
-    });
-  };
-  // If an asset path is defined, check the file system
-  // for a file first and serve if it exists.
-  if (assetPath) {
-    const handleAsset = serveStatic(assetPath, {
-      cacheControl: false,
-    });
-    local.on("request", (request, response) => {
-      handleAsset(request, response, () => {
-        handleRequest(request, response);
+      const request = message.pipe(remote.request(headers));
+      request.on("response", (responseHeaders) => {
+        const status = responseHeaders[":status"] ?? 500;
+        rewriteRemoteHostToLocalHostInHeaders(
+          responseHeaders,
+          previewToken.host,
+          port
+        );
+        for (const name of Object.keys(responseHeaders)) {
+          if (name.startsWith(":")) {
+            delete responseHeaders[name];
+          }
+        }
+        response.writeHead(status, responseHeaders);
+        request.pipe(response, { end: true });
       });
-    });
-  } else {
-    local.on("request", handleRequest);
-  }
-  // HTTP/1 -> WebSocket (over HTTP/1)
-  local.on("upgrade", (message, socket, body) => {
-    const { headers, url } = message;
-    onRequest(headers);
-    headers["host"] = host;
-    const localWebsocket = new WebSocket(message, socket, body);
-    // TODO(soon): Custom WebSocket protocol is not working?
-    const remoteWebsocketClient = new WebSocket.Client(
-      `wss://${host}${url}`,
-      [],
-      { headers }
+    };
+
+    // If an asset path is defined, check the file system
+    // for a file first and serve if it exists.
+    const actualHandleRequest = assetPath
+      ? createHandleAssetsRequest(assetPath, handleRequest)
+      : handleRequest;
+
+    proxy.on("request", actualHandleRequest);
+    cleanupListeners.push(() => proxy.off("request", actualHandleRequest));
+
+    // flush and replay buffered requests
+    requestResponseBufferRef.current.forEach(({ request, response }) =>
+      actualHandleRequest(request, response)
     );
-    localWebsocket.pipe(remoteWebsocketClient).pipe(localWebsocket);
-  });
-  remote.on("close", () => {
-    local.close();
+    requestResponseBufferRef.current = [];
+
+    /** HTTP/1 -> WebSocket (over HTTP/1)  */
+    const handleUpgrade = (
+      message: IncomingMessage,
+      socket: WebSocket,
+      body: Buffer
+    ) => {
+      const { headers, url } = message;
+      addCfPreviewTokenHeader(headers, previewToken.value);
+      headers["host"] = previewToken.host;
+      const localWebsocket = new WebSocket(message, socket, body);
+      // TODO(soon): Custom WebSocket protocol is not working?
+      const remoteWebsocketClient = new WebSocket.Client(
+        `wss://${previewToken.host}${url}`,
+        [],
+        { headers }
+      );
+      localWebsocket.pipe(remoteWebsocketClient).pipe(localWebsocket);
+      // We close down websockets whenever we refresh the token.
+      cleanupListeners.push(() => {
+        localWebsocket.destroy();
+        remoteWebsocketClient.destroy();
+      });
+    };
+    proxy.on("upgrade", handleUpgrade);
+    cleanupListeners.push(() => proxy.off("upgrade", handleUpgrade));
+
+    return () => {
+      cleanupListeners.forEach((d) => d());
+    };
+  }, [previewToken, publicRoot, port, proxy]);
+
+  // Start/stop the server whenever the
+  // containing component is mounted/unmounted.
+  useEffect(() => {
+    proxy.listen(port);
+    console.log(`⬣ Listening at http://localhost:${port}`);
+
+    return () => {
+      proxy.close();
+    };
+  }, [port, proxy]);
+}
+
+function createHandleAssetsRequest(
+  assetPath: string,
+  handleRequest: RequestListener
+) {
+  const handleAsset = serveStatic(assetPath, {
+    cacheControl: false,
   });
-  return local;
+  return (request: IncomingMessage, response: ServerResponse) => {
+    handleAsset(request, response, () => {
+      handleRequest(request, response);
+    });
+  };
 }
 
+/** A Set of headers we want to remove from HTTP/1 requests. */
 const HTTP1_HEADERS = new Set([
   "host",
   "connection",
@@ -101,3 +249,54 @@ const HTTP1_HEADERS = new Set([
   "transfer-encoding",
   "http2-settings",
 ]);
+
+function createProxyServer() {
+  return createServer()
+    .on("request", function (req, res) {
+      // log all requests
+      console.log(
+        new Date().toLocaleTimeString(),
+        req.method,
+        req.url,
+        res.statusCode
+      );
+    })
+    .on("upgrade", (req) => {
+      // log all websocket connections
+      console.log(
+        new Date().toLocaleTimeString(),
+        req.method,
+        req.url,
+        101,
+        "(WebSocket)"
+      );
+    })
+    .on("error", (err) => {
+      // log all connection errors
+      console.error(new Date().toLocaleTimeString(), err);
+    });
+}
+
+function createStreamHandler(
+  previewToken: CfPreviewToken,
+  remote: ClientHttp2Session,
+  port: number
+) {
+  return function handleStream(
+    stream: ServerHttp2Stream,
+    headers: IncomingHttpHeaders
+  ) {
+    addCfPreviewTokenHeader(headers, previewToken.value);
+    headers[":authority"] = previewToken.host;
+    const request = stream.pipe(remote.request(headers));
+    request.on("response", (responseHeaders: IncomingHttpHeaders) => {
+      rewriteRemoteHostToLocalHostInHeaders(
+        responseHeaders,
+        previewToken.host,
+        port
+      );
+      stream.respond(responseHeaders);
+      request.pipe(stream, { end: true });
+    });
+  };
+}

package/src/publish.ts

@@ -1,17 +1,18 @@
+import assert from "node:assert";
+import path from "node:path";
+import { readFile } from "node:fs/promises";
+import * as esbuild from "esbuild";
+import type { Metafile } from "esbuild";
+import { execa } from "execa";
+import tmp from "tmp-promise";
 import type { CfWorkerInit } from "./api/worker";
 import { toFormData } from "./api/form_data";
-import esbuild from "esbuild";
-import tmp from "tmp-promise";
+import { fetchResult } from "./cfetch";
 import type { Config } from "./config";
-import path from "path";
-import { readFile } from "fs/promises";
-import cfetch from "./cfetch";
-import assert from "node:assert";
-import { syncAssets } from "./sites";
 import makeModuleCollector from "./module-collection";
-import { execa } from "execa";
+import { syncAssets } from "./sites";
 
-type CfScriptFormat = void | "modules" | "service-worker";
+type CfScriptFormat = undefined | "modules" | "service-worker";
 
 type Props = {
   config: Config;
@@ -19,13 +20,15 @@ type Props = {
   script?: string;
   name?: string;
   env?: string;
+  compatibilityDate?: string;
+  compatibilityFlags?: string[];
   public?: string;
   site?: string;
   triggers?: (string | number)[];
   routes?: (string | number)[];
   legacyEnv?: boolean;
-  jsxFactory: void | string;
-  jsxFragment: void | string;
+  jsxFactory: undefined | string;
+  jsxFragment: undefined | string;
 };
 
 function sleep(ms: number) {
@@ -48,6 +51,18 @@ export default async function publish(props: Props): Promise<void> {
     __path__,
   } = config;
 
+  const envRootObj =
+    props.env && config.env ? config.env[props.env] || {} : config;
+
+  assert(
+    envRootObj.compatibility_date || props.compatibilityDate,
+    "A compatibility_date is required when publishing. Add one to your wrangler.toml file, or pass it in your terminal as --compatibility_date. See https://developers.cloudflare.com/workers/platform/compatibility-dates for more information."
+  );
+
+  if (accountId === undefined) {
+    throw new Error("No account_id provided.");
+  }
+
   const triggers = props.triggers || config.triggers?.crons;
   const routes = props.routes || config.routes;
 
@@ -64,10 +79,12 @@ export default async function publish(props: Props): Promise<void> {
 
   let file: string;
   if (props.script) {
-    file = props.script;
+    // If the script name comes from the command line it is relative to the current working directory.
+    file = path.resolve(props.script);
   } else {
+    // If the script name comes from the config, then it is relative to the wrangler.toml file.
     assert(build?.upload?.main, "missing main file");
-    file = path.join(path.dirname(__path__), build.upload.main);
+    file = path.resolve(path.dirname(__path__), build.upload.main);
   }
 
   if (props.legacyEnv) {
@@ -76,7 +93,6 @@ export default async function publish(props: Props): Promise<void> {
   const envName = props.env ?? "production";
 
   const destination = await tmp.dir({ unsafeCleanup: true });
-
   if (props.config.build?.command) {
     // TODO: add a deprecation message here?
     console.log("running:", props.config.build.command);
@@ -98,7 +114,7 @@ export default async function publish(props: Props): Promise<void> {
                 path.join(__dirname, "../static-asset-facade.js"),
                 "utf8"
               )
-            ).replace("__ENTRY_POINT__", path.join(process.cwd(), file)),
+            ).replace("__ENTRY_POINT__", file),
             sourcefile: "static-asset-facade.js",
             resolveDir: path.dirname(file),
           },
@@ -120,18 +136,33 @@ export default async function publish(props: Props): Promise<void> {
     ...(jsxFragment && { jsxFragment }),
   });
 
-  const chunks = Object.entries(result.metafile.outputs).find(
-    ([_path, { entryPoint }]) =>
-      entryPoint ===
-      (props.public
-        ? path.join(path.dirname(file), "static-asset-facade.js")
-        : file)
+  // result.metafile is defined because of the `metafile: true` option above.
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  const metafile = result.metafile!;
+  const entryPoints = Object.values(metafile.outputs).filter(
+    (output) => output.entryPoint !== undefined
+  );
+  assert(
+    entryPoints.length > 0,
+    `Cannot find entry-point "${file}" in generated bundle.` +
+      listEntryPoints(entryPoints)
+  );
+  assert(
+    entryPoints.length < 2,
+    "More than one entry-point found for generated bundle." +
+      listEntryPoints(entryPoints)
+  );
+  const entryPointExports = entryPoints[0].exports;
+  const resolvedEntryPointPath = path.resolve(
+    destination.path,
+    // We know that entryPoint is not null because we filtered out those without above.
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    entryPoints[0].entryPoint!
   );
-
   const { format } = props;
   const bundle = {
-    type: chunks[1].exports.length > 0 ? "esm" : "commonjs",
-    exports: chunks[1].exports,
+    type: entryPointExports.length > 0 ? "esm" : "commonjs",
+    exports: entryPointExports,
   };
 
   // TODO: instead of bundling the facade with the worker, we should just bundle the worker and expose it as a module.
@@ -148,14 +179,14 @@ export default async function publish(props: Props): Promise<void> {
     return;
   }
 
-  const content = await readFile(chunks[0], { encoding: "utf-8" });
+  const content = await readFile(resolvedEntryPointPath, { encoding: "utf-8" });
   await destination.cleanup();
 
   // if config.migrations
   // get current migration tag
   let migrations;
-  if ("migrations" in config) {
-    const scripts = await cfetch<{ id: string; migration_tag: string }[]>(
+  if (config.migrations !== undefined) {
+    const scripts = await fetchResult<{ id: string; migration_tag: string }[]>(
       `/accounts/${accountId}/workers/scripts`
     );
     const script = scripts.find(({ id }) => id === scriptName);
@@ -190,17 +221,11 @@ export default async function publish(props: Props): Promise<void> {
     }
   }
 
-  const assets =
-    props.public || props.site || props.config.site?.bucket // TODO: allow both
-      ? await syncAssets(
-          accountId,
-          scriptName,
-          props.public || props.site || props.config.site?.bucket,
-          false
-        )
-      : { manifest: undefined, namespace: undefined };
-
-  const envRootObj = props.env ? config.env[props.env] || {} : config;
+  const assetPath = props.public || props.site || props.config.site?.bucket; // TODO: allow both
+  const assets = assetPath
+    ? await syncAssets(accountId, scriptName, assetPath, false)
+    : { manifest: undefined, namespace: undefined };
+
   const bindings: CfWorkerInit["bindings"] = {
     kv_namespaces: envRootObj.kv_namespaces?.concat(
       assets.namespace
@@ -215,7 +240,7 @@ export default async function publish(props: Props): Promise<void> {
   const worker: CfWorkerInit = {
     name: scriptName,
     main: {
-      name: path.basename(chunks[0]),
+      name: path.basename(resolvedEntryPointPath),
       content: content,
       type: bundle.type === "esm" ? "esm" : "commonjs",
     },
@@ -246,14 +271,15 @@ export default async function publish(props: Props): Promise<void> {
     ? `/accounts/${accountId}/workers/services/${scriptName}/environments/${envName}`
     : `/accounts/${accountId}/workers/scripts/${scriptName}`;
 
-  // Upload the script so it has time to propogate.
-  const { available_on_subdomain } = await cfetch(
-    `${workerUrl}?available_on_subdomain=true`,
+  // Upload the script so it has time to propagate.
+  const { available_on_subdomain } = await fetchResult(
+    workerUrl,
     {
       method: "PUT",
       // @ts-expect-error: TODO: fix this type error!
       body: toFormData(worker),
-    }
+    },
+    new URLSearchParams({ available_on_subdomains: "true" })
   );
 
   const uploadMs = Date.now() - start;
@@ -261,7 +287,7 @@ export default async function publish(props: Props): Promise<void> {
   const deployments: Promise<string[]>[] = [];
 
   const userSubdomain = (
-    await cfetch<{ subdomain: string }>(
+    await fetchResult<{ subdomain: string }>(
       `/accounts/${accountId}/workers/subdomain`
     )
   ).subdomain;
@@ -275,7 +301,7 @@ export default async function publish(props: Props): Promise<void> {
   // TODO: Make this configurable.
   if (!available_on_subdomain) {
     deployments.push(
-      cfetch(`${workerUrl}/subdomain`, {
+      fetchResult(`${workerUrl}/subdomain`, {
         method: "POST",
         body: JSON.stringify({ enabled: true }),
         headers: {
@@ -299,7 +325,7 @@ export default async function publish(props: Props): Promise<void> {
   // Update routing table for the script.
   if (routes && routes.length) {
     deployments.push(
-      cfetch(`${workerUrl}/routes`, {
+      fetchResult(`${workerUrl}/routes`, {
         // TODO: PATCH will not delete previous routes on this script,
         // whereas PUT will. We need to decide on the default behaviour
         // and how to configure it.
@@ -324,7 +350,7 @@ export default async function publish(props: Props): Promise<void> {
   // TODO: rename this to `schedules`?
   if (triggers && triggers.length) {
     deployments.push(
-      cfetch(`${workerUrl}/schedules`, {
+      fetchResult(`${workerUrl}/schedules`, {
         // TODO: Unlike routes, this endpoint does not support PATCH.
         // So technically, this will override any previous schedules.
         // We should change the endpoint to support PATCH.
@@ -348,3 +374,9 @@ export default async function publish(props: Props): Promise<void> {
     console.log(" ", target);
   }
 }
+
+function listEntryPoints(outputs: ValueOf<Metafile["outputs"]>[]): string {
+  return outputs.map((output) => output.entryPoint).join("\n");
+}
+
+type ValueOf<T> = T[keyof T];

package/src/sites.tsx

@@ -1,11 +1,10 @@
-import { readdir, readFile } from "node:fs/promises";
+import crypto from "node:crypto";
 import { createReadStream } from "node:fs";
-import cfetch from "./cfetch";
+import * as path from "node:path";
+import { readdir, readFile } from "node:fs/promises";
+import { fetchResult } from "./cfetch";
 import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv";
 
-import * as path from "path";
-import crypto from "node:crypto";
-
 async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   const files = await readdir(dirPath, { withFileTypes: true });
   for (const file of files) {
@@ -54,7 +53,7 @@ async function createKVNamespaceIfNotAlreadyExisting(
 
   // else we make the namespace
   // TODO: use an export from ./kv
-  const json = await cfetch<{ id: string }>(
+  const json = await fetchResult<{ id: string }>(
     `/accounts/${accountId}/storage/kv/namespaces`,
     {
       method: "POST",
@@ -90,7 +89,11 @@ export async function syncAssets(
   );
 
   const manifest = {};
-  const upload = [];
+  const upload: {
+    key: string;
+    value: string;
+    base64: boolean;
+  }[] = [];
   // TODO: this can be more efficient by parallelising
   for await (const file of getFilesInFolder(dirPath)) {
     // TODO: "exclude:" config