wrangler 2.0.24 → 2.0.25

package/src/miniflare-cli/index.ts

@@ -1,10 +1,23 @@
-import { Log, LogLevel, Miniflare } from "miniflare";
+import { fetch } from "@miniflare/core";
+import {
+	DurableObjectNamespace,
+	DurableObjectStub,
+} from "@miniflare/durable-objects";
+import {
+	Log,
+	LogLevel,
+	Miniflare,
+	Response as MiniflareResponse,
+	Request as MiniflareRequest,
+} from "miniflare";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
+import { FatalError } from "../errors";
 import generateASSETSBinding from "./assets";
 import { enumKeys } from "./enum-keys";
 import { getRequestContextCheckOptions } from "./request-context";
 import type { Options } from "./assets";
+import type { AddressInfo } from "net";
 
 export interface EnablePagesAssetsServiceBindingOptions {
 	proxyPort?: number;
@@ -44,7 +57,45 @@ async function main() {
 		console.log("OPTIONS:\n", JSON.stringify(config, null, 2));
 	}
 
+	config.bindings = {
+		...config.bindings,
+		...Object.fromEntries(
+			Object.entries(
+				config.externalDurableObjects as Record<
+					string,
+					{ name: string; host: string; port: number }
+				>
+			).map(([binding, { name, host, port }]) => {
+				const factory = () => {
+					throw new FatalError(
+						"An external Durable Object instance's state has somehow been attempted to be accessed.",
+						1
+					);
+				};
+				const namespace = new DurableObjectNamespace(name as string, factory);
+				namespace.get = (id) => {
+					const stub = new DurableObjectStub(factory, id);
+					stub.fetch = (...reqArgs) => {
+						const url = `http://${host}${port ? `:${port}` : ""}`;
+						const request = new MiniflareRequest(
+							url,
+							new MiniflareRequest(...reqArgs)
+						);
+						request.headers.set("x-miniflare-durable-object-name", name);
+						request.headers.set("x-miniflare-durable-object-id", id.toString());
+
+						return fetch(request);
+					};
+					return stub;
+				};
+				return [binding, namespace];
+			})
+		),
+	};
+
 	let mf: Miniflare | undefined;
+	let durableObjectsMf: Miniflare | undefined = undefined;
+	let durableObjectsMfPort: number | undefined = undefined;
 
 	try {
 		if (args._[1]) {
@@ -73,12 +124,80 @@ async function main() {
 		// Start Miniflare development server
 		await mf.startServer();
 		await mf.startScheduler();
-		process.send && process.send("ready");
+
+		const internalDurableObjectClassNames = Object.values(
+			config.durableObjects as Record<string, string>
+		);
+
+		if (internalDurableObjectClassNames.length > 0) {
+			durableObjectsMf = new Miniflare({
+				host: config.host,
+				port: 0,
+				script: `
+				export default {
+					fetch(request, env) {
+						return env.DO.fetch(request)
+					}
+				}`,
+				serviceBindings: {
+					DO: async (request: MiniflareRequest) => {
+						request = new MiniflareRequest(request);
+
+						const name = request.headers.get("x-miniflare-durable-object-name");
+						const idString = request.headers.get(
+							"x-miniflare-durable-object-id"
+						);
+						request.headers.delete("x-miniflare-durable-object-name");
+						request.headers.delete("x-miniflare-durable-object-id");
+
+						if (!name || !idString) {
+							return new MiniflareResponse(
+								"[durable-object-proxy-err] Missing `x-miniflare-durable-object-name` or `x-miniflare-durable-object-id` headers.",
+								{ status: 400 }
+							);
+						}
+
+						const namespace = await mf?.getDurableObjectNamespace(name);
+						const id = namespace?.idFromString(idString);
+
+						if (!id) {
+							return new MiniflareResponse(
+								"[durable-object-proxy-err] Could not generate an ID. Possibly due to a mismatched DO name and ID?",
+								{ status: 500 }
+							);
+						}
+
+						const stub = namespace?.get(id);
+
+						if (!stub) {
+							return new MiniflareResponse(
+								"[durable-object-proxy-err] Could not generate a stub. Possibly due to a mismatched DO name and ID?",
+								{ status: 500 }
+							);
+						}
+
+						return stub.fetch(request);
+					},
+				},
+				modules: true,
+			});
+			const server = await durableObjectsMf.startServer();
+			durableObjectsMfPort = (server.address() as AddressInfo).port;
+		}
+
+		process.send &&
+			process.send(
+				JSON.stringify({
+					ready: true,
+					durableObjectsPort: durableObjectsMfPort,
+				})
+			);
 	} catch (e) {
 		mf?.log.error(e as Error);
 		process.exitCode = 1;
 		// Unmount any mounted workers
 		await mf?.dispose();
+		await durableObjectsMf?.dispose();
 	}
 }
 

package/src/pages/build.tsx

@@ -9,24 +9,15 @@ import { buildPlugin } from "./functions/buildPlugin";
 import { buildWorker } from "./functions/buildWorker";
 import { generateConfigFromFileTree } from "./functions/filepath-routing";
 import { writeRoutesModule } from "./functions/routes";
+import { convertRoutesToRoutesJSONSpec } from "./functions/routes-transformation";
 import { pagesBetaWarning, RUNNING_BUILDERS } from "./utils";
 import type { Config } from "./functions/routes";
-import type { ArgumentsCamelCase, Argv } from "yargs";
+import type { YargsOptionsToInterface } from "./types";
+import type { Argv } from "yargs";
 
-type PagesBuildArgs = {
-	directory: string;
-	outfile: string;
-	"output-config-path"?: string;
-	minify: boolean;
-	sourcemap: boolean;
-	"fallback-service": string;
-	watch: boolean;
-	plugin: boolean;
-	"build-output-directory"?: string;
-	"node-compat": boolean;
-};
+type PagesBuildArgs = YargsOptionsToInterface<typeof Options>;
 
-export function Options(yargs: Argv): Argv<PagesBuildArgs> {
+export function Options(yargs: Argv) {
 	return yargs
 		.positional("directory", {
 			type: "string",
@@ -43,6 +34,10 @@ export function Options(yargs: Argv): Argv<PagesBuildArgs> {
 				type: "string",
 				description: "The location for the output config file",
 			},
+			"output-routes-path": {
+				type: "string",
+				description: "The location for the output _routes.json file",
+			},
 			minify: {
 				type: "boolean",
 				default: false,
@@ -87,15 +82,16 @@ export function Options(yargs: Argv): Argv<PagesBuildArgs> {
 export const Handler = async ({
 	directory,
 	outfile,
-	"output-config-path": outputConfigPath,
+	outputConfigPath,
+	outputRoutesPath: routesOutputPath,
 	minify,
 	sourcemap,
 	fallbackService,
 	watch,
 	plugin,
-	"build-output-directory": buildOutputDirectory,
-	"node-compat": nodeCompat,
-}: ArgumentsCamelCase<PagesBuildArgs>) => {
+	buildOutputDirectory,
+	nodeCompat,
+}: PagesBuildArgs) => {
 	if (!isInPagesCI) {
 		// Beta message for `wrangler pages <commands>` usage
 		logger.log(pagesBetaWarning);
@@ -120,6 +116,7 @@ export const Handler = async ({
 		plugin,
 		buildOutputDirectory,
 		nodeCompat,
+		routesOutputPath,
 	});
 	await metrics.sendMetricsEvent("build pages functions");
 };
@@ -135,19 +132,25 @@ export async function buildFunctions({
 	onEnd,
 	plugin = false,
 	buildOutputDirectory,
+	routesOutputPath,
 	nodeCompat,
-}: {
-	outfile: string;
-	outputConfigPath?: string;
+}: Partial<
+	Pick<
+		PagesBuildArgs,
+		| "outputConfigPath"
+		| "minify"
+		| "sourcemap"
+		| "fallbackService"
+		| "watch"
+		| "plugin"
+		| "buildOutputDirectory"
+		| "nodeCompat"
+	>
+> & {
 	functionsDirectory: string;
-	minify?: boolean;
-	sourcemap?: boolean;
-	fallbackService?: string;
-	watch?: boolean;
 	onEnd?: () => void;
-	plugin?: boolean;
-	buildOutputDirectory?: string;
-	nodeCompat?: boolean;
+	outfile: Required<PagesBuildArgs>["outfile"];
+	routesOutputPath?: PagesBuildArgs["outputRoutesPath"];
 }) {
 	RUNNING_BUILDERS.forEach(
 		(runningBuilder) => runningBuilder.stop && runningBuilder.stop()
@@ -161,6 +164,11 @@ export async function buildFunctions({
 		baseURL,
 	});
 
+	if (config.routes && routesOutputPath) {
+		const routesJSON = convertRoutesToRoutesJSONSpec(config.routes);
+		writeFileSync(routesOutputPath, JSON.stringify(routesJSON, null, 2));
+	}
+
 	if (outputConfigPath) {
 		writeFileSync(
 			outputConfigPath,

package/src/pages/constants.ts

@@ -6,3 +6,6 @@ export const MAX_UPLOAD_ATTEMPTS = 5;
 export const MAX_CHECK_MISSING_ATTEMPTS = 5;
 export const SECONDS_TO_WAIT_FOR_PROXY = 5;
 export const isInPagesCI = !!process.env.CF_PAGES;
+/** The max number of rules in _routes.json */
+export const MAX_FUNCTIONS_ROUTES_RULES = 100;
+export const ROUTES_SPEC_VERSION = 1;

package/src/pages/deployments.tsx

@@ -11,14 +11,16 @@ import { requireAuth } from "../user";
 import { PAGES_CONFIG_CACHE_FILENAME } from "./constants";
 import { listProjects } from "./projects";
 import { pagesBetaWarning } from "./utils";
-import type { Deployment, PagesConfigCache } from "./types";
-import type { ArgumentsCamelCase, Argv } from "yargs";
+import type {
+	Deployment,
+	PagesConfigCache,
+	YargsOptionsToInterface,
+} from "./types";
+import type { Argv } from "yargs";
 
-type ListArgs = {
-	"project-name"?: string;
-};
+type ListArgs = YargsOptionsToInterface<typeof ListOptions>;
 
-export function ListOptions(yargs: Argv): Argv<ListArgs> {
+export function ListOptions(yargs: Argv) {
 	return yargs
 		.options({
 			"project-name": {
@@ -30,9 +32,7 @@ export function ListOptions(yargs: Argv): Argv<ListArgs> {
 		.epilogue(pagesBetaWarning);
 }
 
-export async function ListHandler({
-	projectName,
-}: ArgumentsCamelCase<ListArgs>) {
+export async function ListHandler({ projectName }: ListArgs) {
 	const config = getConfigCache<PagesConfigCache>(PAGES_CONFIG_CACHE_FILENAME);
 	const accountId = await requireAuth(config);
 

package/src/pages/dev.tsx

@@ -3,6 +3,7 @@ import { existsSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { watch } from "chokidar";
+import { build as workerJsBuild } from "esbuild";
 import { unstable_dev } from "../api";
 import { FatalError } from "../errors";
 import { logger } from "../logger";
@@ -10,25 +11,18 @@ import * as metrics from "../metrics";
 import { buildFunctions } from "./build";
 import { SECONDS_TO_WAIT_FOR_PROXY } from "./constants";
 import { CLEANUP, CLEANUP_CALLBACKS, pagesBetaWarning } from "./utils";
-import type { ArgumentsCamelCase, Argv } from "yargs";
-
-type PagesDevArgs = {
-	directory?: string;
-	command?: string;
-	local: boolean;
-	port: number;
-	proxy?: number;
-	"script-path": string;
-	binding?: (string | number)[];
-	kv?: (string | number)[];
-	do?: (string | number)[];
-	"live-reload": boolean;
-	"local-protocol"?: "https" | "http";
-	"experimental-enable-local-persistence": boolean;
-	"node-compat": boolean;
-};
+import type { AdditionalDevProps } from "../dev";
+import type { YargsOptionsToInterface } from "./types";
+import type { Plugin } from "esbuild";
+import type { Argv } from "yargs";
+
+const DURABLE_OBJECTS_BINDING_REGEXP = new RegExp(
+	/^(?<binding>[^=]+)=(?<className>[^@\s]+)(@(?<scriptName>.*)$)?$/
+);
 
-export function Options(yargs: Argv): Argv<PagesDevArgs> {
+type PagesDevArgs = YargsOptionsToInterface<typeof Options>;
+
+export function Options(yargs: Argv) {
 	return yargs
 		.positional("directory", {
 			type: "string",
@@ -46,11 +40,20 @@ export function Options(yargs: Argv): Argv<PagesDevArgs> {
 				default: true,
 				description: "Run on my machine",
 			},
+			ip: {
+				type: "string",
+				default: "0.0.0.0",
+				description: "The IP address to listen on",
+			},
 			port: {
 				type: "number",
 				default: 8788,
 				description: "The port to listen on (serve from)",
 			},
+			"inspector-port": {
+				type: "number",
+				describe: "Port for devtools to connect to",
+			},
 			proxy: {
 				type: "number",
 				description: "The port to proxy (where the static assets are served)",
@@ -68,14 +71,18 @@ export function Options(yargs: Argv): Argv<PagesDevArgs> {
 			},
 			kv: {
 				type: "array",
-				description: "KV namespace to bind",
+				description: "KV namespace to bind (--kv KV_BINDING)",
 				alias: "k",
 			},
 			do: {
 				type: "array",
-				description: "Durable Object to bind (NAME=CLASS)",
+				description: "Durable Object to bind (--do NAME=CLASS)",
 				alias: "o",
 			},
+			r2: {
+				type: "array",
+				description: "R2 bucket to bind (--r2 R2_BINDING)",
+			},
 			"live-reload": {
 				type: "boolean",
 				default: false,
@@ -108,19 +115,22 @@ export function Options(yargs: Argv): Argv<PagesDevArgs> {
 export const Handler = async ({
 	local,
 	directory,
+	ip,
 	port,
+	"inspector-port": inspectorPort,
 	proxy: requestedProxyPort,
 	"script-path": singleWorkerScriptPath,
 	binding: bindings = [],
 	kv: kvs = [],
 	do: durableObjects = [],
+	r2: r2s = [],
 	"live-reload": liveReload,
 	"local-protocol": localProtocol,
 	"experimental-enable-local-persistence": experimentalEnableLocalPersistence,
 	"node-compat": nodeCompat,
 	config: config,
 	_: [_pages, _dev, ...remaining],
-}: ArgumentsCamelCase<PagesDevArgs>) => {
+}: PagesDevArgs) => {
 	// Beta message for `wrangler pages <commands>` usage
 	logger.log(pagesBetaWarning);
 
@@ -213,6 +223,23 @@ export const Handler = async ({
 		if (!existsSync(scriptPath)) {
 			logger.log("No functions. Shimming...");
 			scriptPath = resolve(__dirname, "../templates/pages-shim.ts");
+		} else {
+			const runBuild = async () => {
+				try {
+					await workerJsBuild({
+						entryPoints: [scriptPath],
+						write: false,
+						plugins: [blockWorkerJsImports],
+					});
+				} catch {}
+			};
+			await runBuild();
+			watch([scriptPath], {
+				persistent: true,
+				ignoreInitial: true,
+			}).on("all", async () => {
+				await runBuild();
+			});
 		}
 	}
 
@@ -221,7 +248,9 @@ export const Handler = async ({
 	const { stop, waitUntilExit } = await unstable_dev(
 		scriptPath,
 		{
+			ip,
 			port,
+			inspectorPort,
 			watch: true,
 			localProtocol,
 			liveReload,
@@ -237,12 +266,29 @@ export const Handler = async ({
 				binding: val.toString(),
 				id: "",
 			})),
-			durableObjects: durableObjects.map((durableObject) => {
-				const [name, class_name] = durableObject.toString().split("=");
-				return {
-					name,
-					class_name,
-				};
+			durableObjects: durableObjects
+				.map((durableObject) => {
+					const { binding, className, scriptName } =
+						DURABLE_OBJECTS_BINDING_REGEXP.exec(durableObject.toString())
+							?.groups || {};
+
+					if (!binding || !className) {
+						logger.warn(
+							"Could not parse Durable Object binding:",
+							durableObject.toString()
+						);
+						return;
+					}
+
+					return {
+						name: binding,
+						class_name: className,
+						script_name: scriptName,
+					};
+				})
+				.filter(Boolean) as AdditionalDevProps["durableObjects"],
+			r2: r2s.map((binding) => {
+				return { binding: binding.toString(), bucket_name: "" };
 			}),
 
 			enablePagesAssetsServiceBinding: {
@@ -412,3 +458,15 @@ async function spawnProxyProcess({
 
 	return port;
 }
+
+const blockWorkerJsImports: Plugin = {
+	name: "block-worker-js-imports",
+	setup(build) {
+		build.onResolve({ filter: /.*/g }, (_args) => {
+			logger.error(
+				`_worker.js is importing from another file. This will throw an error if deployed.\nYou should bundle your Worker or remove the import if it is unused.`
+			);
+			return null;
+		});
+	},
+};

package/src/pages/functions/buildPlugin.ts

@@ -29,6 +29,10 @@ export function buildPlugin({
 		bundle: true,
 		format: "esm",
 		target: "esnext",
+		loader: {
+			".html": "text",
+			".txt": "text",
+		},
 		outfile,
 		minify,
 		sourcemap,

package/src/pages/functions/buildWorker.ts

@@ -34,6 +34,10 @@ export function buildWorker({
 		bundle: true,
 		format: "esm",
 		target: "esnext",
+		loader: {
+			".html": "text",
+			".txt": "text",
+		},
 		outfile,
 		minify,
 		sourcemap,

package/src/pages/functions/routes-consolidation.test.ts

@@ -0,0 +1,66 @@
+import { consolidateRoutes } from "./routes-consolidation";
+
+describe("route-consolidation", () => {
+	describe("consolidateRoutes()", () => {
+		it("should consolidate redundant routes", () => {
+			expect(consolidateRoutes(["/api/foo", "/api/*"])).toEqual(["/api/*"]);
+			expect(
+				consolidateRoutes([
+					"/api/foo",
+					"/api/foo/*",
+					"/api/bar/*",
+					"/api/*",
+					"/foo",
+					"/foo/bar",
+					"/bar/*",
+					"/bar/baz/*",
+					"/bar/baz/hello",
+				])
+			).toEqual(["/api/*", "/foo", "/foo/bar", "/bar/*"]);
+		});
+		it("should consolidate thousands of redundant routes", () => {
+			// Test to make sure the consolidator isn't horribly slow
+			const routes: string[] = [];
+			const limit = 1000;
+			for (let i = 0; i < limit; i++) {
+				// Add 3 routes per id
+				const id = `some-id-${i}`;
+				routes.push(`/${id}/*`, `/${id}/foo`, `/${id}/bar/*`);
+			}
+			const consolidated = consolidateRoutes(routes);
+			expect(consolidated.length).toEqual(limit);
+			// Should be all unique
+			expect(Array.from(new Set(consolidated)).length).toEqual(limit);
+			// Should all have pattern `/$id/*`
+			expect(
+				consolidated.every((route) => route.match(/\/[a-z0-9-]+\/\*/) !== null)
+			).toEqual(true);
+		});
+
+		it("should consolidate many redundant sub-routes", () => {
+			const routes: string[] = [];
+			const limit = 15;
+
+			// Create $limit of top-level catch-all routes, with a lot of sub-routes
+			for (let i = 0; i < limit; i++) {
+				routes.push(`/foo-${i}/*`);
+				for (let j = 0; j < limit; j++) {
+					routes.push(`/foo-${i}/bar-${j}/hello`);
+					for (let k = 0; k < limit; k++) {
+						routes.push(`/foo-${i}/bar-${j}/baz-${k}/*`);
+						routes.push(`/foo-${i}/bar-${j}/baz-${k}/profile`);
+					}
+				}
+			}
+
+			const consolidated = consolidateRoutes(routes);
+			expect(consolidated.length).toEqual(limit);
+			// Should be all unique
+			expect(Array.from(new Set(consolidated)).length).toEqual(limit);
+			// Should all have pattern `/$id/*`
+			expect(
+				consolidated.every((route) => route.match(/\/[a-z0-9-]+\/\*/) !== null)
+			).toEqual(true);
+		});
+	});
+});

package/src/pages/functions/routes-consolidation.ts

@@ -0,0 +1,29 @@
+/**
+ * consolidateRoutes consolidates redundant routes - eg. ["/api/*"", "/api/foo"] -> ["/api/*""]
+ * @param routes If this is the same order as Functions routes (with most-specific first),
+ *   it will be more efficient to reverse it first. Should be in the format: /api/foo, /api/*
+ * @returns Non-redundant list of routes
+ */
+export function consolidateRoutes(routes: string[]): string[] {
+	// create a map of the routes
+	const routesMap = new Map<string, boolean>();
+	for (const route of routes) {
+		routesMap.set(route, true);
+	}
+	// Find routes that might render other routes redundant
+	for (const route of routes.filter((r) => r.endsWith("/*"))) {
+		// Make sure the route still exists in the map
+		if (routesMap.has(route)) {
+			// Remove splat at the end, leaving the /
+			// eg. /api/* -> /api/
+			const routeTrimmed = route.substring(0, route.length - 1);
+			for (const nextRoute of routesMap.keys()) {
+				// Delete any route that has the wildcard route as a prefix
+				if (nextRoute !== route && nextRoute.startsWith(routeTrimmed)) {
+					routesMap.delete(nextRoute);
+				}
+			}
+		}
+	}
+	return Array.from(routesMap.keys());
+}