wrangler 2.0.25 → 2.0.26

package/src/pages/dev.tsx

@@ -1,6 +1,6 @@
 import { execSync, spawn } from "node:child_process";
 import { existsSync } from "node:fs";
-import { tmpdir } from "node:os";
+import { homedir, tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { watch } from "chokidar";
 import { build as workerJsBuild } from "esbuild";
@@ -10,6 +10,7 @@ import { logger } from "../logger";
 import * as metrics from "../metrics";
 import { buildFunctions } from "./build";
 import { SECONDS_TO_WAIT_FOR_PROXY } from "./constants";
+import { FunctionsNoRoutesError, getFunctionsNoRoutesWarning } from "./errors";
 import { CLEANUP, CLEANUP_CALLBACKS, pagesBetaWarning } from "./utils";
 import type { AdditionalDevProps } from "../dev";
 import type { YargsOptionsToInterface } from "./types";
@@ -40,6 +41,27 @@ export function Options(yargs: Argv) {
 				default: true,
 				description: "Run on my machine",
 			},
+			"compatibility-date": {
+				describe: "Date to use for compatibility checks",
+				type: "string",
+			},
+			"compatibility-flags": {
+				describe: "Flags to use for compatibility checks",
+				alias: "compatibility-flag",
+				type: "string",
+				array: true,
+			},
+
+			// TODO
+			// For now, all Pages projects are set to 2021-11-02. We're adding compat date soon, and we can then adopt `wrangler dev`'s `default: true`.
+			// However, it looks like it isn't actually connected up properly in `wrangler dev` at the moment, hence commenting this out for now.
+
+			// latest: {
+			// 	describe: "Use the latest version of the worker runtime",
+			// 	type: "boolean",
+			// 	default: false,
+			// },
+
 			ip: {
 				type: "string",
 				default: "0.0.0.0",
@@ -115,6 +137,8 @@ export function Options(yargs: Argv) {
 export const Handler = async ({
 	local,
 	directory,
+	"compatibility-date": compatibilityDate = "2021-11-02",
+	"compatibility-flags": compatibilityFlags,
 	ip,
 	port,
 	"inspector-port": inspectorPort,
@@ -143,7 +167,7 @@ export const Handler = async ({
 	}
 
 	const functionsDirectory = "./functions";
-	const usingFunctions = existsSync(functionsDirectory);
+	let usingFunctions = existsSync(functionsDirectory);
 
 	const command = remaining;
 
@@ -169,8 +193,9 @@ export const Handler = async ({
 		(promiseResolve) => (scriptReadyResolve = promiseResolve)
 	);
 
-	let scriptPath: string;
+	let scriptPath = "";
 
+	// Try to use Functions
 	if (usingFunctions) {
 		const outfile = join(tmpdir(), `./functionsWorker-${Math.random()}.js`);
 		scriptPath = outfile;
@@ -182,36 +207,63 @@ export const Handler = async ({
 		}
 
 		logger.log(`Compiling worker to "${outfile}"...`);
-
+		const onEnd = () => scriptReadyResolve();
 		try {
 			await buildFunctions({
 				outfile,
 				functionsDirectory,
 				sourcemap: true,
 				watch: true,
-				onEnd: () => scriptReadyResolve(),
+				onEnd,
 				buildOutputDirectory: directory,
 				nodeCompat,
 			});
 			await metrics.sendMetricsEvent("build pages functions");
-		} catch {}
 
-		watch([functionsDirectory], {
-			persistent: true,
-			ignoreInitial: true,
-		}).on("all", async () => {
-			await buildFunctions({
-				outfile,
-				functionsDirectory,
-				sourcemap: true,
-				watch: true,
-				onEnd: () => scriptReadyResolve(),
-				buildOutputDirectory: directory,
-				nodeCompat,
+			// If Functions found routes, continue using Functions
+			watch([functionsDirectory], {
+				persistent: true,
+				ignoreInitial: true,
+			}).on("all", async () => {
+				try {
+					await buildFunctions({
+						outfile,
+						functionsDirectory,
+						sourcemap: true,
+						watch: true,
+						onEnd,
+						buildOutputDirectory: directory,
+						nodeCompat,
+					});
+					await metrics.sendMetricsEvent("build pages functions");
+				} catch (e) {
+					if (e instanceof FunctionsNoRoutesError) {
+						logger.warn(
+							getFunctionsNoRoutesWarning(functionsDirectory, "skipping")
+						);
+					} else {
+						throw e;
+					}
+				}
 			});
-			await metrics.sendMetricsEvent("build pages functions");
-		});
-	} else {
+		} catch (e) {
+			// If there are no Functions, then Pages will only serve assets.
+			if (e instanceof FunctionsNoRoutesError) {
+				logger.warn(
+					getFunctionsNoRoutesWarning(functionsDirectory, "skipping")
+				);
+				// Resolve anyway and run without Functions
+				onEnd();
+				// Turn off Functions
+				usingFunctions = false;
+			} else {
+				throw e;
+			}
+		}
+	}
+	// Depending on the result of building Functions, we may not actually be using
+	// Functions even if the directory exists.
+	if (!usingFunctions) {
 		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 		scriptReadyResolve!();
 
@@ -245,6 +297,15 @@ export const Handler = async ({
 
 	await scriptReadyPromise;
 
+	if (scriptPath === "") {
+		// Failed to get a script with or without Functions,
+		// something really bad must have happend.
+		throw new FatalError(
+			"Failed to start wrangler pages dev due to an unknown error",
+			1
+		);
+	}
+
 	const { stop, waitUntilExit } = await unstable_dev(
 		scriptPath,
 		{
@@ -254,8 +315,8 @@ export const Handler = async ({
 			watch: true,
 			localProtocol,
 			liveReload,
-
-			compatibilityDate: "2021-11-02",
+			compatibilityDate,
+			compatibilityFlags,
 			nodeCompat,
 			vars: Object.fromEntries(
 				bindings
@@ -365,7 +426,8 @@ function getPort(pid: number) {
 	let command: string, regExp: RegExp;
 
 	if (isWindows()) {
-		command = "\\windows\\system32\\netstat.exe -nao";
+		const drive = homedir().split(":\\")[0];
+		command = drive + ":\\windows\\system32\\netstat.exe -nao";
 		regExp = new RegExp(`TCP\\s+.*:(\\d+)\\s+.*:\\d+\\s+LISTENING\\s+${pid}`);
 	} else {
 		command = "lsof -nPi";

package/src/pages/errors.ts

@@ -0,0 +1,22 @@
+/**
+ * Pages error when no routes are found in the functions directory
+ */
+export class FunctionsNoRoutesError extends Error {
+	constructor(message: string) {
+		super(message);
+	}
+}
+/**
+ * Exit code for `pages functions build` when no routes are found.
+ */
+export const EXIT_CODE_FUNCTIONS_NO_ROUTES_ERROR = 156;
+
+/** Warning message for when buildFunctions throws FunctionsNoRoutesError */
+export function getFunctionsNoRoutesWarning(
+	functionsDirectory: string,
+	suffix?: string
+) {
+	return `No routes found when building Functions directory: ${functionsDirectory}${
+		suffix ? " - " + suffix : ""
+	}`;
+}

package/src/pages/functions/routes-consolidation.test.ts

@@ -1,6 +1,7 @@
-import { consolidateRoutes } from "./routes-consolidation";
+import { consolidateRoutes, shortenRoute } from "./routes-consolidation";
 
 describe("route-consolidation", () => {
+	const maxRuleLength = 100; // from constants.MAX_FUNCTIONS_ROUTES_RULE_LENGTH
 	describe("consolidateRoutes()", () => {
 		it("should consolidate redundant routes", () => {
 			expect(consolidateRoutes(["/api/foo", "/api/*"])).toEqual(["/api/*"]);
@@ -62,5 +63,188 @@ describe("route-consolidation", () => {
 				consolidated.every((route) => route.match(/\/[a-z0-9-]+\/\*/) !== null)
 			).toEqual(true);
 		});
+
+		it("should truncate long single-level path into catch-all path, removing other paths", () => {
+			expect(
+				consolidateRoutes([
+					// [/aaaaaaa, /foo] -> [/*]
+					"/" + "a".repeat(maxRuleLength * 2),
+					"/foo",
+					"/bar/*",
+					"/baz/bagel/coffee",
+				])
+			).toEqual(["/*"]);
+		});
+
+		it("should truncate long nested path, removing other paths", () => {
+			expect(
+				consolidateRoutes([
+					// [/aaaaaaa, /foo] -> [/*]
+					"/foo/" + "a".repeat(maxRuleLength * 2),
+					"/foo/bar",
+				])
+			).toEqual(["/foo/*"]);
+		});
+	});
+
+	describe(`shortenRoute()`, () => {
+		it("should allow max length path", () => {
+			const route = "/" + "a".repeat(maxRuleLength - 1);
+			// Make sure we don't have an off-by-one error, that'd be embarrassing...
+			expect(route.length).toEqual(maxRuleLength);
+			expect(
+				// Should stay the same
+				shortenRoute(route)
+			).toEqual(route);
+		});
+
+		it("should allow max length path (with slash)", () => {
+			const route = "/" + "a".repeat(maxRuleLength - 2) + "/";
+			expect(route.length).toEqual(maxRuleLength);
+			expect(
+				// Should stay the same
+				shortenRoute(route)
+			).toEqual(route);
+		});
+
+		it("should allow max length wildcard path", () => {
+			const route = "/" + "a".repeat(maxRuleLength - 3) + "/*";
+			expect(route.length).toEqual(maxRuleLength);
+			expect(
+				// Should stay the same
+				shortenRoute(route)
+			).toEqual(route);
+		});
+
+		it("should truncate long specific path to shorter wildcard path", () => {
+			const short = shortenRoute(
+				// /aaa/bbb -> /aaa/*
+				"/" +
+					"a".repeat(maxRuleLength * 0.6) +
+					"/" +
+					"b".repeat(maxRuleLength * 0.6)
+			);
+			expect(short).toEqual("/" + "a".repeat(maxRuleLength * 0.6) + "/*");
+			expect(short.length).toBeLessThanOrEqual(maxRuleLength);
+		});
+
+		it("should truncate long specific path (with slash) to shorter wildcard path", () => {
+			const short = shortenRoute(
+				// /aaa/bbb/ -> /aaa/*
+				"/" +
+					"a".repeat(maxRuleLength * 0.6) +
+					"/" +
+					"b".repeat(maxRuleLength * 0.6) +
+					"/"
+			);
+			expect(short).toEqual("/" + "a".repeat(maxRuleLength * 0.6) + "/*");
+			expect(short.length).toBeLessThanOrEqual(maxRuleLength);
+		});
+
+		it("should truncate long wildcard path to shorter wildcard path", () => {
+			const short = shortenRoute(
+				// /aaa/bbb/* -> /aaa/*
+				"/" +
+					"a".repeat(maxRuleLength * 0.6) +
+					"/" +
+					"b".repeat(maxRuleLength * 0.6) +
+					"/*"
+			);
+			expect(short).toEqual("/" + "a".repeat(maxRuleLength * 0.6) + "/*");
+			expect(short.length).toBeLessThanOrEqual(maxRuleLength);
+		});
+
+		it("should truncate long single-level specific path to catch-all path", () => {
+			expect(
+				shortenRoute(
+					// /aaa -> /*
+					"/" + "a".repeat(maxRuleLength * 2)
+				)
+			).toEqual("/*");
+		});
+
+		it("should truncate long single-level specific path (with slash) to catch-all path", () => {
+			expect(
+				shortenRoute(
+					// /aaa/ -> /*
+					"/" + "a".repeat(maxRuleLength * 2) + "/"
+				)
+			).toEqual("/*");
+		});
+
+		it("should truncate long single-level wildcard path to catch-all path", () => {
+			expect(
+				shortenRoute(
+					// /aaa/* -> /*
+					"/" + "a".repeat(maxRuleLength * 2) + "/*"
+				)
+			).toEqual("/*");
+		});
+
+		it("should truncate many single-character segements", () => {
+			const short = shortenRoute(
+				// /a/a/a -> /a/a/*
+				"/a".repeat(maxRuleLength) // 2x limit
+			);
+			expect(short).toEqual("/a".repeat(maxRuleLength / 2 - 1) + "/*");
+			// Should be the exact max length
+			expect(short.length).toEqual(maxRuleLength);
+		});
+
+		it("should truncate many double-character segements", () => {
+			// === odd ===
+			const short = shortenRoute(
+				// /aa/aa/aa -> /aa/aa/*
+				"/aa".repeat(maxRuleLength) // 3x limit
+			);
+			expect(short).toEqual("/aa".repeat(maxRuleLength / 3 - 1) + "/*");
+			// Should be the exact max length
+			expect(short.length).toEqual(maxRuleLength - 2); // -2 because of the odd number
+		});
+
+		it("should truncate many single-character segements with wildcard", () => {
+			const short = shortenRoute(
+				// /a/a/a -> /a/a/*
+				"/a".repeat(maxRuleLength) + "/*" // 2x limit
+			);
+			expect(short).toEqual("/a".repeat(maxRuleLength / 2 - 1) + "/*");
+			// Should be the exact max length
+			expect(short.length).toEqual(maxRuleLength);
+		});
+
+		it("should truncate many double-character segements with wildcard", () => {
+			const short = shortenRoute(
+				// /aa/aa/aa -> /aa/*
+				"/aa".repeat(maxRuleLength) + "/*" // 2x limit
+			);
+			expect(short).toEqual("/aa".repeat(maxRuleLength / 3 - 1) + "/*");
+			// Should be the exact max length
+			expect(short.length).toEqual(maxRuleLength - 2); // -2 because of the odd number
+		});
+
+		// This is probably the best test here - tests variable-length segments, up until the max.
+		// This ensures that it's always able to shorten rules, without failing and returning "/*"
+		// The other tests are great for ensuring exact sequences instead of only asserting length, though.
+		for (const suffix of ["", "/", "/*"]) {
+			// Test each type of path: /a, /a/a, /a/*
+			it(`should truncate many variable-character segements (suffix="${suffix}") without truncating to /*`, () => {
+				// "/" + 97 chars + "/*" === 100
+				for (let i = 1; i < maxRuleLength - 2; i++) {
+					const segment = "/" + "a".repeat(i);
+					// make sure the segment isn't too long since we are testing not resulting to /*
+					expect(segment.length).toBeLessThanOrEqual(maxRuleLength);
+					const route =
+						segment.repeat((maxRuleLength / segment.length) * 2) + suffix;
+					// Make sure we made the rule too long
+					expect(route.length).toBeGreaterThan(maxRuleLength);
+					const short = shortenRoute(route);
+
+					// Make sure it's not over the limit
+					expect(short.length).toBeLessThanOrEqual(maxRuleLength);
+					// It should never have to fall back to /*
+					expect(short).not.toEqual("/*");
+				}
+			});
+		}
 	});
 });

package/src/pages/functions/routes-consolidation.ts

@@ -1,3 +1,5 @@
+import { MAX_FUNCTIONS_ROUTES_RULE_LENGTH } from "../constants";
+
 /**
  * consolidateRoutes consolidates redundant routes - eg. ["/api/*"", "/api/foo"] -> ["/api/*""]
  * @param routes If this is the same order as Functions routes (with most-specific first),
@@ -5,13 +7,18 @@
  * @returns Non-redundant list of routes
  */
 export function consolidateRoutes(routes: string[]): string[] {
+	// First we need to trim any rules that are too long and deduplicate the result
+	const routesShortened = Array.from(
+		new Set(routes.map((route) => shortenRoute(route)))
+	);
+
 	// create a map of the routes
 	const routesMap = new Map<string, boolean>();
-	for (const route of routes) {
+	for (const route of routesShortened) {
 		routesMap.set(route, true);
 	}
 	// Find routes that might render other routes redundant
-	for (const route of routes.filter((r) => r.endsWith("/*"))) {
+	for (const route of routesShortened.filter((r) => r.endsWith("/*"))) {
 		// Make sure the route still exists in the map
 		if (routesMap.has(route)) {
 			// Remove splat at the end, leaving the /
@@ -27,3 +34,40 @@ export function consolidateRoutes(routes: string[]): string[] {
 	}
 	return Array.from(routesMap.keys());
 }
+
+/**
+ * Shortens a route until it's within the rule length limit defined in
+ * constants.MAX_FUNCTIONS_ROUTES_RULE_LENGTH
+ * Eg. /aaa/bbb -> /aaa/*
+ * @param routeToShorten Route to shorten if needed
+ * @param maxLength Max length of route to try to shorten to
+ */
+export function shortenRoute(
+	routeToShorten: string,
+	maxLength: number = MAX_FUNCTIONS_ROUTES_RULE_LENGTH
+): string {
+	if (routeToShorten.length <= maxLength) {
+		return routeToShorten;
+	}
+
+	let route = routeToShorten;
+	// May have to try multiple times for longer segments
+	for (let i = 0; i < routeToShorten.length; i++) {
+		// Shorten to the first slash within the limit
+		for (let j = maxLength - 1 - i; j > 0; j--) {
+			if (route[j] === "/") {
+				route = route.slice(0, j) + "/*";
+				break;
+			}
+		}
+		if (route.length <= maxLength) {
+			break;
+		}
+	}
+
+	// If we failed to shorten it, fall back to include-all rather than breaking
+	if (route.length > maxLength) {
+		route = "/*";
+	}
+	return route;
+}

package/src/pages/functions/routes-transformation.ts

@@ -13,9 +13,6 @@ interface RoutesJSONSpec {
 
 type RoutesJSONRouteInput = Pick<RouteConfig, "routePath" | "middleware">[];
 
-/**
- * TODO can we do better naming?
- */
 export function convertRoutesToGlobPatterns(
 	routes: RoutesJSONRouteInput
 ): string[] {

package/src/pages/functions.tsx

@@ -0,0 +1,96 @@
+import { existsSync, lstatSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { FatalError } from "../errors";
+import { logger } from "../logger";
+import { isInPagesCI, ROUTES_SPEC_VERSION } from "./constants";
+import {
+	isRoutesJSONSpec,
+	optimizeRoutesJSONSpec,
+} from "./functions/routes-transformation";
+import { pagesBetaWarning } from "./utils";
+import type { YargsOptionsToInterface } from "./types";
+import type { Argv } from "yargs";
+
+type OptimizeRoutesArgs = YargsOptionsToInterface<typeof OptimizeRoutesOptions>;
+
+export function OptimizeRoutesOptions(yargs: Argv) {
+	return yargs
+		.options({
+			"routes-path": {
+				type: "string",
+				demandOption: true,
+				description: "The location of the _routes.json file",
+			},
+		})
+		.options({
+			"output-routes-path": {
+				type: "string",
+				demandOption: true,
+				description: "The location of the optimized output routes file",
+			},
+		});
+}
+
+export async function OptimizeRoutesHandler({
+	routesPath,
+	outputRoutesPath,
+}: OptimizeRoutesArgs) {
+	if (!isInPagesCI) {
+		// Beta message for `wrangler pages <commands>` usage
+		logger.log(pagesBetaWarning);
+	}
+
+	let routesFileContents: string;
+	const routesOutputDirectory = path.dirname(outputRoutesPath);
+
+	if (!existsSync(routesPath)) {
+		throw new FatalError(
+			`Oops! File ${routesPath} does not exist. Please make sure --routes-path is a valid file path (for example "/public/_routes.json").`,
+			1
+		);
+	}
+
+	if (
+		!existsSync(routesOutputDirectory) ||
+		!lstatSync(routesOutputDirectory).isDirectory()
+	) {
+		throw new FatalError(
+			`Oops! Folder ${routesOutputDirectory} does not exist. Please make sure --output-routes-path is a valid file path (for example "/public/_routes.json").`,
+			1
+		);
+	}
+
+	try {
+		routesFileContents = readFileSync(routesPath, "utf-8");
+	} catch (err) {
+		throw new FatalError(`Error while reading ${routesPath} file: ${err}`);
+	}
+
+	const routes = JSON.parse(routesFileContents);
+
+	if (!isRoutesJSONSpec(routes)) {
+		throw new FatalError(
+			`
+      Invalid _routes.json file found at: ${routesPath}. Please make sure the JSON object has the following format:
+      {
+        version: ${ROUTES_SPEC_VERSION};
+        include: string[];
+        exclude: string[];
+      }
+      `,
+			1
+		);
+	}
+
+	const optimizedRoutes = optimizeRoutesJSONSpec(routes);
+	const optimizedRoutesContents = JSON.stringify(optimizedRoutes);
+
+	try {
+		writeFileSync(outputRoutesPath, optimizedRoutesContents);
+	} catch (err) {
+		throw new FatalError(
+			`Error writing to ${outputRoutesPath} file: ${err}`,
+			1
+		);
+	}
+}