wrangler 0.0.0-e5a3814 → 0.0.0-e60dc909

package/src/pages/functions/filepath-routing.ts

@@ -0,0 +1,189 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { build } from "esbuild";
+import { toUrlPath } from "../../paths";
+import type { UrlPath } from "../../paths";
+import type { HTTPMethod, RouteConfig } from "./routes";
+
+export async function generateConfigFromFileTree({
+	baseDir,
+	baseURL,
+}: {
+	baseDir: string;
+	baseURL: UrlPath;
+}) {
+	let routeEntries: RouteConfig[] = [];
+
+	if (!baseURL.startsWith("/")) {
+		baseURL = `/${baseURL}` as UrlPath;
+	}
+
+	if (baseURL.endsWith("/")) {
+		baseURL = baseURL.slice(0, -1) as UrlPath;
+	}
+
+	await forEachFile(baseDir, async (filepath) => {
+		const ext = path.extname(filepath);
+		if (/^\.(mjs|js|ts|tsx|jsx)$/.test(ext)) {
+			const { metafile } = await build({
+				metafile: true,
+				write: false,
+				bundle: false,
+				entryPoints: [path.resolve(filepath)],
+			});
+			const exportNames: string[] = [];
+			if (metafile) {
+				for (const output in metafile?.outputs) {
+					exportNames.push(...metafile.outputs[output].exports);
+				}
+			}
+			for (const exportName of exportNames) {
+				const [match, method = ""] = (exportName.match(
+					/^onRequest(Get|Post|Put|Patch|Delete|Options|Head)?$/
+				) ?? []) as (string | undefined)[];
+
+				if (match) {
+					const basename = path.basename(filepath).slice(0, -ext.length);
+
+					const isIndexFile = basename === "index";
+					// TODO: deprecate _middleware_ in favor of _middleware
+					const isMiddlewareFile =
+						basename === "_middleware" || basename === "_middleware_";
+
+					let routePath = path
+						.relative(baseDir, filepath)
+						.slice(0, -ext.length);
+					let mountPath = path.dirname(routePath);
+
+					if (isIndexFile || isMiddlewareFile) {
+						routePath = path.dirname(routePath);
+					}
+
+					if (routePath === ".") {
+						routePath = "";
+					}
+					if (mountPath === ".") {
+						mountPath = "";
+					}
+
+					routePath = `${baseURL}/${routePath}`;
+					mountPath = `${baseURL}/${mountPath}`;
+
+					routePath = routePath.replace(/\[\[([^\]]+)\]\]/g, ":$1*"); // transform [[id]] => :id*
+					routePath = routePath.replaceAll(/\[([^\]]+)\]/g, ":$1"); // transform [id] => :id
+					mountPath = mountPath.replace(/\[\[([^\]]+)\]\]/g, ":$1*"); // transform [[id]] => :id*
+					mountPath = mountPath.replaceAll(/\[([^\]]+)\]/g, ":$1"); // transform [id] => :id
+
+					// These are used as module specifiers so UrlPaths are okay to use even on Windows
+					const modulePath = toUrlPath(path.relative(baseDir, filepath));
+
+					const routeEntry: RouteConfig = {
+						routePath: toUrlPath(routePath),
+						mountPath: toUrlPath(mountPath),
+						method: method.toUpperCase() as HTTPMethod,
+						[isMiddlewareFile ? "middleware" : "module"]: [
+							`${modulePath}:${exportName}`,
+						],
+					};
+
+					routeEntries.push(routeEntry);
+				}
+			}
+		}
+	});
+	// Combine together any routes (index routes) which contain both a module and a middleware
+	routeEntries = routeEntries.reduce(
+		(acc: typeof routeEntries, { routePath, ...rest }) => {
+			const existingRouteEntry = acc.find(
+				(routeEntry) =>
+					routeEntry.routePath === routePath &&
+					routeEntry.method === rest.method
+			);
+			if (existingRouteEntry !== undefined) {
+				Object.assign(existingRouteEntry, rest);
+			} else {
+				acc.push({ routePath, ...rest });
+			}
+			return acc;
+		},
+		[]
+	);
+
+	routeEntries.sort((a, b) => compareRoutes(a, b));
+
+	return {
+		routes: routeEntries,
+	};
+}
+
+// Ensure routes are produced in order of precedence so that
+// more specific routes aren't occluded from matching due to
+// less specific routes appearing first in the route list.
+export function compareRoutes(
+	{ routePath: routePathA, method: methodA }: RouteConfig,
+	{ routePath: routePathB, method: methodB }: RouteConfig
+) {
+	function parseRoutePath(routePath: UrlPath): string[] {
+		return routePath.slice(1).split("/").filter(Boolean);
+	}
+
+	const segmentsA = parseRoutePath(routePathA);
+	const segmentsB = parseRoutePath(routePathB);
+
+	// sort routes with fewer segments after those with more segments
+	if (segmentsA.length !== segmentsB.length) {
+		return segmentsB.length - segmentsA.length;
+	}
+
+	for (let i = 0; i < segmentsA.length; i++) {
+		const isWildcardA = segmentsA[i].includes("*");
+		const isWildcardB = segmentsB[i].includes("*");
+		const isParamA = segmentsA[i].includes(":");
+		const isParamB = segmentsB[i].includes(":");
+
+		// sort wildcard segments after non-wildcard segments
+		if (isWildcardA && !isWildcardB) return 1;
+		if (!isWildcardA && isWildcardB) return -1;
+
+		// sort dynamic param segments after non-param segments
+		if (isParamA && !isParamB) return 1;
+		if (!isParamA && isParamB) return -1;
+	}
+
+	// sort routes that specify an HTTP before those that don't
+	if (methodA && !methodB) return -1;
+	if (!methodA && methodB) return 1;
+
+	// all else equal, just sort the paths lexicographically
+	return routePathA.localeCompare(routePathB);
+}
+
+async function forEachFile<T>(
+	baseDir: string,
+	fn: (filepath: string) => T | Promise<T>
+) {
+	const searchPaths = [baseDir];
+	const returnValues: T[] = [];
+
+	while (isNotEmpty(searchPaths)) {
+		const cwd = searchPaths.shift();
+		const dir = await fs.readdir(cwd, { withFileTypes: true });
+		for (const entry of dir) {
+			const pathname = path.join(cwd, entry.name);
+			if (entry.isDirectory()) {
+				searchPaths.push(pathname);
+			} else if (entry.isFile()) {
+				returnValues.push(await fn(pathname));
+			}
+		}
+	}
+
+	return returnValues;
+}
+
+interface NonEmptyArray<T> extends Array<T> {
+	shift(): T;
+}
+function isNotEmpty<T>(array: T[]): array is NonEmptyArray<T> {
+	return array.length > 0;
+}

package/src/pages/functions/identifiers.ts

@@ -0,0 +1,78 @@
+export const RESERVED_KEYWORDS = [
+	"do",
+	"if",
+	"in",
+	"for",
+	"let",
+	"new",
+	"try",
+	"var",
+	"case",
+	"else",
+	"enum",
+	"eval",
+	"null",
+	"this",
+	"true",
+	"void",
+	"with",
+	"await",
+	"break",
+	"catch",
+	"class",
+	"const",
+	"false",
+	"super",
+	"throw",
+	"while",
+	"yield",
+	"delete",
+	"export",
+	"import",
+	"public",
+	"return",
+	"static",
+	"switch",
+	"typeof",
+	"default",
+	"extends",
+	"finally",
+	"package",
+	"private",
+	"continue",
+	"debugger",
+	"function",
+	"arguments",
+	"interface",
+	"protected",
+	"implements",
+	"instanceof",
+	"NaN",
+	"Infinity",
+	"undefined",
+];
+
+export const reservedKeywordRegex = new RegExp(
+	`^${RESERVED_KEYWORDS.join("|")}$`
+);
+
+export const identifierStartRegex = /[$_\p{ID_Start}]/u;
+
+export const identifierPartRegex = /[$_\u200C\u200D\p{ID_Continue}]/u;
+
+export const identifierNameRegex =
+	/^(?:[$_\p{ID_Start}])(?:[$_\u200C\u200D\p{ID_Continue}])*$/u;
+
+export const validIdentifierRegex = new RegExp(
+	`(?!(${reservedKeywordRegex.source})$)${identifierNameRegex.source}`,
+	"u"
+);
+
+export const isValidIdentifier = (identifier: string) =>
+	validIdentifierRegex.test(identifier);
+
+export const normalizeIdentifier = (identifier: string) =>
+	identifier.replace(
+		/(?:^[^$_\p{ID_Start}])|[^$_\u200C\u200D\p{ID_Continue}]/gu,
+		"_"
+	);

package/src/pages/functions/routes-consolidation.test.ts

@@ -0,0 +1,250 @@
+import { consolidateRoutes, shortenRoute } from "./routes-consolidation";
+
+describe("route-consolidation", () => {
+	const maxRuleLength = 100; // from constants.MAX_FUNCTIONS_ROUTES_RULE_LENGTH
+	describe("consolidateRoutes()", () => {
+		it("should consolidate redundant routes", () => {
+			expect(consolidateRoutes(["/api/foo", "/api/*"])).toEqual(["/api/*"]);
+			expect(
+				consolidateRoutes([
+					"/api/foo",
+					"/api/foo/*",
+					"/api/bar/*",
+					"/api/*",
+					"/foo",
+					"/foo/bar",
+					"/bar/*",
+					"/bar/baz/*",
+					"/bar/baz/hello",
+				])
+			).toEqual(["/api/*", "/foo", "/foo/bar", "/bar/*"]);
+		});
+		it("should consolidate thousands of redundant routes", () => {
+			// Test to make sure the consolidator isn't horribly slow
+			const routes: string[] = [];
+			const limit = 1000;
+			for (let i = 0; i < limit; i++) {
+				// Add 3 routes per id
+				const id = `some-id-${i}`;
+				routes.push(`/${id}/*`, `/${id}/foo`, `/${id}/bar/*`);
+			}
+			const consolidated = consolidateRoutes(routes);
+			expect(consolidated.length).toEqual(limit);
+			// Should be all unique
+			expect(Array.from(new Set(consolidated)).length).toEqual(limit);
+			// Should all have pattern `/$id/*`
+			expect(
+				consolidated.every((route) => route.match(/\/[a-z0-9-]+\/\*/) !== null)
+			).toEqual(true);
+		});
+
+		it("should consolidate many redundant sub-routes", () => {
+			const routes: string[] = [];
+			const limit = 15;
+
+			// Create $limit of top-level catch-all routes, with a lot of sub-routes
+			for (let i = 0; i < limit; i++) {
+				routes.push(`/foo-${i}/*`);
+				for (let j = 0; j < limit; j++) {
+					routes.push(`/foo-${i}/bar-${j}/hello`);
+					for (let k = 0; k < limit; k++) {
+						routes.push(`/foo-${i}/bar-${j}/baz-${k}/*`);
+						routes.push(`/foo-${i}/bar-${j}/baz-${k}/profile`);
+					}
+				}
+			}
+
+			const consolidated = consolidateRoutes(routes);
+			expect(consolidated.length).toEqual(limit);
+			// Should be all unique
+			expect(Array.from(new Set(consolidated)).length).toEqual(limit);
+			// Should all have pattern `/$id/*`
+			expect(
+				consolidated.every((route) => route.match(/\/[a-z0-9-]+\/\*/) !== null)
+			).toEqual(true);
+		});
+
+		it("should truncate long single-level path into catch-all path, removing other paths", () => {
+			expect(
+				consolidateRoutes([
+					// [/aaaaaaa, /foo] -> [/*]
+					"/" + "a".repeat(maxRuleLength * 2),
+					"/foo",
+					"/bar/*",
+					"/baz/bagel/coffee",
+				])
+			).toEqual(["/*"]);
+		});
+
+		it("should truncate long nested path, removing other paths", () => {
+			expect(
+				consolidateRoutes([
+					// [/aaaaaaa, /foo] -> [/*]
+					"/foo/" + "a".repeat(maxRuleLength * 2),
+					"/foo/bar",
+				])
+			).toEqual(["/foo/*"]);
+		});
+	});
+
+	describe(`shortenRoute()`, () => {
+		it("should allow max length path", () => {
+			const route = "/" + "a".repeat(maxRuleLength - 1);
+			// Make sure we don't have an off-by-one error, that'd be embarrassing...
+			expect(route.length).toEqual(maxRuleLength);
+			expect(
+				// Should stay the same
+				shortenRoute(route)
+			).toEqual(route);
+		});
+
+		it("should allow max length path (with slash)", () => {
+			const route = "/" + "a".repeat(maxRuleLength - 2) + "/";
+			expect(route.length).toEqual(maxRuleLength);
+			expect(
+				// Should stay the same
+				shortenRoute(route)
+			).toEqual(route);
+		});
+
+		it("should allow max length wildcard path", () => {
+			const route = "/" + "a".repeat(maxRuleLength - 3) + "/*";
+			expect(route.length).toEqual(maxRuleLength);
+			expect(
+				// Should stay the same
+				shortenRoute(route)
+			).toEqual(route);
+		});
+
+		it("should truncate long specific path to shorter wildcard path", () => {
+			const short = shortenRoute(
+				// /aaa/bbb -> /aaa/*
+				"/" +
+					"a".repeat(maxRuleLength * 0.6) +
+					"/" +
+					"b".repeat(maxRuleLength * 0.6)
+			);
+			expect(short).toEqual("/" + "a".repeat(maxRuleLength * 0.6) + "/*");
+			expect(short.length).toBeLessThanOrEqual(maxRuleLength);
+		});
+
+		it("should truncate long specific path (with slash) to shorter wildcard path", () => {
+			const short = shortenRoute(
+				// /aaa/bbb/ -> /aaa/*
+				"/" +
+					"a".repeat(maxRuleLength * 0.6) +
+					"/" +
+					"b".repeat(maxRuleLength * 0.6) +
+					"/"
+			);
+			expect(short).toEqual("/" + "a".repeat(maxRuleLength * 0.6) + "/*");
+			expect(short.length).toBeLessThanOrEqual(maxRuleLength);
+		});
+
+		it("should truncate long wildcard path to shorter wildcard path", () => {
+			const short = shortenRoute(
+				// /aaa/bbb/* -> /aaa/*
+				"/" +
+					"a".repeat(maxRuleLength * 0.6) +
+					"/" +
+					"b".repeat(maxRuleLength * 0.6) +
+					"/*"
+			);
+			expect(short).toEqual("/" + "a".repeat(maxRuleLength * 0.6) + "/*");
+			expect(short.length).toBeLessThanOrEqual(maxRuleLength);
+		});
+
+		it("should truncate long single-level specific path to catch-all path", () => {
+			expect(
+				shortenRoute(
+					// /aaa -> /*
+					"/" + "a".repeat(maxRuleLength * 2)
+				)
+			).toEqual("/*");
+		});
+
+		it("should truncate long single-level specific path (with slash) to catch-all path", () => {
+			expect(
+				shortenRoute(
+					// /aaa/ -> /*
+					"/" + "a".repeat(maxRuleLength * 2) + "/"
+				)
+			).toEqual("/*");
+		});
+
+		it("should truncate long single-level wildcard path to catch-all path", () => {
+			expect(
+				shortenRoute(
+					// /aaa/* -> /*
+					"/" + "a".repeat(maxRuleLength * 2) + "/*"
+				)
+			).toEqual("/*");
+		});
+
+		it("should truncate many single-character segements", () => {
+			const short = shortenRoute(
+				// /a/a/a -> /a/a/*
+				"/a".repeat(maxRuleLength) // 2x limit
+			);
+			expect(short).toEqual("/a".repeat(maxRuleLength / 2 - 1) + "/*");
+			// Should be the exact max length
+			expect(short.length).toEqual(maxRuleLength);
+		});
+
+		it("should truncate many double-character segements", () => {
+			// === odd ===
+			const short = shortenRoute(
+				// /aa/aa/aa -> /aa/aa/*
+				"/aa".repeat(maxRuleLength) // 3x limit
+			);
+			expect(short).toEqual("/aa".repeat(maxRuleLength / 3 - 1) + "/*");
+			// Should be the exact max length
+			expect(short.length).toEqual(maxRuleLength - 2); // -2 because of the odd number
+		});
+
+		it("should truncate many single-character segements with wildcard", () => {
+			const short = shortenRoute(
+				// /a/a/a -> /a/a/*
+				"/a".repeat(maxRuleLength) + "/*" // 2x limit
+			);
+			expect(short).toEqual("/a".repeat(maxRuleLength / 2 - 1) + "/*");
+			// Should be the exact max length
+			expect(short.length).toEqual(maxRuleLength);
+		});
+
+		it("should truncate many double-character segements with wildcard", () => {
+			const short = shortenRoute(
+				// /aa/aa/aa -> /aa/*
+				"/aa".repeat(maxRuleLength) + "/*" // 2x limit
+			);
+			expect(short).toEqual("/aa".repeat(maxRuleLength / 3 - 1) + "/*");
+			// Should be the exact max length
+			expect(short.length).toEqual(maxRuleLength - 2); // -2 because of the odd number
+		});
+
+		// This is probably the best test here - tests variable-length segments, up until the max.
+		// This ensures that it's always able to shorten rules, without failing and returning "/*"
+		// The other tests are great for ensuring exact sequences instead of only asserting length, though.
+		for (const suffix of ["", "/", "/*"]) {
+			// Test each type of path: /a, /a/a, /a/*
+			it(`should truncate many variable-character segements (suffix="${suffix}") without truncating to /*`, () => {
+				// "/" + 97 chars + "/*" === 100
+				for (let i = 1; i < maxRuleLength - 2; i++) {
+					const segment = "/" + "a".repeat(i);
+					// make sure the segment isn't too long since we are testing not resulting to /*
+					expect(segment.length).toBeLessThanOrEqual(maxRuleLength);
+					const route =
+						segment.repeat((maxRuleLength / segment.length) * 2) + suffix;
+					// Make sure we made the rule too long
+					expect(route.length).toBeGreaterThan(maxRuleLength);
+					const short = shortenRoute(route);
+
+					// Make sure it's not over the limit
+					expect(short.length).toBeLessThanOrEqual(maxRuleLength);
+					// It should never have to fall back to /*
+					expect(short).not.toEqual("/*");
+				}
+			});
+		}
+	});
+});

package/src/pages/functions/routes-consolidation.ts

@@ -0,0 +1,73 @@
+import { MAX_FUNCTIONS_ROUTES_RULE_LENGTH } from "../constants";
+
+/**
+ * consolidateRoutes consolidates redundant routes - eg. ["/api/*"", "/api/foo"] -> ["/api/*""]
+ * @param routes If this is the same order as Functions routes (with most-specific first),
+ *   it will be more efficient to reverse it first. Should be in the format: /api/foo, /api/*
+ * @returns Non-redundant list of routes
+ */
+export function consolidateRoutes(routes: string[]): string[] {
+	// First we need to trim any rules that are too long and deduplicate the result
+	const routesShortened = Array.from(
+		new Set(routes.map((route) => shortenRoute(route)))
+	);
+
+	// create a map of the routes
+	const routesMap = new Map<string, boolean>();
+	for (const route of routesShortened) {
+		routesMap.set(route, true);
+	}
+	// Find routes that might render other routes redundant
+	for (const route of routesShortened.filter((r) => r.endsWith("/*"))) {
+		// Make sure the route still exists in the map
+		if (routesMap.has(route)) {
+			// Remove splat at the end, leaving the /
+			// eg. /api/* -> /api/
+			const routeTrimmed = route.substring(0, route.length - 1);
+			for (const nextRoute of routesMap.keys()) {
+				// Delete any route that has the wildcard route as a prefix
+				if (nextRoute !== route && nextRoute.startsWith(routeTrimmed)) {
+					routesMap.delete(nextRoute);
+				}
+			}
+		}
+	}
+	return Array.from(routesMap.keys());
+}
+
+/**
+ * Shortens a route until it's within the rule length limit defined in
+ * constants.MAX_FUNCTIONS_ROUTES_RULE_LENGTH
+ * Eg. /aaa/bbb -> /aaa/*
+ * @param routeToShorten Route to shorten if needed
+ * @param maxLength Max length of route to try to shorten to
+ */
+export function shortenRoute(
+	routeToShorten: string,
+	maxLength: number = MAX_FUNCTIONS_ROUTES_RULE_LENGTH
+): string {
+	if (routeToShorten.length <= maxLength) {
+		return routeToShorten;
+	}
+
+	let route = routeToShorten;
+	// May have to try multiple times for longer segments
+	for (let i = 0; i < routeToShorten.length; i++) {
+		// Shorten to the first slash within the limit
+		for (let j = maxLength - 1 - i; j > 0; j--) {
+			if (route[j] === "/") {
+				route = route.slice(0, j) + "/*";
+				break;
+			}
+		}
+		if (route.length <= maxLength) {
+			break;
+		}
+	}
+
+	// If we failed to shorten it, fall back to include-all rather than breaking
+	if (route.length > maxLength) {
+		route = "/*";
+	}
+	return route;
+}