wrangler 0.0.0-e178d6fb → 0.0.0-e1c2f5b9

package/src/miniflare-cli/assets.ts

@@ -1,35 +1,64 @@
 import { existsSync, lstatSync, readFileSync } from "node:fs";
 import { join } from "node:path";
-import { fetch as miniflareFetch } from "@miniflare/core";
+import { createMetadataObject } from "@cloudflare/pages-shared/metadata-generator/createMetadataObject";
+import { parseHeaders } from "@cloudflare/pages-shared/metadata-generator/parseHeaders";
+import { parseRedirects } from "@cloudflare/pages-shared/metadata-generator/parseRedirects";
 import { watch } from "chokidar";
 import { getType } from "mime";
-import { Response } from "miniflare";
-import type { Headers as MiniflareHeaders } from "@miniflare/core";
-import type { Log } from "miniflare";
+import { hashFile } from "../pages/hash";
+import type { Metadata } from "@cloudflare/pages-shared/asset-server/metadata";
 import type {
-	Request as MiniflareRequest,
-	RequestInfo,
-	RequestInit,
-} from "miniflare";
+	ParsedRedirects,
+	ParsedHeaders,
+} from "@cloudflare/pages-shared/metadata-generator/types";
+import type {
+	RequestInfo as TreRequestInfo,
+	RequestInit as TreRequestInit,
+} from "@miniflare/tre";
+
+interface Logger {
+	log: (message: string) => void;
+	warn: (message: string) => void;
+	error: (error: Error) => void;
+}
 
 export interface Options {
-	log: Log;
+	log: Logger;
 	proxyPort?: number;
 	directory?: string;
+	tre: boolean;
 }
 
 export default async function generateASSETSBinding(options: Options) {
 	const assetsFetch =
 		options.directory !== undefined
-			? await generateAssetsFetch(options.directory, options.log)
+			? await generateAssetsFetch(options.directory, options.log, options.tre)
 			: invalidAssetsFetch;
 
-	return async function (request: MiniflareRequest) {
+	const miniflare = options.tre
+		? await import("@miniflare/tre")
+		: await import("@miniflare/core");
+
+	const Request = miniflare.Request;
+	const Response = miniflare.Response;
+	// WebSockets won't work with `--experimental-local` until we expose something like `upgradingFetch` from `@miniflare/tre`.
+	const fetch = (
+		options.tre
+			? miniflare.fetch
+			: (await import("@miniflare/web-sockets")).upgradingFetch
+	) as (request: Request) => Promise<Response>;
+
+	return async function (miniflareRequest: Request) {
 		if (options.proxyPort) {
 			try {
-				const url = new URL(request.url);
+				const url = new URL(miniflareRequest.url);
 				url.host = `localhost:${options.proxyPort}`;
-				return await miniflareFetch(url, request);
+				const proxyRequest = new Request(url, miniflareRequest);
+				if (proxyRequest.headers.get("Upgrade") === "websocket") {
+					proxyRequest.headers.delete("Sec-WebSocket-Accept");
+					proxyRequest.headers.delete("Sec-WebSocket-Key");
+				}
+				return await fetch(proxyRequest as Request);
 			} catch (thrown) {
 				options.log.error(new Error(`Could not proxy request: ${thrown}`));
 
@@ -40,7 +69,7 @@ export default async function generateASSETSBinding(options: Options) {
 			}
 		} else {
 			try {
-				return await assetsFetch(request);
+				return await assetsFetch(miniflareRequest);
 			} catch (thrown) {
 				options.log.error(new Error(`Could not serve static asset: ${thrown}`));
 
@@ -54,239 +83,36 @@ export default async function generateASSETSBinding(options: Options) {
 	};
 }
 
-function escapeRegex(str: string) {
-	return str.replace(/[-/\\^$*+?.()|[]{}]/g, "\\$&");
-}
-
-type Replacements = Record<string, string>;
-
-function replacer(str: string, replacements: Replacements) {
-	for (const [replacement, value] of Object.entries(replacements)) {
-		str = str.replace(`:${replacement}`, value);
-	}
-	return str;
-}
-
-function generateRulesMatcher<T>(
-	rules?: Record<string, T>,
-	replacerFn: (match: T, replacements: Replacements) => T = (match) => match
-) {
-	// TODO: How can you test cross-host rules?
-	if (!rules) return () => [];
-
-	const compiledRules = Object.entries(rules)
-		.map(([rule, match]) => {
-			const crossHost = rule.startsWith("https://");
-
-			rule = rule.split("*").map(escapeRegex).join("(?<splat>.*)");
-
-			const host_matches = rule.matchAll(
-				/(?<=^https:\\\/\\\/[^/]*?):([^\\]+)(?=\\)/g
-			);
-			for (const hostMatch of host_matches) {
-				rule = rule.split(hostMatch[0]).join(`(?<${hostMatch[1]}>[^/.]+)`);
-			}
-
-			const path_matches = rule.matchAll(/:(\w+)/g);
-			for (const pathMatch of path_matches) {
-				rule = rule.split(pathMatch[0]).join(`(?<${pathMatch[1]}>[^/]+)`);
-			}
-
-			rule = "^" + rule + "$";
-
-			try {
-				const regExp = new RegExp(rule);
-				return [{ crossHost, regExp }, match];
-			} catch {}
-		})
-		.filter((value) => value !== undefined) as [
-		{ crossHost: boolean; regExp: RegExp },
-		T
-	][];
-
-	return ({ request }: { request: MiniflareRequest }) => {
-		const { pathname, host } = new URL(request.url);
-
-		return compiledRules
-			.map(([{ crossHost, regExp }, match]) => {
-				const test = crossHost ? `https://${host}${pathname}` : pathname;
-				const result = regExp.exec(test);
-				if (result) {
-					return replacerFn(match, result.groups || {});
-				}
-			})
-			.filter((value) => value !== undefined) as T[];
-	};
-}
-
-function generateHeadersMatcher(headersFile: string) {
-	if (existsSync(headersFile)) {
-		const contents = readFileSync(headersFile).toString();
-
-		// TODO: Log errors
-		const lines = contents
-			.split("\n")
-			.map((line) => line.trim())
-			.filter((line) => !line.startsWith("#") && line !== "");
-
-		const rules: Record<string, Record<string, string>> = {};
-		let rule: { path: string; headers: Record<string, string> } | undefined =
-			undefined;
-
-		for (const line of lines) {
-			if (/^([^\s]+:\/\/|^\/)/.test(line)) {
-				if (rule && Object.keys(rule.headers).length > 0) {
-					rules[rule.path] = rule.headers;
-				}
-
-				const path = validateURL(line);
-				if (path) {
-					rule = {
-						path,
-						headers: {},
-					};
-					continue;
-				}
-			}
-
-			if (!line.includes(":")) continue;
-
-			const [rawName, ...rawValue] = line.split(":");
-			const name = rawName.trim().toLowerCase();
-			const value = rawValue.join(":").trim();
-
-			if (name === "") continue;
-			if (!rule) continue;
-
-			const existingValues = rule.headers[name];
-			rule.headers[name] = existingValues
-				? `${existingValues}, ${value}`
-				: value;
-		}
-
-		if (rule && Object.keys(rule.headers).length > 0) {
-			rules[rule.path] = rule.headers;
-		}
-
-		const rulesMatcher = generateRulesMatcher(rules, (match, replacements) =>
-			Object.fromEntries(
-				Object.entries(match).map(([name, value]) => [
-					name,
-					replacer(value, replacements),
-				])
-			)
-		);
-
-		return (request: MiniflareRequest) => {
-			const matches = rulesMatcher({
-				request,
-			});
-			if (matches) return matches;
-		};
-	} else {
-		return () => undefined;
-	}
-}
-
-function generateRedirectsMatcher(redirectsFile: string) {
-	if (existsSync(redirectsFile)) {
-		const contents = readFileSync(redirectsFile).toString();
-
-		// TODO: Log errors
-		const lines = contents
-			.split("\n")
-			.map((line) => line.trim())
-			.filter((line) => !line.startsWith("#") && line !== "");
-
-		const rules = Object.fromEntries(
-			lines
-				.map((line) => line.split(" "))
-				.filter((tokens) => tokens.length === 2 || tokens.length === 3)
-				.map((tokens) => {
-					const from = validateURL(tokens[0], true, false, false);
-					const to = validateURL(tokens[1], false, true, true);
-					let status: number | undefined = parseInt(tokens[2]) || 302;
-					status = [301, 302, 303, 307, 308].includes(status)
-						? status
-						: undefined;
-
-					return from && to && status ? [from, { to, status }] : undefined;
-				})
-				.filter((rule) => rule !== undefined) as [
-				string,
-				{ to: string; status?: number }
-			][]
-		);
-
-		const rulesMatcher = generateRulesMatcher(
-			rules,
-			({ status, to }, replacements) => ({
-				status,
-				to: replacer(to, replacements),
-			})
-		);
-
-		return (request: MiniflareRequest) => {
-			const match = rulesMatcher({
-				request,
-			})[0];
-			if (match) return match;
-		};
-	} else {
-		return () => undefined;
-	}
-}
-
-function extractPathname(
-	path = "/",
-	includeSearch: boolean,
-	includeHash: boolean
-) {
-	if (!path.startsWith("/")) path = `/${path}`;
-	const url = new URL(`//${path}`, "relative://");
-	return `${url.pathname}${includeSearch ? url.search : ""}${
-		includeHash ? url.hash : ""
-	}`;
-}
-
-function validateURL(
-	token: string,
-	onlyRelative = false,
-	includeSearch = false,
-	includeHash = false
-) {
-	const host = /^https:\/\/+(?<host>[^/]+)\/?(?<path>.*)/.exec(token);
-	if (host && host.groups && host.groups.host) {
-		if (onlyRelative) return;
-
-		return `https://${host.groups.host}${extractPathname(
-			host.groups.path,
-			includeSearch,
-			includeHash
-		)}`;
-	} else {
-		if (!token.startsWith("/") && onlyRelative) token = `/${token}`;
-
-		const path = /^\//.exec(token);
-		if (path) {
-			try {
-				return extractPathname(token, includeSearch, includeHash);
-			} catch {}
-		}
-	}
-	return "";
-}
-
-function hasFileExtension(pathname: string) {
-	return /\/.+\.[a-z0-9]+$/i.test(pathname);
-}
-
 async function generateAssetsFetch(
 	directory: string,
-	log: Log
-): Promise<typeof miniflareFetch> {
+	log: Logger,
+	tre: boolean
+): Promise<typeof fetch> {
 	// Defer importing miniflare until we really need it
-	const { Headers, Request } = await import("@miniflare/core");
+
+	// NOTE: These dynamic imports bring in `global` type augmentations from
+	// `@cloudflare/pages-shared/environment-polyfills/types.ts`, allowing us to
+	// use `fetch`, `Headers`, `Request` and `Response` as globals in this file
+	// and the *entire* `miniflare-cli` TypeScript project.
+	const polyfill = tre
+		? (
+				await import(
+					"@cloudflare/pages-shared/environment-polyfills/miniflare-tre"
+				)
+		  ).default
+		: (await import("@cloudflare/pages-shared/environment-polyfills/miniflare"))
+				.default;
+
+	await polyfill();
+
+	const miniflare = tre
+		? await import("@miniflare/tre")
+		: await import("@miniflare/core");
+	const Request = miniflare.Request;
+
+	const { generateHandler, parseQualityWeightedList } = await import(
+		"@cloudflare/pages-shared/asset-server/handler"
+	);
 
 	const headersFile = join(directory, "_headers");
 	const redirectsFile = join(directory, "_redirects");
@@ -294,249 +120,130 @@ async function generateAssetsFetch(
 
 	const ignoredFiles = [headersFile, redirectsFile, workerFile];
 
-	const assetExists = (path: string) => {
-		path = join(directory, path);
-		return (
-			existsSync(path) &&
-			lstatSync(path).isFile() &&
-			!ignoredFiles.includes(path)
-		);
-	};
-
-	const getAsset = (path: string) => {
-		if (assetExists(path)) {
-			return join(directory, path);
-		}
-	};
+	let redirects: ParsedRedirects | undefined;
+	if (existsSync(redirectsFile)) {
+		const contents = readFileSync(redirectsFile, "utf-8");
+		redirects = parseRedirects(contents);
+	}
 
-	let redirectsMatcher = generateRedirectsMatcher(redirectsFile);
-	let headersMatcher = generateHeadersMatcher(headersFile);
+	let headers: ParsedHeaders | undefined;
+	if (existsSync(headersFile)) {
+		const contents = readFileSync(headersFile, "utf-8");
+		headers = parseHeaders(contents);
+	}
 
-	watch([headersFile, redirectsFile], {
-		persistent: true,
-	}).on("change", (path) => {
-		switch (path) {
-			case headersFile: {
-				log.log("_headers modified. Re-evaluating...");
-				headersMatcher = generateHeadersMatcher(headersFile);
-				break;
-			}
-			case redirectsFile: {
-				log.log("_redirects modified. Re-evaluating...");
-				redirectsMatcher = generateRedirectsMatcher(redirectsFile);
-				break;
-			}
-		}
+	let metadata = createMetadataObject({
+		redirects,
+		headers,
+		logger: log.warn.bind(log),
 	});
 
-	const serveAsset = (file: string) => {
-		return readFileSync(file);
-	};
-
-	const generateResponse = (request: MiniflareRequest) => {
-		const url = new URL(request.url);
-
-		const deconstructedResponse: {
-			status: number;
-			headers: MiniflareHeaders;
-			body?: Buffer;
-		} = {
-			status: 200,
-			headers: new Headers(),
-			body: undefined,
-		};
-
-		const match = redirectsMatcher(request);
-		if (match) {
-			const { status, to } = match;
-
-			let location = to;
-			let search;
-
-			if (to.startsWith("/")) {
-				search = new URL(location, "http://fakehost").search;
-			} else {
-				search = new URL(location).search;
-			}
-
-			location = `${location}${search ? "" : url.search}`;
-
-			if (status && [301, 302, 303, 307, 308].includes(status)) {
-				deconstructedResponse.status = status;
-			} else {
-				deconstructedResponse.status = 302;
-			}
-
-			deconstructedResponse.headers.set("Location", location);
-			return deconstructedResponse;
-		}
-
-		if (!request.method?.match(/^(get|head)$/i)) {
-			deconstructedResponse.status = 405;
-			return deconstructedResponse;
-		}
-
-		const notFound = () => {
-			let cwd = url.pathname;
-			while (cwd) {
-				cwd = cwd.slice(0, cwd.lastIndexOf("/"));
-
-				if ((asset = getAsset(`${cwd}/404.html`))) {
-					deconstructedResponse.status = 404;
-					deconstructedResponse.body = serveAsset(asset);
-					deconstructedResponse.headers.set(
-						"Content-Type",
-						getType(asset) || "application/octet-stream"
-					);
-					return deconstructedResponse;
+	watch([headersFile, redirectsFile], { persistent: true }).on(
+		"change",
+		(path) => {
+			switch (path) {
+				case headersFile: {
+					log.log("_headers modified. Re-evaluating...");
+					const contents = readFileSync(headersFile).toString();
+					headers = parseHeaders(contents);
+					break;
+				}
+				case redirectsFile: {
+					log.log("_redirects modified. Re-evaluating...");
+					const contents = readFileSync(redirectsFile).toString();
+					redirects = parseRedirects(contents);
+					break;
 				}
 			}
 
-			if ((asset = getAsset(`/index.html`))) {
-				deconstructedResponse.body = serveAsset(asset);
-				deconstructedResponse.headers.set(
-					"Content-Type",
-					getType(asset) || "application/octet-stream"
-				);
-				return deconstructedResponse;
-			}
-
-			deconstructedResponse.status = 404;
-			return deconstructedResponse;
-		};
-
-		let asset;
-
-		if (url.pathname.endsWith("/")) {
-			if ((asset = getAsset(`${url.pathname}/index.html`))) {
-				deconstructedResponse.body = serveAsset(asset);
-				deconstructedResponse.headers.set(
-					"Content-Type",
-					getType(asset) || "application/octet-stream"
-				);
-				return deconstructedResponse;
-			} else if (
-				(asset = getAsset(`${url.pathname.replace(/\/$/, ".html")}`))
-			) {
-				deconstructedResponse.status = 301;
-				deconstructedResponse.headers.set(
-					"Location",
-					`${url.pathname.slice(0, -1)}${url.search}`
-				);
-				return deconstructedResponse;
-			}
+			metadata = createMetadataObject({
+				redirects,
+				headers,
+				logger: log.warn,
+			});
 		}
+	);
 
-		if (url.pathname.endsWith("/index")) {
-			deconstructedResponse.status = 301;
-			deconstructedResponse.headers.set(
-				"Location",
-				`${url.pathname.slice(0, -"index".length)}${url.search}`
-			);
-			return deconstructedResponse;
-		}
+	const generateResponse = async (request: Request) => {
+		const assetKeyEntryMap = new Map<string, string>();
+
+		return await generateHandler<string>({
+			request,
+			metadata: metadata as Metadata,
+			xServerEnvHeader: "dev",
+			logError: console.error,
+			findAssetEntryForPath: async (path) => {
+				const filepath = join(directory, path);
+
+				if (
+					existsSync(filepath) &&
+					lstatSync(filepath).isFile() &&
+					!ignoredFiles.includes(filepath)
+				) {
+					const hash = hashFile(filepath);
+					assetKeyEntryMap.set(hash, filepath);
+					return hash;
+				}
 
-		if ((asset = getAsset(url.pathname))) {
-			if (url.pathname.endsWith(".html")) {
-				const extensionlessPath = url.pathname.slice(0, -".html".length);
-				if (getAsset(extensionlessPath) || extensionlessPath === "/") {
-					deconstructedResponse.body = serveAsset(asset);
-					deconstructedResponse.headers.set(
-						"Content-Type",
-						getType(asset) || "application/octet-stream"
-					);
-					return deconstructedResponse;
+				return null;
+			},
+			getAssetKey: (assetEntry) => {
+				return assetEntry;
+			},
+			negotiateContent: (contentRequest) => {
+				let rawAcceptEncoding: string | undefined;
+				if (
+					contentRequest.cf &&
+					"clientAcceptEncoding" in contentRequest.cf &&
+					contentRequest.cf.clientAcceptEncoding
+				) {
+					rawAcceptEncoding = contentRequest.cf.clientAcceptEncoding as string;
 				} else {
-					deconstructedResponse.status = 301;
-					deconstructedResponse.headers.set(
-						"Location",
-						`${extensionlessPath}${url.search}`
-					);
-					return deconstructedResponse;
+					rawAcceptEncoding =
+						contentRequest.headers.get("Accept-Encoding") || undefined;
 				}
-			} else {
-				deconstructedResponse.body = serveAsset(asset);
-				deconstructedResponse.headers.set(
-					"Content-Type",
-					getType(asset) || "application/octet-stream"
-				);
-				return deconstructedResponse;
-			}
-		} else if (hasFileExtension(url.pathname)) {
-			notFound();
-			return deconstructedResponse;
-		}
 
-		if ((asset = getAsset(`${url.pathname}.html`))) {
-			deconstructedResponse.body = serveAsset(asset);
-			deconstructedResponse.headers.set(
-				"Content-Type",
-				getType(asset) || "application/octet-stream"
-			);
-			return deconstructedResponse;
-		}
+				const acceptEncoding = parseQualityWeightedList(rawAcceptEncoding);
 
-		if ((asset = getAsset(`${url.pathname}/index.html`))) {
-			deconstructedResponse.status = 301;
-			deconstructedResponse.headers.set(
-				"Location",
-				`${url.pathname}/${url.search}`
-			);
-			return deconstructedResponse;
-		} else {
-			notFound();
-			return deconstructedResponse;
-		}
-	};
-
-	const attachHeaders = (
-		request: MiniflareRequest,
-		deconstructedResponse: {
-			status: number;
-			headers: MiniflareHeaders;
-			body?: Buffer;
-		}
-	) => {
-		const headers = deconstructedResponse.headers;
-		const newHeaders = new Headers({});
-		const matches = headersMatcher(request) || [];
-
-		matches.forEach((match) => {
-			Object.entries(match).forEach(([name, value]) => {
-				newHeaders.append(name, `${value}`);
-			});
-		});
+				if (
+					acceptEncoding["identity"] === 0 ||
+					(acceptEncoding["*"] === 0 &&
+						acceptEncoding["identity"] === undefined)
+				) {
+					throw new Error("No acceptable encodings available");
+				}
 
-		const combinedHeaders = {
-			...Object.fromEntries(headers.entries()),
-			...Object.fromEntries(newHeaders.entries()),
-		};
+				return { encoding: null };
+			},
+			fetchAsset: async (assetKey) => {
+				const filepath = assetKeyEntryMap.get(assetKey);
+				if (!filepath) {
+					throw new Error(
+						"Could not fetch asset. Please file an issue on GitHub (https://github.com/cloudflare/wrangler2/issues/new/choose) with reproduction steps."
+					);
+				}
+				const body = readFileSync(filepath) as unknown as ReadableStream;
+
+				let contentType = getType(filepath) || "application/octet-stream";
+				if (
+					contentType.startsWith("text/") &&
+					!contentType.includes("charset")
+				) {
+					contentType = `${contentType}; charset=utf-8`;
+				}
 
-		deconstructedResponse.headers = new Headers({});
-		Object.entries(combinedHeaders).forEach(([name, value]) => {
-			if (value) deconstructedResponse.headers.set(name, value);
+				return { body, contentType };
+			},
 		});
 	};
 
-	return async (input: RequestInfo, init?: RequestInit) => {
+	return (async (input: TreRequestInfo, init?: TreRequestInit) => {
 		const request = new Request(input, init);
-		const deconstructedResponse = generateResponse(request);
-		attachHeaders(request, deconstructedResponse);
-
-		const headers = new Headers();
-
-		[...deconstructedResponse.headers.entries()].forEach(([name, value]) => {
-			if (value) headers.set(name, value);
-		});
-
-		return new Response(deconstructedResponse.body, {
-			headers,
-			status: deconstructedResponse.status,
-		});
-	};
+		return await generateResponse(request as unknown as Request);
+	}) as typeof fetch;
 }
 
-const invalidAssetsFetch: typeof miniflareFetch = () => {
+const invalidAssetsFetch: typeof fetch = () => {
 	throw new Error(
 		"Trying to fetch assets directly when there is no `directory` option specified."
 	);