wrangler 0.0.0-e7a0cd5b → 0.0.0-e87ad068

package/src/pages/{dev.tsx → dev.ts}

@@ -1,5 +1,5 @@
 import { execSync, spawn } from "node:child_process";
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync, lstatSync, readFileSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { watch } from "chokidar";
@@ -10,23 +10,29 @@ import { FatalError } from "../errors";
 import { logger } from "../logger";
 import * as metrics from "../metrics";
 import { getBasePath } from "../paths";
-import { buildFunctions } from "./build";
+import { buildFunctions } from "./buildFunctions";
 import { ROUTES_SPEC_VERSION, SECONDS_TO_WAIT_FOR_PROXY } from "./constants";
 import { FunctionsNoRoutesError, getFunctionsNoRoutesWarning } from "./errors";
+import {
+	buildRawWorker,
+	checkRawWorker,
+	traverseAndBuildWorkerJSDirectory,
+} from "./functions/buildWorker";
 import { validateRoutes } from "./functions/routes-validation";
 import { CLEANUP, CLEANUP_CALLBACKS, pagesBetaWarning } from "./utils";
 import type { AdditionalDevProps } from "../dev";
+import type { CfModule } from "../worker";
+import type {
+	CommonYargsArgv,
+	StrictYargsOptionsToInterface,
+} from "../yargs-types";
 import type { RoutesJSONSpec } from "./functions/routes-transformation";
-import type { YargsOptionsToInterface } from "./types";
-import type { Argv } from "yargs";
 
 const DURABLE_OBJECTS_BINDING_REGEXP = new RegExp(
 	/^(?<binding>[^=]+)=(?<className>[^@\s]+)(@(?<scriptName>.*)$)?$/
 );
 
-type PagesDevArgs = YargsOptionsToInterface<typeof Options>;
-
-export function Options(yargs: Argv) {
+export function Options(yargs: CommonYargsArgv) {
 	return yargs
 		.positional("directory", {
 			type: "string",
@@ -78,6 +84,16 @@ export function Options(yargs: Argv) {
 				description:
 					"The location of the single Worker script if not using functions",
 			},
+			bundle: {
+				type: "boolean",
+				default: undefined,
+				hidden: true,
+			},
+			"no-bundle": {
+				type: "boolean",
+				default: false,
+				description: "Whether to run bundling on `_worker.js`",
+			},
 			binding: {
 				type: "array",
 				description: "Bind variable/secret (KEY=VALUE)",
@@ -129,11 +145,16 @@ export function Options(yargs: Argv) {
 				requiresArg: true,
 			},
 			"node-compat": {
-				describe: "Enable node.js compatibility",
+				describe: "Enable Node.js compatibility",
 				default: false,
 				type: "boolean",
 				hidden: true,
 			},
+			"experimental-local": {
+				describe: "Run on my machine using the Cloudflare Workers runtime",
+				type: "boolean",
+				default: false,
+			},
 			config: {
 				describe: "Pages does not support wrangler.toml",
 				type: "string",
@@ -150,35 +171,36 @@ export function Options(yargs: Argv) {
 export const Handler = async ({
 	local,
 	directory,
-	"compatibility-date": compatibilityDate,
-	"compatibility-flags": compatibilityFlags,
+	compatibilityDate,
+	compatibilityFlags,
 	ip,
 	port,
-	"inspector-port": inspectorPort,
+	inspectorPort,
 	proxy: requestedProxyPort,
-	"script-path": singleWorkerScriptPath,
+	bundle,
+	noBundle,
+	scriptPath: singleWorkerScriptPath,
 	binding: bindings = [],
 	kv: kvs = [],
 	do: durableObjects = [],
 	d1: d1s = [],
 	r2: r2s = [],
-	"live-reload": liveReload,
-	"local-protocol": localProtocol,
+	liveReload,
+	localProtocol,
 	experimentalEnableLocalPersistence,
 	persist,
 	persistTo,
-	"node-compat": nodeCompat,
+	nodeCompat: legacyNodeCompat,
+	experimentalLocal,
 	config: config,
 	_: [_pages, _dev, ...remaining],
 	logLevel,
-}: PagesDevArgs) => {
+}: StrictYargsOptionsToInterface<typeof Options>) => {
 	// Beta message for `wrangler pages <commands>` usage
 	logger.log(pagesBetaWarning);
 
-	type LogLevelArg = "debug" | "info" | "log" | "warn" | "error" | "none";
 	if (logLevel) {
-		// The YargsOptionsToInterface doesn't handle the passing in of Unions from choices in Yargs
-		logger.loggerLevel = logLevel as LogLevelArg;
+		logger.loggerLevel = logLevel;
 	}
 
 	if (!local) {
@@ -238,31 +260,73 @@ export const Handler = async ({
 		directory !== undefined
 			? join(directory, singleWorkerScriptPath)
 			: singleWorkerScriptPath;
+	const usingWorkerDirectory =
+		existsSync(workerScriptPath) && lstatSync(workerScriptPath).isDirectory();
 	const usingWorkerScript = existsSync(workerScriptPath);
+	// TODO: Here lies a known bug. If you specify both `--bundle` and `--no-bundle`, this behavior is undefined and you will get unexpected results.
+	// There is no sane way to get the true value out of yargs, so here we are.
+	const enableBundling = bundle ?? !noBundle;
 
 	const functionsDirectory = "./functions";
 	let usingFunctions = !usingWorkerScript && existsSync(functionsDirectory);
 
 	let scriptPath = "";
 
-	if (usingWorkerScript) {
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		scriptReadyResolve!();
-
-		scriptPath = workerScriptPath;
+	const nodejsCompat = compatibilityFlags?.includes("nodejs_compat");
+	let modules: CfModule[] = [];
 
+	if (usingWorkerDirectory) {
 		const runBuild = async () => {
-			try {
-				await esbuild.build({
-					entryPoints: [scriptPath],
-					write: false,
-					plugins: [blockWorkerJsImports],
-				});
-			} catch {}
+			const bundleResult = await traverseAndBuildWorkerJSDirectory({
+				workerJSDirectory: workerScriptPath,
+				buildOutputDirectory: directory ?? ".",
+				nodejsCompat,
+			});
+			modules = bundleResult.modules;
+			scriptPath = bundleResult.resolvedEntryPointPath;
 		};
 
+		await runBuild().then(() => scriptReadyResolve());
+
+		watch([workerScriptPath], {
+			persistent: true,
+			ignoreInitial: true,
+		}).on("all", async () => {
+			await runBuild();
+		});
+	} else if (usingWorkerScript) {
+		scriptPath = workerScriptPath;
+		let runBuild = async () => {
+			await checkRawWorker(workerScriptPath, () => scriptReadyResolve());
+		};
+
+		if (enableBundling) {
+			// We want to actually run the `_worker.js` script through the bundler
+			// So update the final path to the script that will be uploaded and
+			// change the `runBuild()` function to bundle the `_worker.js`.
+			scriptPath = join(tmpdir(), `./bundledWorker-${Math.random()}.mjs`);
+			runBuild = async () => {
+				try {
+					await buildRawWorker({
+						workerScriptPath: usingWorkerDirectory
+							? join(workerScriptPath, "index.js")
+							: workerScriptPath,
+						outfile: scriptPath,
+						directory: directory ?? ".",
+						nodejsCompat,
+						local: true,
+						sourcemap: true,
+						watch: true,
+						onEnd: () => scriptReadyResolve(),
+					});
+				} catch (e: unknown) {
+					logger.warn("Failed to bundle _worker.js.", e);
+				}
+			};
+		}
+
 		await runBuild();
-		watch([scriptPath], {
+		watch([workerScriptPath], {
 			persistent: true,
 			ignoreInitial: true,
 		}).on("all", async () => {
@@ -270,45 +334,47 @@ export const Handler = async ({
 		});
 	} else if (usingFunctions) {
 		// Try to use Functions
-		const outfile = join(tmpdir(), `./functionsWorker-${Math.random()}.js`);
-		scriptPath = outfile;
+		scriptPath = join(tmpdir(), `./functionsWorker-${Math.random()}.mjs`);
 
-		if (nodeCompat) {
+		if (legacyNodeCompat) {
 			console.warn(
-				"Enabling node.js compatibility mode for builtins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
+				"Enabling Node.js compatibility mode for builtins and globals. This is experimental and has serious tradeoffs. Please see https://github.com/ionic-team/rollup-plugin-node-polyfills/ for more details."
 			);
 		}
 
-		logger.log(`Compiling worker to "${outfile}"...`);
+		if (legacyNodeCompat && nodejsCompat) {
+			throw new FatalError(
+				"The `nodejs_compat` compatibility flag cannot be used in conjunction with the legacy `--node-compat` flag. If you want to use the Workers runtime Node.js compatibility features, please remove the `--node-compat` argument from your CLI command or `node_compat = true` from your config file.",
+				1
+			);
+		}
+
+		logger.log(`Compiling worker to "${scriptPath}"...`);
 		const onEnd = () => scriptReadyResolve();
 		try {
-			await buildFunctions({
-				outfile,
-				functionsDirectory,
-				sourcemap: true,
-				watch: true,
-				onEnd,
-				buildOutputDirectory: directory,
-				nodeCompat,
-			});
-			await metrics.sendMetricsEvent("build pages functions");
+			const buildFn = async () => {
+				await buildFunctions({
+					outfile: scriptPath,
+					functionsDirectory,
+					sourcemap: true,
+					watch: true,
+					onEnd,
+					buildOutputDirectory: directory,
+					legacyNodeCompat,
+					nodejsCompat,
+					local: true,
+				});
+				await metrics.sendMetricsEvent("build pages functions");
+			};
 
+			await buildFn();
 			// If Functions found routes, continue using Functions
 			watch([functionsDirectory], {
 				persistent: true,
 				ignoreInitial: true,
 			}).on("all", async () => {
 				try {
-					await buildFunctions({
-						outfile,
-						functionsDirectory,
-						sourcemap: true,
-						watch: true,
-						onEnd,
-						buildOutputDirectory: directory,
-						nodeCompat,
-					});
-					await metrics.sendMetricsEvent("build pages functions");
+					await buildFn();
 				} catch (e) {
 					if (e instanceof FunctionsNoRoutesError) {
 						logger.warn(
@@ -349,7 +415,7 @@ export const Handler = async ({
 
 	if (scriptPath === "") {
 		// Failed to get a script with or without Functions,
-		// something really bad must have happend.
+		// something really bad must have happened.
 		throw new FatalError(
 			"Failed to start wrangler pages dev due to an unknown error",
 			1
@@ -359,7 +425,10 @@ export const Handler = async ({
 	let entrypoint = scriptPath;
 
 	// custom _routes.json apply only to Functions or Advanced Mode Pages projects
-	if (directory && (usingFunctions || usingWorkerScript)) {
+	if (
+		directory &&
+		(usingFunctions || usingWorkerScript || usingWorkerDirectory)
+	) {
 		const routesJSONPath = join(directory, "_routes.json");
 
 		if (existsSync(routesJSONPath)) {
@@ -431,7 +500,7 @@ export const Handler = async ({
 					 * If _routes.json is invalid, don't exit but instead fallback to a sensible default
 					 * and continue to serve the assets. At the same time make sure we warn users that we
 					 * we detected an invalid file and that we'll be using a default.
-					 * This basically equivalates to serving a Functions or _worker.js project as is,
+					 * This basically equates to serving a Functions or _worker.js project as is,
 					 * without applying any additional routing rules on top.
 					 */
 					const error =
@@ -460,77 +529,86 @@ export const Handler = async ({
 		}
 	}
 
-	const { stop, waitUntilExit } = await unstable_dev(
-		entrypoint,
-		{
-			ip,
-			port,
-			inspectorPort,
-			watch: true,
-			localProtocol,
-			liveReload,
-			compatibilityDate,
-			compatibilityFlags,
-			nodeCompat,
-			vars: Object.fromEntries(
-				bindings
-					.map((binding) => binding.toString().split("="))
-					.map(([key, ...values]) => [key, values.join("=")])
-			),
-			kv: kvs.map((val) => ({
-				binding: val.toString(),
-				id: "",
-			})),
-			durableObjects: durableObjects
-				.map((durableObject) => {
-					const { binding, className, scriptName } =
-						DURABLE_OBJECTS_BINDING_REGEXP.exec(durableObject.toString())
-							?.groups || {};
-
-					if (!binding || !className) {
-						logger.warn(
-							"Could not parse Durable Object binding:",
-							durableObject.toString()
-						);
-						return;
-					}
-
-					return {
-						name: binding,
-						class_name: className,
-						script_name: scriptName,
-					};
-				})
-				.filter(Boolean) as AdditionalDevProps["durableObjects"],
-			r2: r2s.map((binding) => {
-				return { binding: binding.toString(), bucket_name: "" };
-			}),
+	const { stop, waitUntilExit } = await unstable_dev(entrypoint, {
+		ip,
+		port,
+		inspectorPort,
+		localProtocol,
+		compatibilityDate,
+		compatibilityFlags,
+		nodeCompat: legacyNodeCompat,
+		vars: Object.fromEntries(
+			bindings
+				.map((binding) => binding.toString().split("="))
+				.map(([key, ...values]) => [key, values.join("=")])
+		),
+		kv: kvs.map((binding) => ({
+			binding: binding.toString(),
+			id: "",
+		})),
+		durableObjects: durableObjects
+			.map((durableObject) => {
+				const { binding, className, scriptName } =
+					DURABLE_OBJECTS_BINDING_REGEXP.exec(durableObject.toString())
+						?.groups || {};
+
+				if (!binding || !className) {
+					logger.warn(
+						"Could not parse Durable Object binding:",
+						durableObject.toString()
+					);
+					return;
+				}
 
+				return {
+					name: binding,
+					class_name: className,
+					script_name: scriptName,
+				};
+			})
+			.filter(Boolean) as AdditionalDevProps["durableObjects"],
+		r2: r2s.map((binding) => {
+			return { binding: binding.toString(), bucket_name: "" };
+		}),
+		rules: usingWorkerDirectory
+			? [
+					{
+						type: "ESModule",
+						globs: ["**/*.js"],
+					},
+			  ]
+			: undefined,
+		bundle: enableBundling,
+		persist,
+		persistTo,
+		inspect: undefined,
+		logLevel,
+		experimental: {
+			processEntrypoint: true,
+			additionalModules: modules,
 			d1Databases: d1s.map((binding) => ({
 				binding: binding.toString(),
 				database_id: "", // Required for types, but unused by dev
 				database_name: `local-${binding}`,
 			})),
-
+			disableExperimentalWarning: true,
 			enablePagesAssetsServiceBinding: {
 				proxyPort,
 				directory,
 			},
+			experimentalLocal,
+			liveReload,
 			forceLocal: true,
-			persist,
-			persistTo,
 			showInteractiveDevSession: undefined,
-			inspect: true,
-			logPrefix: "pages",
-			logLevel: logLevel ?? "warn",
+			testMode: false,
+			watch: true,
 		},
-		{ testMode: false, disableExperimentalWarning: true }
-	);
+	});
 	await metrics.sendMetricsEvent("run pages dev");
 
 	CLEANUP_CALLBACKS.push(stop);
 
-	waitUntilExit().then(() => {
+	void waitUntilExit().then(() => {
 		CLEANUP();
 		process.exit(0);
 	});
@@ -679,15 +757,3 @@ async function spawnProxyProcess({
 
 	return port;
 }
-
-const blockWorkerJsImports: esbuild.Plugin = {
-	name: "block-worker-js-imports",
-	setup(build: esbuild.PluginBuild) {
-		build.onResolve({ filter: /.*/g }, (_args) => {
-			logger.error(
-				`_worker.js is importing from another file. This will throw an error if deployed.\nYou should bundle your Worker or remove the import if it is unused.`
-			);
-			return null;
-		});
-	},
-};

package/src/pages/errors.ts

@@ -9,6 +9,7 @@ import { RoutesValidationError } from "./functions/routes-validation";
  * Exit code for `pages functions build` when no routes are found.
  */
 export const EXIT_CODE_FUNCTIONS_NO_ROUTES_ERROR = 156;
+export const EXIT_CODE_FUNCTIONS_NOTHING_TO_BUILD_ERROR = 157;
 
 /**
  * Pages error when no routes are found in the functions directory

package/src/pages/functions/buildPlugin.ts

@@ -1,72 +1,57 @@
 import { access, lstat } from "node:fs/promises";
-import { dirname, relative, resolve } from "node:path";
-import NodeGlobalsPolyfills from "@esbuild-plugins/node-globals-polyfill";
-import NodeModulesPolyfills from "@esbuild-plugins/node-modules-polyfill";
-import { build } from "esbuild";
+import { relative, resolve } from "node:path";
+import { bundleWorker } from "../../bundle";
 import { getBasePath } from "../../paths";
+import { D1_BETA_PREFIX } from "../../worker";
+import { buildNotifierPlugin } from "./buildWorker";
+import type { Options as WorkerOptions } from "./buildWorker";
 
-type Options = {
-	routesModule: string;
-	outfile: string;
-	minify?: boolean;
-	sourcemap?: boolean;
-	watch?: boolean;
-	nodeCompat?: boolean;
-	onEnd?: () => void;
-};
+type Options = Omit<
+	WorkerOptions,
+	"outfile" | "fallbackService" | "buildOutputDirectory" | "nodejsCompat"
+> & { outdir: string };
 
 export function buildPlugin({
 	routesModule,
-	outfile = "bundle.js",
+	outdir,
 	minify = false,
 	sourcemap = false,
 	watch = false,
-	nodeCompat,
 	onEnd = () => {},
+	legacyNodeCompat,
+	functionsDirectory,
+	local,
+	betaD1Shims,
 }: Options) {
-	return build({
-		entryPoints: [resolve(getBasePath(), "templates/pages-template-plugin.ts")],
-		inject: [routesModule],
-		bundle: true,
-		format: "esm",
-		target: "esnext",
-		loader: {
-			".html": "text",
-			".txt": "text",
+	return bundleWorker(
+		{
+			file: resolve(getBasePath(), "templates/pages-template-plugin.ts"),
+			directory: functionsDirectory,
+			format: "modules",
+			moduleRoot: functionsDirectory,
 		},
-		outfile,
-		minify,
-		sourcemap,
-		watch,
-		allowOverwrite: true,
-		define: { ...(nodeCompat ? { global: "globalThis" } : {}) },
-		plugins: [
-			{
-				name: "wrangler notifier and monitor",
-				setup(pluginBuild) {
-					pluginBuild.onEnd((result) => {
-						if (result.errors.length > 0) {
-							console.error(
-								`${result.errors.length} error(s) and ${result.warnings.length} warning(s) when compiling Worker.`
-							);
-						} else if (result.warnings.length > 0) {
-							console.warn(
-								`${result.warnings.length} warning(s) when compiling Worker.`
-							);
-							onEnd();
-						} else {
-							console.log("Compiled Worker successfully.");
-							onEnd();
-						}
-					});
-				},
-			},
-			{
-				name: "Assets",
-				setup(pluginBuild) {
-					if (pluginBuild.initialOptions.outfile) {
-						const outdir = dirname(pluginBuild.initialOptions.outfile);
-
+		resolve(outdir),
+		{
+			inject: [routesModule],
+			entryName: "index",
+			minify,
+			sourcemap,
+			watch,
+			legacyNodeCompat,
+			// We don't currently have a mechanism for Plugins 'requiring' a specific compat date/flag,
+			// but if someone wants to publish a Plugin which does require this new `nodejs_compat` flag
+			// and they document that on their README.md, we should let them.
+			nodejsCompat: true,
+			define: {},
+			betaD1Shims: (betaD1Shims || []).map(
+				(binding) => `${D1_BETA_PREFIX}${binding}`
+			),
+			doBindings: [], // Pages functions don't support internal Durable Objects
+			plugins: [
+				buildNotifierPlugin(onEnd),
+				{
+					name: "Assets",
+					setup(pluginBuild) {
 						pluginBuild.onResolve({ filter: /^assets:/ }, async (args) => {
 							const directory = resolve(
 								args.resolveDir,
@@ -94,17 +79,36 @@ export function buildPlugin({
 
 							return { path, external: true, namespace: "assets" };
 						});
-					}
+					},
+				},
+				// TODO: Replace this with a proper outdir solution for Plugins
+				// But for now, let's just mark all wasm/bin files as external
+				{
+					name: "Mark externals",
+					setup(pluginBuild) {
+						pluginBuild.onResolve(
+							{ filter: /.*\.(wasm|bin)$/ },
+							async (args) => {
+								return {
+									external: true,
+									path: `./${relative(
+										outdir,
+										resolve(args.resolveDir, args.path)
+									)}`,
+								};
+							}
+						);
+					},
 				},
-			},
-			...(nodeCompat
-				? [
-						NodeGlobalsPolyfills({
-							buffer: true,
-						}),
-						NodeModulesPolyfills(),
-				  ]
-				: []),
-		],
-	});
+			],
+			serveAssetsFromWorker: false,
+			disableModuleCollection: false,
+			rules: [],
+			checkFetch: local,
+			targetConsumer: local ? "dev" : "publish",
+			local,
+			experimentalLocal: false,
+			forPages: true,
+		}
+	);
 }