wrangler 0.0.0-e7a0cd5b → 0.0.0-e7cf0caa

package/src/dev/local.tsx

@@ -1,12 +1,15 @@
 import assert from "node:assert";
 import { fork } from "node:child_process";
 import { realpathSync } from "node:fs";
-import { readFile, writeFile } from "node:fs/promises";
+import { readFile } from "node:fs/promises";
 import path from "node:path";
 import chalk from "chalk";
+import getPort from "get-port";
 import { npxImport } from "npx-import";
 import { useState, useEffect, useRef } from "react";
 import onExit from "signal-exit";
+import { fetch } from "undici";
+import { performApiFetch } from "../cfetch/internal";
 import { registerWorker } from "../dev-registry";
 import useInspector from "../inspect";
 import { logger } from "../logger";
@@ -16,9 +19,11 @@ import {
 } from "../module-collection";
 import { getBasePath } from "../paths";
 import { waitForPortToBeAvailable } from "../proxy";
+import { requireAuth } from "../user";
 import type { Config } from "../config";
 import type { WorkerRegistry } from "../dev-registry";
-import type { EnablePagesAssetsServiceBindingOptions } from "../miniflare-cli";
+import type { LoggerLevel } from "../logger";
+import type { EnablePagesAssetsServiceBindingOptions } from "../miniflare-cli/types";
 import type { AssetPaths } from "../sites";
 import type {
 	CfWorkerInit,
@@ -37,6 +42,8 @@ import type { EsbuildBundle } from "./use-esbuild";
 import type {
 	Miniflare as Miniflare3Type,
 	MiniflareOptions as Miniflare3Options,
+	Log as Miniflare3LogType,
+	CloudflareFetch,
 } from "@miniflare/tre";
 import type { MiniflareOptions } from "miniflare";
 import type { ChildProcess } from "node:child_process";
@@ -51,8 +58,8 @@ export interface LocalProps {
 	bindings: CfWorkerInit["bindings"];
 	workerDefinitions: WorkerRegistry | undefined;
 	assetPaths: AssetPaths | undefined;
-	port: number;
-	ip: string;
+	initialPort: number;
+	initialIp: string;
 	rules: Config["rules"];
 	inspectorPort: number;
 	localPersistencePath: string | null;
@@ -63,18 +70,31 @@ export interface LocalProps {
 	localUpstream: string | undefined;
 	inspect: boolean;
 	onReady: ((ip: string, port: number) => void) | undefined;
-	logPrefix?: string;
 	enablePagesAssetsServiceBinding?: EnablePagesAssetsServiceBindingOptions;
 	testScheduled?: boolean;
-	experimentalLocal?: boolean;
+	experimentalLocal: boolean | undefined;
+	accountId: string | undefined; // Account ID? In local mode??? :exploding_head:
+	experimentalLocalRemoteKv: boolean | undefined;
 }
 
+type InspectorJSON = {
+	id: string;
+	title: string;
+	type: "node";
+	description: string;
+	webSocketDebuggerUrl: string;
+	devtoolsFrontendUrl: string;
+	devtoolsFrontendUrlCompat: string;
+	faviconUrl: string;
+	url: string;
+}[];
+
 export function Local(props: LocalProps) {
 	const { inspectorUrl } = useLocalWorker(props);
 	useInspector({
 		inspectorUrl,
 		port: props.inspectorPort,
-		logToTerminal: false,
+		logToTerminal: props.experimentalLocal ?? false,
 	});
 	return null;
 }
@@ -89,28 +109,34 @@ function useLocalWorker({
 	bindings,
 	workerDefinitions,
 	assetPaths,
-	port,
-	inspectorPort,
+	initialPort,
 	rules,
 	localPersistencePath,
 	liveReload,
-	ip,
+	initialIp,
 	crons,
 	queueConsumers,
 	localProtocol,
 	localUpstream,
 	inspect,
 	onReady,
-	logPrefix,
 	enablePagesAssetsServiceBinding,
 	experimentalLocal,
+	accountId,
+	experimentalLocalRemoteKv,
 }: LocalProps) {
 	// TODO: pass vars via command line
 	const local = useRef<ChildProcess>();
 	const experimentalLocalRef = useRef<Miniflare3Type>();
 	const removeSignalExitListener = useRef<() => void>();
+	const removeExperimentalLocalSignalExitListener = useRef<() => void>();
 	const [inspectorUrl, setInspectorUrl] = useState<string | undefined>();
 
+	// Our inspector proxy server will be binding to `LocalProps`'s `inspectorPort`.
+	// If we attempted to bind Node.js/workerd to the same inspector port, we'd get a port already in use error.
+	// Therefore, generate a new random port for our runtime's to bind their inspector service to.
+	const runtimeInspectorPortRef = useRef<number>();
+
 	useEffect(() => {
 		if (bindings.services && bindings.services.length > 0) {
 			logger.warn(
@@ -136,22 +162,6 @@ function useLocalWorker({
 		async function startLocalWorker() {
 			if (!bundle || !format) return;
 
-			// port for the worker
-			await waitForPortToBeAvailable(port, {
-				retryPeriod: 200,
-				timeout: 2000,
-				abortSignal: abortController.signal,
-			});
-
-			// In local mode, we want to copy all referenced modules into
-			// the output bundle directory before starting up
-			for (const module of bundle.modules) {
-				await writeFile(
-					path.join(path.dirname(bundle.path), module.name),
-					module.content
-				);
-			}
-
 			const scriptPath = realpathSync(bundle.path);
 
 			const upstream =
@@ -174,12 +184,15 @@ function useLocalWorker({
 				bundle,
 			});
 
-			const { forkOptions, miniflareCLIPath } = setupMiniflareOptions({
+			runtimeInspectorPortRef.current ??= await getPort();
+			const runtimeInspectorPort = runtimeInspectorPortRef.current;
+
+			const { forkOptions, miniflareCLIPath, options } = setupMiniflareOptions({
 				workerName,
-				port,
+				port: initialPort,
 				scriptPath,
 				localProtocol,
-				ip,
+				ip: initialIp,
 				format,
 				rules,
 				compatibilityDate,
@@ -201,35 +214,102 @@ function useLocalWorker({
 				dataBlobBindings,
 				crons,
 				upstream,
-				logPrefix,
 				workerDefinitions,
 				enablePagesAssetsServiceBinding,
 			});
 
 			if (experimentalLocal) {
-				// TODO: refactor setupMiniflareOptions so we don't need to parse here
-				const mf2Options: MiniflareOptions = JSON.parse(forkOptions[0]);
-				const mf = await setupExperimentalLocal(mf2Options, format, bundle);
-				await mf.ready;
-				experimentalLocalRef.current = mf;
-				removeSignalExitListener.current = onExit(() => {
-					logger.log("⎔ Shutting down experimental local server.");
-					mf.dispose();
-					experimentalLocalRef.current = undefined;
+				const log = await buildMiniflare3Logger();
+				const mf3Options = await transformMf2OptionsToMf3Options({
+					miniflare2Options: options,
+					format,
+					bundle,
+					log,
+					enablePagesAssetsServiceBinding,
+					kvNamespaces: bindings?.kv_namespaces,
+					r2Buckets: bindings?.r2_buckets,
+					d1Databases: bindings?.d1_databases,
+					authenticatedAccountId: accountId,
+					kvRemote: experimentalLocalRemoteKv,
+					inspectorPort: runtimeInspectorPort,
 				});
+
+				const current = experimentalLocalRef.current;
+
+				if (current === undefined) {
+					// If we don't have an active Miniflare instance, create a new one
+					const { Miniflare } = await getMiniflare3();
+					if (abortController.signal.aborted) return;
+					const mf = new Miniflare(mf3Options);
+					experimentalLocalRef.current = mf;
+					removeExperimentalLocalSignalExitListener.current = onExit(() => {
+						logger.log("⎔ Shutting down experimental local server.");
+						void mf.dispose();
+						experimentalLocalRef.current = undefined;
+					});
+					await mf.ready;
+				} else {
+					// Otherwise, reuse the existing instance with its loopback server
+					// and just update the options
+					if (abortController.signal.aborted) return;
+					logger.log("⎔ Reloading experimental local server.");
+					await current.setOptions(mf3Options);
+				}
+
+				try {
+					// fetch the inspector JSON response from the DevTools Inspector protocol
+					const inspectorJSONArr = (await (
+						await fetch(`http://127.0.0.1:${runtimeInspectorPort}/json`)
+					).json()) as InspectorJSON;
+
+					const foundInspectorURL = inspectorJSONArr?.find((inspectorJSON) =>
+						inspectorJSON.id.startsWith("core:user")
+					)?.webSocketDebuggerUrl;
+					if (foundInspectorURL === undefined) {
+						setInspectorUrl(undefined);
+					} else {
+						const url = new URL(foundInspectorURL);
+						// Force inspector URL to be different on each reload so `useEffect`
+						// in `useInspector` is re-run to connect to newly restarted
+						// `workerd` server when updating options. Can't use a query param
+						// here as that seems to cause an infinite connection loop, can't
+						// use a hash as those are forbidden by `ws`, so username it is.
+						url.username = `${Date.now()}-${Math.floor(
+							Math.random() * Number.MAX_SAFE_INTEGER
+						)}`;
+						setInspectorUrl(url.toString());
+					}
+				} catch (error: unknown) {
+					logger.error("Error attempting to retrieve Debugger URL:", error);
+				}
+
 				return;
 			}
 
-			const nodeOptions = setupNodeOptions({ inspect, ip, inspectorPort });
+			// Wait for the Worker port to be available. We don't want to do this in experimental local
+			// mode, as we only `dispose()` the Miniflare 3 instance, and shutdown the server when
+			// unmounting the component, not when props change. If we did, we'd just timeout every time.
+			await waitForPortToBeAvailable(initialPort, {
+				retryPeriod: 200,
+				timeout: 2000,
+				abortSignal: abortController.signal,
+			});
+
+			const nodeOptions = setupNodeOptions({
+				inspect,
+				inspectorPort: runtimeInspectorPort,
+			});
 			logger.log("⎔ Starting a local server...");
 
+			const hasColourSupport =
+				chalk.supportsColor.hasBasic && process.env.FORCE_COLOR !== "0";
 			const child = (local.current = fork(miniflareCLIPath, forkOptions, {
 				cwd: path.dirname(scriptPath),
 				execArgv: nodeOptions,
 				stdio: "pipe",
 				env: {
 					...process.env,
-					FORCE_COLOR: chalk.supportsColor.hasBasic ? "1" : undefined,
+					FORCE_COLOR: hasColourSupport ? "1" : undefined,
 				},
 			}));
 
@@ -241,21 +321,21 @@ function useLocalWorker({
 						await registerWorker(workerName, {
 							protocol: localProtocol,
 							mode: "local",
-							port,
-							host: ip,
+							port: message.port,
+							host: initialIp,
 							durableObjects: internalDurableObjects.map((binding) => ({
 								name: binding.name,
 								className: binding.class_name,
 							})),
 							...(message.durableObjectsPort
 								? {
-										durableObjectsHost: ip,
+										durableObjectsHost: initialIp,
 										durableObjectsPort: message.durableObjectsPort,
 								  }
 								: {}),
 						});
 					}
-					onReady?.(ip, message.mfPort);
+					onReady?.(initialIp, message.port);
 				}
 			});
 
@@ -307,7 +387,13 @@ function useLocalWorker({
 		}
 
 		startLocalWorker().catch((err) => {
-			logger.error("local worker:", err);
+			if (err.code === "ERR_RUNTIME_FAILURE") {
+				// Don't log a full verbose stack-trace when Miniflare 3's workerd instance fails to start.
+				// workerd will log its own errors, and our stack trace won't have any useful information.
+				logger.error(err.message);
+			} else {
+				logger.error("local worker:", err);
+			}
 		});
 
 		return () => {
@@ -317,13 +403,6 @@ function useLocalWorker({
 				local.current?.kill();
 				local.current = undefined;
 			}
-			if (experimentalLocalRef.current) {
-				logger.log("⎔ Shutting down experimental local server.");
-				// Initialisation errors are also thrown asynchronously by dispose().
-				// The catch() above should've caught them though.
-				experimentalLocalRef.current?.dispose().catch(() => {});
-				experimentalLocalRef.current = undefined;
-			}
 			removeSignalExitListener.current?.();
 			removeSignalExitListener.current = undefined;
 		};
@@ -331,9 +410,8 @@ function useLocalWorker({
 		bundle,
 		workerName,
 		format,
-		port,
-		inspectorPort,
-		ip,
+		initialPort,
+		initialIp,
 		queueConsumers,
 		bindings.queues,
 		bindings.durable_objects,
@@ -357,11 +435,32 @@ function useLocalWorker({
 		localProtocol,
 		localUpstream,
 		inspect,
-		logPrefix,
 		onReady,
 		enablePagesAssetsServiceBinding,
 		experimentalLocal,
+		accountId,
+		experimentalLocalRemoteKv,
 	]);
+
+	// Rather than disposing the Miniflare instance on every reload, only dispose
+	// it if local mode is disabled and the `Local` component is unmounted. This
+	// allows us to use the more efficient `Miniflare#setOptions` on reload which
+	// retains internal state (e.g. the Miniflare loopback server).
+	useEffect(
+		() => () => {
+			if (experimentalLocalRef.current) {
+				logger.log("⎔ Shutting down experimental local server.");
+				// Initialisation errors are also thrown asynchronously by dispose().
+				// The catch() above should've caught them though.
+				experimentalLocalRef.current?.dispose().catch(() => {});
+				experimentalLocalRef.current = undefined;
+			}
+			removeExperimentalLocalSignalExitListener.current?.();
+			removeExperimentalLocalSignalExitListener.current = undefined;
+		},
+		[]
+	);
+
 	return { inspectorUrl };
 }
 
@@ -474,7 +573,6 @@ interface SetupMiniflareOptionsProps {
 	dataBlobBindings: Record<string, string>;
 	crons: Config["triggers"]["crons"];
 	upstream: string | undefined;
-	logPrefix: string | undefined;
 	workerDefinitions: WorkerRegistry | undefined;
 	enablePagesAssetsServiceBinding?: EnablePagesAssetsServiceBindingOptions;
 }
@@ -506,15 +604,15 @@ export function setupMiniflareOptions({
 	dataBlobBindings,
 	crons,
 	upstream,
-	logPrefix,
 	workerDefinitions,
 	enablePagesAssetsServiceBinding,
 }: SetupMiniflareOptionsProps): {
 	miniflareCLIPath: string;
 	forkOptions: string[];
+	options: MiniflareOptions;
 } {
 	// It's now getting _really_ messy now with Pages ASSETS binding outside and the external Durable Objects inside.
-	const options = {
+	const options: MiniflareOptions = {
 		name: workerName,
 		port,
 		scriptPath,
@@ -618,7 +716,6 @@ export function setupMiniflareOptions({
 		crons,
 		upstream,
 		logLevel: logger.loggerLevel,
-		logOptions: logPrefix ? { prefix: logPrefix } : undefined,
 		enablePagesAssetsServiceBinding,
 	};
 	// The path to the Miniflare CLI assumes that this file is being run from
@@ -633,16 +730,14 @@ export function setupMiniflareOptions({
 	if (enablePagesAssetsServiceBinding) {
 		forkOptions.push(JSON.stringify(enablePagesAssetsServiceBinding));
 	}
-	return { miniflareCLIPath, forkOptions };
+	return { miniflareCLIPath, forkOptions, options };
 }
 
 export function setupNodeOptions({
 	inspect,
-	ip,
 	inspectorPort,
 }: {
 	inspect: boolean;
-	ip: string;
 	inspectorPort: number;
 }) {
 	const nodeOptions = [
@@ -651,33 +746,123 @@ export function setupNodeOptions({
 		// "--log=VERBOSE", // uncomment this to Miniflare to log "everything"!
 	];
 	if (inspect) {
-		nodeOptions.push("--inspect=" + `${ip}:${inspectorPort}`); // start Miniflare listening for a debugger to attach
+		nodeOptions.push("--inspect=" + `127.0.0.1:${inspectorPort}`); // start Miniflare listening for a debugger to attach
 	}
 	return nodeOptions;
 }
 
-// Caching of the `npx-import`ed `@miniflare/tre` package
-// eslint-disable-next-line @typescript-eslint/consistent-type-imports
-let Miniflare: typeof import("@miniflare/tre").Miniflare;
+export interface SetupMiniflare3Options {
+	// Regular Miniflare 2 options to transform
+	miniflare2Options: MiniflareOptions;
+	// Miniflare 3 requires all modules to be manually specified
+	format: CfScriptFormat;
+	bundle: EsbuildBundle;
+
+	// Miniflare's logger
+	log: Miniflare3LogType;
+
+	enablePagesAssetsServiceBinding?: EnablePagesAssetsServiceBindingOptions;
+
+	// Miniflare 3 accepts namespace/bucket names in addition to binding names.
+	// This means multiple workers persisting to the same location can have
+	// different binding names for the same namespace/bucket. Therefore, we need
+	// the full KV/R2 arrays. This is also required for remote KV storage, as
+	// we need actual namespace IDs to connect to.
+	kvNamespaces: CfKvNamespace[] | undefined;
+	r2Buckets: CfR2Bucket[] | undefined;
+	d1Databases: CfD1Database[] | undefined;
+
+	// Account ID to use for authenticated Cloudflare fetch. If true, prompt
+	// user for ID if multiple available.
+	authenticatedAccountId: string | true | undefined;
+	// Whether to read/write from/to real KV namespaces
+	kvRemote: boolean | undefined;
+
+	// Port to start DevTools inspector server on
+	inspectorPort: number;
+}
+
+export async function buildMiniflare3Logger(): Promise<Miniflare3LogType> {
+	const { Log, NoOpLog, LogLevel } = await getMiniflare3();
 
-function arrayToObject(values: string[] = []): Record<string, string> {
-	return Object.fromEntries(values.map((value) => [value, value]));
+	let level = logger.loggerLevel.toUpperCase() as Uppercase<LoggerLevel>;
+	if (level === "LOG") level = "INFO";
+	const logLevel = LogLevel[level];
+
+	return logLevel === LogLevel.NONE ? new NoOpLog() : new Log(logLevel);
 }
 
-export async function setupExperimentalLocal(
-	mf2Options: MiniflareOptions,
-	format: CfScriptFormat,
-	bundle: EsbuildBundle
-): Promise<Miniflare3Type> {
+export async function transformMf2OptionsToMf3Options({
+	miniflare2Options,
+	format,
+	bundle,
+	log,
+	enablePagesAssetsServiceBinding,
+	kvNamespaces,
+	r2Buckets,
+	d1Databases,
+	authenticatedAccountId,
+	kvRemote,
+	inspectorPort,
+}: SetupMiniflare3Options): Promise<Miniflare3Options> {
+	// Build authenticated Cloudflare API fetch function if required
+	let cloudflareFetch: CloudflareFetch | undefined;
+	if (kvRemote && authenticatedAccountId !== undefined) {
+		const { Response: Miniflare3Response } = await getMiniflare3();
+		const preferredAccountId =
+			authenticatedAccountId === true ? undefined : authenticatedAccountId;
+		const accountId = await requireAuth({ account_id: preferredAccountId });
+		cloudflareFetch = async (resource, searchParams, init) => {
+			resource = `/accounts/${accountId}/${resource}`;
+			const response = await performApiFetch(resource, init, searchParams);
+			return new Miniflare3Response(response.body, response);
+		};
+	}
+
 	const options: Miniflare3Options = {
-		...mf2Options,
+		...miniflare2Options,
 		// Miniflare 3 distinguishes between binding name and namespace/bucket IDs.
-		// For now, just use the same value as we did in Miniflare 2.
-		// TODO: use defined KV preview ID if any
-		kvNamespaces: arrayToObject(mf2Options.kvNamespaces),
-		r2Buckets: arrayToObject(mf2Options.r2Buckets),
+		kvNamespaces: Object.fromEntries(
+			kvNamespaces?.map(({ binding, id }) => [binding, id]) ?? []
+		),
+		r2Buckets: Object.fromEntries(
+			r2Buckets?.map(({ binding, bucket_name }) => [binding, bucket_name]) ?? []
+		),
+		d1Databases: Object.fromEntries(
+			d1Databases?.map(({ binding, database_id, preview_database_id }) => [
+				binding,
+				preview_database_id ?? database_id,
+			]) ?? []
+		),
+		inspectorPort,
+		verbose: true,
+		cloudflareFetch,
+		log,
 	};
 
+	if (enablePagesAssetsServiceBinding !== undefined) {
+		// `../miniflare-cli/assets` dynamically imports`@cloudflare/pages-shared/environment-polyfills`.
+		// `@cloudflare/pages-shared/environment-polyfills/types.ts` defines `global`
+		// augmentations that pollute the `import`-site's typing environment.
+		//
+		// We `require` instead of `import`ing here to avoid polluting the main
+		// `wrangler` TypeScript project with the `global` augmentations. This
+		// relies on the fact that `require` is untyped.
+		//
+		// eslint-disable-next-line @typescript-eslint/no-var-requires
+		const generateASSETSBinding = require("../miniflare-cli/assets").default;
+		options.serviceBindings = {
+			...options.serviceBindings,
+			ASSETS: (await generateASSETSBinding({
+				log,
+				...enablePagesAssetsServiceBinding,
+				tre: true,
+				// We can get rid of this `any` easily once we do experimental-local/tre by default
+				// eslint-disable-next-line @typescript-eslint/no-explicit-any
+			})) as any,
+		};
+	}
+
 	if (format === "modules") {
 		// Manually specify all modules from the bundle. If we didn't do this,
 		// Miniflare 3 would try collect them automatically again itself.
@@ -688,30 +873,48 @@ export async function setupExperimentalLocal(
 		const root = path.dirname(bundle.path);
 
 		assert.strictEqual(bundle.type, "esm");
+		// Required for source mapped paths to resolve correctly
+		options.modulesRoot = root;
 		options.modules = [
 			// Entrypoint
 			{
 				type: "ESModule",
-				path: path.relative(root, bundle.path),
+				path: bundle.path,
 				contents: await readFile(bundle.path, "utf-8"),
 			},
 			// Misc (WebAssembly, etc, ...)
 			...bundle.modules.map((module) => ({
 				type: ModuleTypeToRuleType[module.type ?? "esm"],
-				path: module.name,
+				path: path.resolve(root, module.name),
 				contents: module.content,
 			})),
 		];
 	}
 
-	logger.log("⎔ Starting an experimental local server...");
-
-	if (Miniflare === undefined) {
-		({ Miniflare } = await npxImport<
-			// eslint-disable-next-line @typescript-eslint/consistent-type-imports
-			typeof import("@miniflare/tre")
-		>("@miniflare/tre@next"));
+	if (kvRemote) {
+		// `kvPersist` is always assigned a truthy value in `setupMiniflareOptions`
+		assert(options.kvPersist);
+		const kvRemoteCache =
+			options.kvPersist === true
+				? // If storing in temporary directory, find this path from the bundle
+				  // output path
+				  path.join(path.dirname(bundle.path), ".mf", "kv-remote")
+				: // Otherwise, `kvPersist` looks like `.../kv`, so rewrite it to
+				  // `kv-remote` since the expected metadata format for remote storage
+				  // is different to local
+				  path.join(path.dirname(options.kvPersist), "kv-remote");
+		options.kvPersist = `remote:?cache=${encodeURIComponent(kvRemoteCache)}`;
 	}
 
-	return new Miniflare(options);
+	return options;
+}
+
+// Caching of the `npx-import`ed `@miniflare/tre` package
+// eslint-disable-next-line @typescript-eslint/consistent-type-imports
+let miniflare3Module: typeof import("@miniflare/tre");
+export async function getMiniflare3(): Promise<
+	// eslint-disable-next-line @typescript-eslint/consistent-type-imports
+	typeof import("@miniflare/tre")
+> {
+	return (miniflare3Module ??= await npxImport("@miniflare/tre@3.0.0-next.12"));
 }