wrangler 0.0.0-e30d1013 → 0.0.0-e311bbbf

package/src/dev/local.tsx

@@ -1,13 +1,15 @@
 import assert from "node:assert";
 import { fork } from "node:child_process";
 import { realpathSync } from "node:fs";
-import { readFile, writeFile } from "node:fs/promises";
+import { readFile } from "node:fs/promises";
 import path from "node:path";
 import chalk from "chalk";
 import getPort from "get-port";
 import { npxImport } from "npx-import";
 import { useState, useEffect, useRef } from "react";
 import onExit from "signal-exit";
+import { fetch } from "undici";
+import { performApiFetch } from "../cfetch/internal";
 import { registerWorker } from "../dev-registry";
 import useInspector from "../inspect";
 import { logger } from "../logger";
@@ -17,9 +19,11 @@ import {
 } from "../module-collection";
 import { getBasePath } from "../paths";
 import { waitForPortToBeAvailable } from "../proxy";
+import { requireAuth } from "../user";
 import type { Config } from "../config";
 import type { WorkerRegistry } from "../dev-registry";
-import type { EnablePagesAssetsServiceBindingOptions } from "../miniflare-cli";
+import type { LoggerLevel } from "../logger";
+import type { EnablePagesAssetsServiceBindingOptions } from "../miniflare-cli/types";
 import type { AssetPaths } from "../sites";
 import type {
 	CfWorkerInit,
@@ -38,6 +42,8 @@ import type { EsbuildBundle } from "./use-esbuild";
 import type {
 	Miniflare as Miniflare3Type,
 	MiniflareOptions as Miniflare3Options,
+	Log as Miniflare3LogType,
+	CloudflareFetch,
 } from "@miniflare/tre";
 import type { MiniflareOptions } from "miniflare";
 import type { ChildProcess } from "node:child_process";
@@ -52,8 +58,8 @@ export interface LocalProps {
 	bindings: CfWorkerInit["bindings"];
 	workerDefinitions: WorkerRegistry | undefined;
 	assetPaths: AssetPaths | undefined;
-	port: number;
-	ip: string;
+	initialPort: number;
+	initialIp: string;
 	rules: Config["rules"];
 	inspectorPort: number;
 	localPersistencePath: string | null;
@@ -64,18 +70,31 @@ export interface LocalProps {
 	localUpstream: string | undefined;
 	inspect: boolean;
 	onReady: ((ip: string, port: number) => void) | undefined;
-	logPrefix?: string;
 	enablePagesAssetsServiceBinding?: EnablePagesAssetsServiceBindingOptions;
 	testScheduled?: boolean;
-	experimentalLocal?: boolean;
+	experimentalLocal: boolean | undefined;
+	accountId: string | undefined; // Account ID? In local mode??? :exploding_head:
+	experimentalLocalRemoteKv: boolean | undefined;
 }
 
+type InspectorJSON = {
+	id: string;
+	title: string;
+	type: "node";
+	description: string;
+	webSocketDebuggerUrl: string;
+	devtoolsFrontendUrl: string;
+	devtoolsFrontendUrlCompat: string;
+	faviconUrl: string;
+	url: string;
+}[];
+
 export function Local(props: LocalProps) {
 	const { inspectorUrl } = useLocalWorker(props);
 	useInspector({
 		inspectorUrl,
 		port: props.inspectorPort,
-		logToTerminal: false,
+		logToTerminal: props.experimentalLocal ?? false,
 	});
 	return null;
 }
@@ -90,21 +109,21 @@ function useLocalWorker({
 	bindings,
 	workerDefinitions,
 	assetPaths,
-	port,
-	inspectorPort,
+	initialPort,
 	rules,
 	localPersistencePath,
 	liveReload,
-	ip,
+	initialIp,
 	crons,
 	queueConsumers,
 	localProtocol,
 	localUpstream,
 	inspect,
 	onReady,
-	logPrefix,
 	enablePagesAssetsServiceBinding,
 	experimentalLocal,
+	accountId,
+	experimentalLocalRemoteKv,
 }: LocalProps) {
 	// TODO: pass vars via command line
 	const local = useRef<ChildProcess>();
@@ -113,6 +132,11 @@ function useLocalWorker({
 	const removeExperimentalLocalSignalExitListener = useRef<() => void>();
 	const [inspectorUrl, setInspectorUrl] = useState<string | undefined>();
 
+	// Our inspector proxy server will be binding to `LocalProps`'s `inspectorPort`.
+	// If we attempted to bind Node.js/workerd to the same inspector port, we'd get a port already in use error.
+	// Therefore, generate a new random port for our runtime's to bind their inspector service to.
+	const runtimeInspectorPortRef = useRef<number>();
+
 	useEffect(() => {
 		if (bindings.services && bindings.services.length > 0) {
 			logger.warn(
@@ -138,22 +162,6 @@ function useLocalWorker({
 		async function startLocalWorker() {
 			if (!bundle || !format) return;
 
-			// port for the worker
-			await waitForPortToBeAvailable(port, {
-				retryPeriod: 200,
-				timeout: 2000,
-				abortSignal: abortController.signal,
-			});
-
-			// In local mode, we want to copy all referenced modules into
-			// the output bundle directory before starting up
-			for (const module of bundle.modules) {
-				await writeFile(
-					path.join(path.dirname(bundle.path), module.name),
-					module.content
-				);
-			}
-
 			const scriptPath = realpathSync(bundle.path);
 
 			const upstream =
@@ -176,12 +184,15 @@ function useLocalWorker({
 				bundle,
 			});
 
+			runtimeInspectorPortRef.current ??= await getPort();
+			const runtimeInspectorPort = runtimeInspectorPortRef.current;
+
 			const { forkOptions, miniflareCLIPath, options } = setupMiniflareOptions({
 				workerName,
-				port,
+				port: initialPort,
 				scriptPath,
 				localProtocol,
-				ip,
+				ip: initialIp,
 				format,
 				rules,
 				compatibilityDate,
@@ -203,23 +214,37 @@ function useLocalWorker({
 				dataBlobBindings,
 				crons,
 				upstream,
-				logPrefix,
 				workerDefinitions,
 				enablePagesAssetsServiceBinding,
 			});
 
 			if (experimentalLocal) {
-				const mf3Options = await transformLocalOptions(options, format, bundle);
+				const log = await buildMiniflare3Logger();
+				const mf3Options = await transformMf2OptionsToMf3Options({
+					miniflare2Options: options,
+					format,
+					bundle,
+					log,
+					enablePagesAssetsServiceBinding,
+					kvNamespaces: bindings?.kv_namespaces,
+					r2Buckets: bindings?.r2_buckets,
+					d1Databases: bindings?.d1_databases,
+					authenticatedAccountId: accountId,
+					kvRemote: experimentalLocalRemoteKv,
+					inspectorPort: runtimeInspectorPort,
+				});
+
 				const current = experimentalLocalRef.current;
+
 				if (current === undefined) {
 					// If we don't have an active Miniflare instance, create a new one
-					const Miniflare = await getMiniflare3Constructor();
+					const { Miniflare } = await getMiniflare3();
 					if (abortController.signal.aborted) return;
 					const mf = new Miniflare(mf3Options);
 					experimentalLocalRef.current = mf;
 					removeExperimentalLocalSignalExitListener.current = onExit(() => {
 						logger.log("⎔ Shutting down experimental local server.");
-						mf.dispose();
+						void mf.dispose();
 						experimentalLocalRef.current = undefined;
 					});
 					await mf.ready;
@@ -230,19 +255,61 @@ function useLocalWorker({
 					logger.log("⎔ Reloading experimental local server.");
 					await current.setOptions(mf3Options);
 				}
+
+				try {
+					// fetch the inspector JSON response from the DevTools Inspector protocol
+					const inspectorJSONArr = (await (
+						await fetch(`http://127.0.0.1:${runtimeInspectorPort}/json`)
+					).json()) as InspectorJSON;
+
+					const foundInspectorURL = inspectorJSONArr?.find((inspectorJSON) =>
+						inspectorJSON.id.startsWith("core:user")
+					)?.webSocketDebuggerUrl;
+					if (foundInspectorURL === undefined) {
+						setInspectorUrl(undefined);
+					} else {
+						const url = new URL(foundInspectorURL);
+						// Force inspector URL to be different on each reload so `useEffect`
+						// in `useInspector` is re-run to connect to newly restarted
+						// `workerd` server when updating options. Can't use a query param
+						// here as that seems to cause an infinite connection loop, can't
+						// use a hash as those are forbidden by `ws`, so username it is.
+						url.username = `${Date.now()}-${Math.floor(
+							Math.random() * Number.MAX_SAFE_INTEGER
+						)}`;
+						setInspectorUrl(url.toString());
+					}
+				} catch (error: unknown) {
+					logger.error("Error attempting to retrieve Debugger URL:", error);
+				}
+
 				return;
 			}
 
-			const nodeOptions = setupNodeOptions({ inspect, ip, inspectorPort });
+			// Wait for the Worker port to be available. We don't want to do this in experimental local
+			// mode, as we only `dispose()` the Miniflare 3 instance, and shutdown the server when
+			// unmounting the component, not when props change. If we did, we'd just timeout every time.
+			await waitForPortToBeAvailable(initialPort, {
+				retryPeriod: 200,
+				timeout: 2000,
+				abortSignal: abortController.signal,
+			});
+
+			const nodeOptions = setupNodeOptions({
+				inspect,
+				inspectorPort: runtimeInspectorPort,
+			});
 			logger.log("⎔ Starting a local server...");
 
+			const hasColourSupport =
+				chalk.supportsColor.hasBasic && process.env.FORCE_COLOR !== "0";
 			const child = (local.current = fork(miniflareCLIPath, forkOptions, {
 				cwd: path.dirname(scriptPath),
 				execArgv: nodeOptions,
 				stdio: "pipe",
 				env: {
 					...process.env,
-					FORCE_COLOR: chalk.supportsColor.hasBasic ? "1" : undefined,
+					FORCE_COLOR: hasColourSupport ? "1" : undefined,
 				},
 			}));
 
@@ -254,21 +321,21 @@ function useLocalWorker({
 						await registerWorker(workerName, {
 							protocol: localProtocol,
 							mode: "local",
-							port,
-							host: ip,
+							port: message.port,
+							host: initialIp,
 							durableObjects: internalDurableObjects.map((binding) => ({
 								name: binding.name,
 								className: binding.class_name,
 							})),
 							...(message.durableObjectsPort
 								? {
-										durableObjectsHost: ip,
+										durableObjectsHost: initialIp,
 										durableObjectsPort: message.durableObjectsPort,
 								  }
 								: {}),
 						});
 					}
-					onReady?.(ip, message.mfPort);
+					onReady?.(initialIp, message.port);
 				}
 			});
 
@@ -320,7 +387,13 @@ function useLocalWorker({
 		}
 
 		startLocalWorker().catch((err) => {
-			logger.error("local worker:", err);
+			if (err.code === "ERR_RUNTIME_FAILURE") {
+				// Don't log a full verbose stack-trace when Miniflare 3's workerd instance fails to start.
+				// workerd will log its own errors, and our stack trace won't have any useful information.
+				logger.error(err.message);
+			} else {
+				logger.error("local worker:", err);
+			}
 		});
 
 		return () => {
@@ -337,9 +410,8 @@ function useLocalWorker({
 		bundle,
 		workerName,
 		format,
-		port,
-		inspectorPort,
-		ip,
+		initialPort,
+		initialIp,
 		queueConsumers,
 		bindings.queues,
 		bindings.durable_objects,
@@ -363,10 +435,11 @@ function useLocalWorker({
 		localProtocol,
 		localUpstream,
 		inspect,
-		logPrefix,
 		onReady,
 		enablePagesAssetsServiceBinding,
 		experimentalLocal,
+		accountId,
+		experimentalLocalRemoteKv,
 	]);
 
 	// Rather than disposing the Miniflare instance on every reload, only dispose
@@ -500,7 +573,6 @@ interface SetupMiniflareOptionsProps {
 	dataBlobBindings: Record<string, string>;
 	crons: Config["triggers"]["crons"];
 	upstream: string | undefined;
-	logPrefix: string | undefined;
 	workerDefinitions: WorkerRegistry | undefined;
 	enablePagesAssetsServiceBinding?: EnablePagesAssetsServiceBindingOptions;
 }
@@ -532,7 +604,6 @@ export function setupMiniflareOptions({
 	dataBlobBindings,
 	crons,
 	upstream,
-	logPrefix,
 	workerDefinitions,
 	enablePagesAssetsServiceBinding,
 }: SetupMiniflareOptionsProps): {
@@ -645,7 +716,6 @@ export function setupMiniflareOptions({
 		crons,
 		upstream,
 		logLevel: logger.loggerLevel,
-		logOptions: logPrefix ? { prefix: logPrefix } : undefined,
 		enablePagesAssetsServiceBinding,
 	};
 	// The path to the Miniflare CLI assumes that this file is being run from
@@ -665,11 +735,9 @@ export function setupMiniflareOptions({
 
 export function setupNodeOptions({
 	inspect,
-	ip,
 	inspectorPort,
 }: {
 	inspect: boolean;
-	ip: string;
 	inspectorPort: number;
 }) {
 	const nodeOptions = [
@@ -678,31 +746,123 @@ export function setupNodeOptions({
 		// "--log=VERBOSE", // uncomment this to Miniflare to log "everything"!
 	];
 	if (inspect) {
-		nodeOptions.push("--inspect=" + `${ip}:${inspectorPort}`); // start Miniflare listening for a debugger to attach
+		nodeOptions.push("--inspect=" + `127.0.0.1:${inspectorPort}`); // start Miniflare listening for a debugger to attach
 	}
 	return nodeOptions;
 }
 
-function arrayToObject(values: string[] = []): Record<string, string> {
-	return Object.fromEntries(values.map((value) => [value, value]));
+export interface SetupMiniflare3Options {
+	// Regular Miniflare 2 options to transform
+	miniflare2Options: MiniflareOptions;
+	// Miniflare 3 requires all modules to be manually specified
+	format: CfScriptFormat;
+	bundle: EsbuildBundle;
+
+	// Miniflare's logger
+	log: Miniflare3LogType;
+
+	enablePagesAssetsServiceBinding?: EnablePagesAssetsServiceBindingOptions;
+
+	// Miniflare 3 accepts namespace/bucket names in addition to binding names.
+	// This means multiple workers persisting to the same location can have
+	// different binding names for the same namespace/bucket. Therefore, we need
+	// the full KV/R2 arrays. This is also required for remote KV storage, as
+	// we need actual namespace IDs to connect to.
+	kvNamespaces: CfKvNamespace[] | undefined;
+	r2Buckets: CfR2Bucket[] | undefined;
+	d1Databases: CfD1Database[] | undefined;
+
+	// Account ID to use for authenticated Cloudflare fetch. If true, prompt
+	// user for ID if multiple available.
+	authenticatedAccountId: string | true | undefined;
+	// Whether to read/write from/to real KV namespaces
+	kvRemote: boolean | undefined;
+
+	// Port to start DevTools inspector server on
+	inspectorPort: number;
 }
 
-export async function transformLocalOptions(
-	mf2Options: MiniflareOptions,
-	format: CfScriptFormat,
-	bundle: EsbuildBundle
-): Promise<Miniflare3Options> {
+export async function buildMiniflare3Logger(): Promise<Miniflare3LogType> {
+	const { Log, NoOpLog, LogLevel } = await getMiniflare3();
+
+	let level = logger.loggerLevel.toUpperCase() as Uppercase<LoggerLevel>;
+	if (level === "LOG") level = "INFO";
+	const logLevel = LogLevel[level];
+
+	return logLevel === LogLevel.NONE ? new NoOpLog() : new Log(logLevel);
+}
+
+export async function transformMf2OptionsToMf3Options({
+	miniflare2Options,
+	format,
+	bundle,
+	log,
+	enablePagesAssetsServiceBinding,
+	kvNamespaces,
+	r2Buckets,
+	d1Databases,
+	authenticatedAccountId,
+	kvRemote,
+	inspectorPort,
+}: SetupMiniflare3Options): Promise<Miniflare3Options> {
+	// Build authenticated Cloudflare API fetch function if required
+	let cloudflareFetch: CloudflareFetch | undefined;
+	if (kvRemote && authenticatedAccountId !== undefined) {
+		const { Response: Miniflare3Response } = await getMiniflare3();
+		const preferredAccountId =
+			authenticatedAccountId === true ? undefined : authenticatedAccountId;
+		const accountId = await requireAuth({ account_id: preferredAccountId });
+		cloudflareFetch = async (resource, searchParams, init) => {
+			resource = `/accounts/${accountId}/${resource}`;
+			const response = await performApiFetch(resource, init, searchParams);
+			return new Miniflare3Response(response.body, response);
+		};
+	}
+
 	const options: Miniflare3Options = {
-		...mf2Options,
+		...miniflare2Options,
 		// Miniflare 3 distinguishes between binding name and namespace/bucket IDs.
-		// For now, just use the same value as we did in Miniflare 2.
-		// TODO: use defined KV preview ID if any
-		kvNamespaces: arrayToObject(mf2Options.kvNamespaces),
-		r2Buckets: arrayToObject(mf2Options.r2Buckets),
-		inspectorPort: await getPort({ port: 9229 }),
+		kvNamespaces: Object.fromEntries(
+			kvNamespaces?.map(({ binding, id }) => [binding, id]) ?? []
+		),
+		r2Buckets: Object.fromEntries(
+			r2Buckets?.map(({ binding, bucket_name }) => [binding, bucket_name]) ?? []
+		),
+		d1Databases: Object.fromEntries(
+			d1Databases?.map(({ binding, database_id, preview_database_id }) => [
+				binding,
+				preview_database_id ?? database_id,
+			]) ?? []
+		),
+		inspectorPort,
 		verbose: true,
+		cloudflareFetch,
+		log,
 	};
 
+	if (enablePagesAssetsServiceBinding !== undefined) {
+		// `../miniflare-cli/assets` dynamically imports`@cloudflare/pages-shared/environment-polyfills`.
+		// `@cloudflare/pages-shared/environment-polyfills/types.ts` defines `global`
+		// augmentations that pollute the `import`-site's typing environment.
+		//
+		// We `require` instead of `import`ing here to avoid polluting the main
+		// `wrangler` TypeScript project with the `global` augmentations. This
+		// relies on the fact that `require` is untyped.
+		//
+		// eslint-disable-next-line @typescript-eslint/no-var-requires
+		const generateASSETSBinding = require("../miniflare-cli/assets").default;
+		options.serviceBindings = {
+			...options.serviceBindings,
+			ASSETS: (await generateASSETSBinding({
+				log,
+				...enablePagesAssetsServiceBinding,
+				tre: true,
+				// We can get rid of this `any` easily once we do experimental-local/tre by default
+				// eslint-disable-next-line @typescript-eslint/no-explicit-any
+			})) as any,
+		};
+	}
+
 	if (format === "modules") {
 		// Manually specify all modules from the bundle. If we didn't do this,
 		// Miniflare 3 would try collect them automatically again itself.
@@ -713,34 +873,48 @@ export async function transformLocalOptions(
 		const root = path.dirname(bundle.path);
 
 		assert.strictEqual(bundle.type, "esm");
+		// Required for source mapped paths to resolve correctly
+		options.modulesRoot = root;
 		options.modules = [
 			// Entrypoint
 			{
 				type: "ESModule",
-				path: path.relative(root, bundle.path),
+				path: bundle.path,
 				contents: await readFile(bundle.path, "utf-8"),
 			},
 			// Misc (WebAssembly, etc, ...)
 			...bundle.modules.map((module) => ({
 				type: ModuleTypeToRuleType[module.type ?? "esm"],
-				path: module.name,
+				path: path.resolve(root, module.name),
 				contents: module.content,
 			})),
 		];
 	}
 
+	if (kvRemote) {
+		// `kvPersist` is always assigned a truthy value in `setupMiniflareOptions`
+		assert(options.kvPersist);
+		const kvRemoteCache =
+			options.kvPersist === true
+				? // If storing in temporary directory, find this path from the bundle
+				  // output path
+				  path.join(path.dirname(bundle.path), ".mf", "kv-remote")
+				: // Otherwise, `kvPersist` looks like `.../kv`, so rewrite it to
+				  // `kv-remote` since the expected metadata format for remote storage
+				  // is different to local
+				  path.join(path.dirname(options.kvPersist), "kv-remote");
+		options.kvPersist = `remote:?cache=${encodeURIComponent(kvRemoteCache)}`;
+	}
+
 	return options;
 }
 
 // Caching of the `npx-import`ed `@miniflare/tre` package
 // eslint-disable-next-line @typescript-eslint/consistent-type-imports
-let Miniflare: typeof import("@miniflare/tre").Miniflare;
-export async function getMiniflare3Constructor(): Promise<typeof Miniflare> {
-	if (Miniflare === undefined) {
-		({ Miniflare } = await npxImport<
-			// eslint-disable-next-line @typescript-eslint/consistent-type-imports
-			typeof import("@miniflare/tre")
-		>("@miniflare/tre@3.0.0-next.5"));
-	}
-	return Miniflare;
+let miniflare3Module: typeof import("@miniflare/tre");
+export async function getMiniflare3(): Promise<
+	// eslint-disable-next-line @typescript-eslint/consistent-type-imports
+	typeof import("@miniflare/tre")
+> {
+	return (miniflare3Module ??= await npxImport("@miniflare/tre@3.0.0-next.12"));
 }

package/src/dev/remote.tsx

@@ -1,5 +1,7 @@
 import { readFile } from "node:fs/promises";
 import path from "node:path";
+import { Text } from "ink";
+import SelectInput from "ink-select-input";
 import React, { useState, useEffect, useRef } from "react";
 import { useErrorHandler } from "react-error-boundary";
 import { printBundleSize } from "../bundle-reporter";
@@ -10,9 +12,9 @@ import {
 import useInspector from "../inspect";
 import { logger } from "../logger";
 import { startPreviewServer, usePreviewServer } from "../proxy";
+import { helpIfErrorIsSizeOrScriptStartup } from "../publish/publish";
 import { syncAssets } from "../sites";
 import {
-	ChooseAccount,
 	getAccountChoices,
 	requireApiToken,
 	saveAccountToCache,
@@ -302,13 +304,13 @@ export function useWorker(
 			// TODO: Once we get service bindings working in the
 			// edge preview server, we can define remote dev service bindings
 			// and you can uncomment this code.
-			// https://github.com/cloudflare/wrangler2/issues/1182
+			// https://github.com/cloudflare/workers-sdk/issues/1182
 
 			/*
 			if (name) {
 				await registerWorker(name, {
 					mode: "remote",
-					// upstream protocol is always https (https://github.com/cloudflare/wrangler2/issues/583)
+					// upstream protocol is always https (https://github.com/cloudflare/workers-sdk/issues/583)
 					protocol: "https",
 					port: undefined,
 					host: workerPreviewToken.host,
@@ -493,6 +495,7 @@ export async function getRemotePreviewToken(props: RemoteProps) {
 				// code 10049 happens when the preview token expires
 				logger.log("Preview token expired, restart server to fetch a new one");
 			} else {
+				helpIfErrorIsSizeOrScriptStartup(err, props.bundle?.dependencies || {});
 				logger.error("Error on remote worker:", err);
 			}
 		}
@@ -601,3 +604,29 @@ function getWorkerAccountAndContext(props: {
 
 	return { workerAccount, workerContext };
 }
+
+/**
+ * A component that allows the user to select from a list of available accounts.
+ */
+function ChooseAccount(props: {
+	accounts: ChooseAccountItem[];
+	onSelect: (account: { name: string; id: string }) => void;
+	onError: (error: Error) => void;
+}) {
+	return (
+		<>
+			<Text bold>Select an account from below:</Text>
+			<SelectInput
+				items={props.accounts.map((item) => ({
+					key: item.id,
+					label: item.name,
+					value: item,
+				}))}
+				onSelect={(item) => {
+					logger.log(`Using account: "${item.value.name} - ${item.value.id}"`);
+					props.onSelect({ id: item.value.id, name: item.value.name });
+				}}
+			/>
+		</>
+	);
+}