wrangler 2.0.16 → 2.0.19

package/src/metrics/metrics-config.ts

@@ -0,0 +1,222 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fetchResult } from "../cfetch";
+import { getConfigCache, saveToConfigCache } from "../config-cache";
+import { confirm } from "../dialogs";
+import isInteractive from "../is-interactive";
+import { logger } from "../logger";
+import { getAPIToken } from "../user";
+
+/**
+ * The date that the metrics being gathered was last updated in a way that would require
+ * the user to give their permission again.
+ *
+ * When reading from a config file, we check the recorded date in the config file against
+ * this one here. We ignore the permissions set in the the file if the recorded date is older.
+ * This allows us to prompt the user to re-opt-in when we make substantive changes to our metrics
+ * gathering.
+ */
+export const CURRENT_METRICS_DATE = new Date(2022, 6, 4);
+export const USER_ID_CACHE_PATH = "user-id.json";
+
+export interface MetricsConfigOptions {
+	/**
+	 * Defines whether to send metrics to Cloudflare:
+	 * If defined, then use this value for whether the dispatch is enabled.
+	 * Otherwise, infer the enabled value from the user configuration.
+	 */
+	sendMetrics?: boolean;
+	/**
+	 * When true, do not make any CF API requests.
+	 */
+	offline?: boolean;
+}
+
+/**
+ * The information needed to set up the MetricsDispatcher.
+ */
+export interface MetricsConfig {
+	/** True if usage tracking is enabled. */
+	enabled: boolean;
+	/** A UID that identifies this user and machine pair for Wrangler. */
+	deviceId: string;
+	/** The currently logged in user - undefined if not logged in. */
+	userId: string | undefined;
+}
+
+/**
+ * Get hold of the permissions and device-id for metrics dispatch.
+ *
+ * The device-id is just a unique identifier sent along with events to help
+ * to collate metrics. Once generated, this id is cached in the metrics config file.
+ *
+ * The permissions define whether we can send metrics or not. They can come from a variety of places:
+ * - the `send_metrics` setting in `wrangler.toml`
+ * - a cached response from the current user
+ * - prompting the user to opt-in to metrics
+ *
+ * If the user was prompted to opt-in, then their response is cached in the metrics config file.
+ *
+ * If the current process is not interactive then we will cannot prompt the user and just assume
+ * we cannot send metrics if there is no cached or project-level preference available.
+ */
+export async function getMetricsConfig({
+	sendMetrics,
+	offline = false,
+}: MetricsConfigOptions): Promise<MetricsConfig> {
+	const config = await readMetricsConfig();
+	const deviceId = await getDeviceId(config);
+	const userId = await getUserId(offline);
+
+	// If the project is explicitly set the `send_metrics` options in `wrangler.toml`
+	// then use that and ignore any user preference.
+	if (sendMetrics !== undefined) {
+		return { enabled: sendMetrics, deviceId, userId };
+	}
+
+	// Get the user preference from the metrics config.
+	const permission = config.permission;
+	if (permission !== undefined) {
+		if (new Date(permission.date) >= CURRENT_METRICS_DATE) {
+			return { enabled: permission.enabled, deviceId, userId };
+		} else if (permission.enabled) {
+			logger.log(
+				"Usage metrics tracking has changed since you last granted permission."
+			);
+		}
+	}
+
+	// We couldn't get the metrics permission from the project-level nor the user-level config.
+	// If we are not interactive then just bail out.
+	if (!isInteractive()) {
+		return { enabled: false, deviceId, userId };
+	}
+
+	// Otherwise, let's ask the user and store the result in the metrics config.
+	const enabled = await confirm(
+		"Would you like to help improve Wrangler by sending usage metrics to Cloudflare?"
+	);
+	logger.log(
+		`Your choice has been saved in the following file: ${path.relative(
+			process.cwd(),
+			getMetricsConfigPath()
+		)}.\n\n` +
+			"  You can override the user level setting for a project in `wrangler.toml`:\n\n" +
+			"   - to disable sending metrics for a project: `send_metrics = false`\n" +
+			"   - to enable sending metrics for a project: `send_metrics = true`"
+	);
+	await writeMetricsConfig({
+		permission: {
+			enabled,
+			date: CURRENT_METRICS_DATE,
+		},
+		deviceId,
+	});
+	return { enabled, deviceId, userId };
+}
+
+/**
+ * Stringify and write the given info to the metrics config file.
+ */
+export async function writeMetricsConfig(config: MetricsConfigFile) {
+	await mkdir(path.dirname(getMetricsConfigPath()), { recursive: true });
+	await writeFile(
+		getMetricsConfigPath(),
+		JSON.stringify(
+			config,
+			(_key, value) => (value instanceof Date ? value.toISOString() : value),
+			"\t"
+		)
+	);
+}
+
+/**
+ * Read and parse the metrics config file.
+ */
+export async function readMetricsConfig(): Promise<MetricsConfigFile> {
+	try {
+		return JSON.parse(
+			await readFile(getMetricsConfigPath(), "utf8"),
+			(key, value) => (key === "date" ? new Date(value) : value)
+		);
+	} catch {
+		return {};
+	}
+}
+
+/**
+ * Get the path to the metrics config file.
+ */
+function getMetricsConfigPath(): string {
+	return path.resolve(os.homedir(), ".wrangler/config/metrics.json");
+}
+
+/**
+ * The format of the metrics config file.
+ */
+export interface MetricsConfigFile {
+	permission?: {
+		/** True if Wrangler should send metrics to Cloudflare. */
+		enabled: boolean;
+		/** The date that this permission was set. */
+		date: Date;
+	};
+	/** A unique UUID that identifies this device for metrics purposes. */
+	deviceId?: string;
+}
+
+/**
+ * Returns an ID that uniquely identifies Wrangler on this device to help collate events.
+ *
+ * Once created this ID is stored in the metrics config file.
+ */
+async function getDeviceId(config: MetricsConfigFile) {
+	// Get or create the deviceId.
+	const deviceId = config.deviceId ?? randomUUID();
+	if (config.deviceId === undefined) {
+		// We had to create a new deviceID so store it now.
+		await writeMetricsConfig({ ...config, deviceId });
+	}
+	return deviceId;
+}
+
+/**
+ * Returns the ID of the current user, which will be sent with each event.
+ *
+ * The ID is retrieved from the CF API `/user` endpoint if the user is authenticated and then
+ * stored in the `node_modules/.cache`.
+ *
+ * If it is not possible to retrieve the ID (perhaps the user is not logged in) then we just use
+ * `undefined`.
+ */
+async function getUserId(offline: boolean) {
+	// Get the userId from the cache.
+	// If it has not been found in the cache and we are not offline then make an API call to get it.
+	// If we can't work in out then just use `anonymous`.
+	let userId = getConfigCache<{ userId: string }>(USER_ID_CACHE_PATH).userId;
+	if (userId === undefined && !offline) {
+		userId = await fetchUserId();
+		if (userId !== undefined) {
+			saveToConfigCache(USER_ID_CACHE_PATH, { userId });
+		}
+	}
+	return userId;
+}
+
+/**
+ * Ask the Cloudflare API for the User ID of the current user.
+ *
+ * We will only do this if we are not "offline", e.g. not running `wrangler dev --local`.
+ * Quietly return undefined if anything goes wrong.
+ */
+async function fetchUserId(): Promise<string | undefined> {
+	try {
+		return getAPIToken()
+			? (await fetchResult<{ id: string }>("/user")).id
+			: undefined;
+	} catch (e) {
+		return undefined;
+	}
+}

package/src/metrics/metrics-dispatcher.ts

@@ -0,0 +1,93 @@
+import { fetch } from "undici";
+import { version as wranglerVersion } from "../../package.json";
+import { logger } from "../logger";
+import { getMetricsConfig } from "./metrics-config";
+import type { MetricsConfigOptions } from "./metrics-config";
+
+// The SPARROW_SOURCE_KEY is provided at esbuild time as a `define` for production and beta
+// releases. Otherwise it is left undefined, which automatically disables metrics requests.
+declare const SPARROW_SOURCE_KEY: string;
+const SPARROW_URL = "https://sparrow.cloudflare.com";
+
+export async function getMetricsDispatcher(options: MetricsConfigOptions) {
+	return {
+		/**
+		 * Dispatch a event to the analytics target.
+		 *
+		 * The event should follow these conventions
+		 *  - name is of the form `[action] [object]` (lower case)
+		 *  - additional properties are camelCased
+		 */
+		async sendEvent(name: string, properties: Properties = {}): Promise<void> {
+			await dispatch({ type: "event", name, properties });
+		},
+
+		/**
+		 * Dispatch a user profile information to the analytics target.
+		 *
+		 * This call can be used to inform the analytics target of relevant properties associated
+		 * with the current user.
+		 */
+		async identify(properties: Properties): Promise<void> {
+			await dispatch({ type: "identify", name: "identify", properties });
+		},
+	};
+
+	async function dispatch(event: {
+		type: "identify" | "event";
+		name: string;
+		properties: Properties;
+	}): Promise<void> {
+		if (!SPARROW_SOURCE_KEY) {
+			logger.debug(
+				"Metrics dispatcher: Source Key not provided. Be sure to initialize before sending events.",
+				event
+			);
+			return;
+		}
+
+		// Lazily get the config for this dispatcher only when an event is being dispatched.
+		const metricsConfig = await getMetricsConfig(options);
+		if (!metricsConfig.enabled) {
+			logger.debug(
+				`Metrics dispatcher: Dispatching disabled - would have sent ${JSON.stringify(
+					event
+				)}.`
+			);
+			return;
+		}
+
+		try {
+			logger.debug(`Metrics dispatcher: Posting data ${JSON.stringify(event)}`);
+			const body = JSON.stringify({
+				deviceId: metricsConfig.deviceId,
+				userId: metricsConfig.userId,
+				event: event.name,
+				properties: {
+					category: "Workers",
+					wranglerVersion,
+					...event.properties,
+				},
+			});
+
+			await fetch(`${SPARROW_URL}/api/v1/${event.type}`, {
+				method: "POST",
+				headers: {
+					Accept: "*/*",
+					"Content-Type": "application/json",
+					"Sparrow-Source-Key": SPARROW_SOURCE_KEY,
+				},
+				mode: "cors",
+				keepalive: true,
+				body,
+			});
+		} catch (e) {
+			logger.debug(
+				"Metrics dispatcher: Failed to send request:",
+				(e as Error).message
+			);
+		}
+	}
+}
+
+export type Properties = Record<string, unknown>;

package/src/metrics/send-event.ts

@@ -0,0 +1,80 @@
+import { logger } from "../logger";
+import { getMetricsDispatcher } from "./metrics-dispatcher";
+import type { MetricsConfigOptions } from "./metrics-config";
+import type { Properties } from "./metrics-dispatcher";
+
+/** These are event names used by this wrangler client. */
+export type EventNames =
+	| "view accounts"
+	| "deploy worker script"
+	| "begin log stream"
+	| "end log stream"
+	| "create encrypted variable"
+	| "delete encrypted variable"
+	| "list encrypted variables"
+	| "create kv namespace"
+	| "list kv namespaces"
+	| "delete kv namespace"
+	| "write kv key-value"
+	| "list kv keys"
+	| "read kv value"
+	| "delete kv key-value"
+	| "write kv key-values (bulk)"
+	| "delete kv key-values (bulk)"
+	| "create r2 bucket"
+	| "list r2 buckets"
+	| "delete r2 bucket"
+	| "login user"
+	| "logout user"
+	| "create pubsub namespace"
+	| "list pubsub namespaces"
+	| "delete pubsub namespace"
+	| "view pubsub namespace"
+	| "create pubsub broker"
+	| "update pubsub broker"
+	| "list pubsub brokers"
+	| "delete pubsub broker"
+	| "view pubsub broker"
+	| "issue pubsub broker credentials"
+	| "revoke pubsub broker credentials"
+	| "unrevoke pubsub broker credentials"
+	| "list pubsub broker revoked credentials"
+	| "list pubsub broker public-keys"
+	| "list worker namespaces"
+	| "view worker namespace"
+	| "create worker namespace"
+	| "delete worker namespace"
+	| "rename worker namespace"
+	| "create pages project"
+	| "list pages projects"
+	| "deploy pages project"
+	| "list pages projects deployments"
+	| "build pages functions"
+	| "run dev"
+	| "run pages dev";
+
+export function sendMetricsEvent(event: EventNames): void;
+export function sendMetricsEvent(
+	event: EventNames,
+	options: MetricsConfigOptions
+): void;
+export function sendMetricsEvent(
+	event: EventNames,
+	properties: Properties,
+	options: MetricsConfigOptions
+): void;
+/**
+ * Send a metrics event to Cloudflare, if usage tracking is enabled.
+ */
+export function sendMetricsEvent(
+	event: EventNames,
+	...args: [] | [MetricsConfigOptions] | [Properties, MetricsConfigOptions]
+): void {
+	const options = args.pop() ?? {};
+	const properties = (args.pop() ?? {}) as Properties;
+	getMetricsDispatcher(options)
+		.then((metricsDispatcher) => metricsDispatcher.sendEvent(event, properties))
+		.catch((err) => {
+			logger.debug("Error sending metrics event", err);
+		});
+}

package/src/miniflare-cli/index.ts

@@ -41,6 +41,7 @@ async function main() {
 		// Start Miniflare development server
 		await mf.startServer();
 		await mf.startScheduler();
+		process.send && process.send("ready");
 	} catch (e) {
 		mf.log.error(e as Error);
 		process.exitCode = 1;

package/src/package-manager.ts

@@ -1,5 +1,6 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
+import { env } from "node:process";
 import { execa, execaCommandSync } from "execa";
 import { logger } from "./logger";
 
@@ -19,7 +20,9 @@ export async function getPackageManager(cwd: string): Promise<PackageManager> {
 	const hasYarnLock = existsSync(join(cwd, "yarn.lock"));
 	const hasNpmLock = existsSync(join(cwd, "package-lock.json"));
 	const hasPnpmLock = existsSync(join(cwd, "pnpm-lock.yaml"));
+	const userAgent = sniffUserAgent();
 
+	// check for lockfiles
 	if (hasNpmLock) {
 		if (hasNpm) {
 			logger.log(
@@ -60,6 +63,19 @@ export async function getPackageManager(cwd: string): Promise<PackageManager> {
 		}
 	}
 
+	// check the user agent
+	if (userAgent === "npm" && hasNpm) {
+		logger.log("Using npm as package manager.");
+		return { ...NpmPackageManager, cwd };
+	} else if (userAgent === "pnpm" && hasPnpm) {
+		logger.log("Using pnpm as package manager.");
+		return { ...PnpmPackageManager, cwd };
+	} else if (userAgent === "yarn" && hasYarn) {
+		logger.log("Using yarn as package manager.");
+		return { ...YarnPackageManager, cwd };
+	}
+
+	// lastly, check what's installed
 	if (hasNpm) {
 		logger.log("Using npm as package manager.");
 		return { ...NpmPackageManager, cwd };
@@ -172,3 +188,32 @@ function supportsNpm(): Promise<boolean> {
 function supportsPnpm(): Promise<boolean> {
 	return supports("pnpm");
 }
+
+/**
+ * The environment variable `npm_config_user_agent` can be used to
+ * guess the package manager that was used to execute wrangler.
+ * It's imperfect (just like regular user agent sniffing!)
+ * but the package managers we support all set this property:
+ *
+ * - [npm](https://github.com/npm/cli/blob/1415b4bdeeaabb6e0ba12b6b1b0cc56502bd64ab/lib/utils/config/definitions.js#L1945-L1979)
+ * - [pnpm](https://github.com/pnpm/pnpm/blob/cd4f9341e966eb8b411462b48ff0c0612e0a51a7/packages/plugin-commands-script-runners/src/makeEnv.ts#L14)
+ * - [yarn](https://yarnpkg.com/advanced/lifecycle-scripts#environment-variables)
+ */
+function sniffUserAgent(): "npm" | "pnpm" | "yarn" | undefined {
+	const userAgent = env.npm_config_user_agent;
+	if (userAgent === undefined) {
+		return undefined;
+	}
+
+	if (userAgent.includes("yarn")) {
+		return "yarn";
+	}
+
+	if (userAgent.includes("pnpm")) {
+		return "pnpm";
+	}
+
+	if (userAgent.includes("npm")) {
+		return "npm";
+	}
+}

package/src/pages/build.tsx

@@ -2,6 +2,7 @@ import { writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import { logger } from "../logger";
+import * as metrics from "../metrics";
 import { toUrlPath } from "../paths";
 import { isInPagesCI } from "./constants";
 import { buildPlugin } from "./functions/buildPlugin";
@@ -120,6 +121,7 @@ export const Handler = async ({
 		buildOutputDirectory,
 		nodeCompat,
 	});
+	metrics.sendMetricsEvent("build pages functions");
 };
 
 export async function buildFunctions({

package/src/pages/deployments.tsx

@@ -6,6 +6,7 @@ import { format as timeagoFormat } from "timeago.js";
 import { fetchResult } from "../cfetch";
 import { getConfigCache, saveToConfigCache } from "../config-cache";
 import { FatalError } from "../errors";
+import * as metrics from "../metrics";
 import { requireAuth } from "../user";
 import { PAGES_CONFIG_CACHE_FILENAME } from "./constants";
 import { listProjects } from "./projects";
@@ -98,4 +99,5 @@ export async function ListHandler({
 	});
 
 	render(<Table data={data}></Table>, { patchConsole: false });
+	metrics.sendMetricsEvent("list pages projects deployments");
 }

package/src/pages/dev.tsx

@@ -8,6 +8,7 @@ import { getType } from "mime";
 import { getVarsForDev } from "../dev/dev-vars";
 import { FatalError } from "../errors";
 import { logger } from "../logger";
+import * as metrics from "../metrics";
 import { getRequestContextCheckOptions } from "../miniflare-cli/request-context";
 import openInBrowser from "../open-in-browser";
 import { buildFunctions } from "./build";
@@ -172,6 +173,7 @@ export const Handler = async ({
 				buildOutputDirectory: directory,
 				nodeCompat,
 			});
+			metrics.sendMetricsEvent("build pages functions");
 		} catch {}
 
 		watch([functionsDirectory], {
@@ -187,6 +189,7 @@ export const Handler = async ({
 				buildOutputDirectory: directory,
 				nodeCompat,
 			});
+			metrics.sendMetricsEvent("build pages functions");
 		});
 
 		miniflareArgs = {
@@ -318,6 +321,7 @@ export const Handler = async ({
 		// `startServer` might throw if user code contains errors
 		const server = await miniflare.startServer();
 		logger.log(`Serving at http://localhost:${port}/`);
+		metrics.sendMetricsEvent("run pages dev");
 
 		if (process.env.BROWSER !== "none") {
 			await openInBrowser(`http://localhost:${port}/`);

package/src/pages/projects.tsx

@@ -8,6 +8,7 @@ import { getConfigCache, saveToConfigCache } from "../config-cache";
 import { prompt } from "../dialogs";
 import { FatalError } from "../errors";
 import { logger } from "../logger";
+import * as metrics from "../metrics";
 import { requireAuth } from "../user";
 import { PAGES_CONFIG_CACHE_FILENAME } from "./constants";
 import { pagesBetaWarning } from "./utils";
@@ -44,6 +45,7 @@ export async function ListHandler() {
 	});
 
 	render(<Table data={data}></Table>, { patchConsole: false });
+	metrics.sendMetricsEvent("list pages projects");
 }
 
 export const listProjects = async ({
@@ -154,4 +156,5 @@ export async function CreateHandler({
 	logger.log(
 		`To deploy a folder of assets, run 'wrangler pages publish [directory]'.`
 	);
+	metrics.sendMetricsEvent("create pages project");
 }

package/src/pages/publish.tsx

@@ -12,6 +12,7 @@ import { getConfigCache, saveToConfigCache } from "../config-cache";
 import { prompt } from "../dialogs";
 import { FatalError } from "../errors";
 import { logger } from "../logger";
+import * as metrics from "../metrics";
 import { requireAuth } from "../user";
 import { buildFunctions } from "./build";
 import { PAGES_CONFIG_CACHE_FILENAME } from "./constants";
@@ -193,6 +194,7 @@ export const Handler = async ({
 				});
 
 				logger.log(`✨ Successfully created the '${projectName}' project.`);
+				metrics.sendMetricsEvent("create pages project");
 				break;
 			}
 		}
@@ -332,4 +334,5 @@ export const Handler = async ({
 	logger.log(
 		`✨ Deployment complete! Take a peek over at ${deploymentResponse.url}`
 	);
+	metrics.sendMetricsEvent("deploy pages project");
 };

package/src/pages/upload.tsx

@@ -95,7 +95,7 @@ export const upload = async (
 	}
 
 	type FileContainer = {
-		content: string;
+		path: string;
 		contentType: string;
 		sizeInBytes: number;
 		hash: string;
@@ -112,6 +112,14 @@ export const upload = async (
 
 	const directory = resolve(args.directory);
 
+	// TODO(future): Use this to more efficiently load files in and speed up uploading
+	// Limit memory to 1 GB unless more is specified
+	// let maxMemory = 1_000_000_000;
+	// if (process.env.NODE_OPTIONS && (process.env.NODE_OPTIONS.includes('--max-old-space-size=') || process.env.NODE_OPTIONS.includes('--max_old_space_size='))) {
+	// 	const parsed = parser(process.env.NODE_OPTIONS);
+	// 	maxMemory = (parsed['max-old-space-size'] ? parsed['max-old-space-size'] : parsed['max_old_space_size']) * 1000 * 1000; // Turn MB into bytes
+	// }
+
 	const walk = async (
 		dir: string,
 		fileMap: Map<string, FileContainer> = new Map(),
@@ -137,7 +145,6 @@ export const upload = async (
 				} else {
 					const name = relative(startingDir, filepath).split(sep).join("/");
 
-					// TODO: Move this to later so we don't hold as much in memory
 					const fileContent = await readFile(filepath);
 
 					const base64Content = fileContent.toString("base64");
@@ -152,8 +159,9 @@ export const upload = async (
 						);
 					}
 
+					// We don't want to hold the content in memory. We instead only want to read it when it's needed
 					fileMap.set(name, {
-						content: base64Content,
+						path: filepath,
 						contentType: getType(name) || "application/octet-stream",
 						sizeInBytes: filestat.size,
 						hash: blake3hash(base64Content + extension)
@@ -248,17 +256,21 @@ export const upload = async (
 		// Don't upload empty buckets (can happen for tiny projects)
 		if (bucket.files.length === 0) continue;
 
-		const payload: UploadPayloadFile[] = bucket.files.map((file) => ({
-			key: file.hash,
-			value: file.content,
-			metadata: {
-				contentType: file.contentType,
-			},
-			base64: true,
-		}));
-
 		let attempts = 0;
 		const doUpload = async (): Promise<void> => {
+			// Populate the payload only when actually uploading (this is limited to 3 concurrent uploads at 50 MiB per bucket meaning we'd only load in a max of ~150 MiB)
+			// This is so we don't run out of memory trying to upload the files.
+			const payload: UploadPayloadFile[] = await Promise.all(
+				bucket.files.map(async (file) => ({
+					key: file.hash,
+					value: (await readFile(file.path)).toString("base64"),
+					metadata: {
+						contentType: file.contentType,
+					},
+					base64: true,
+				}))
+			);
+
 			try {
 				return await fetchResult(`/pages/assets/upload`, {
 					method: "POST",
@@ -270,9 +282,9 @@ export const upload = async (
 				});
 			} catch (e) {
 				if (attempts < MAX_UPLOAD_ATTEMPTS) {
-					// Linear backoff, 0 second first time, then 1 second etc.
+					// Exponential backoff, 1 second first time, then 2 second, then 4 second etc.
 					await new Promise((resolvePromise) =>
-						setTimeout(resolvePromise, attempts++ * 1000)
+						setTimeout(resolvePromise, Math.pow(2, attempts++) * 1000)
 					);
 
 					if ((e as { code: number }).code === 8000013) {