wrangler 0.0.0-e7a0cd5b → 0.0.0-e87ad068

package/src/{sites.tsx → sites.ts}

@@ -1,6 +1,7 @@
 import assert from "node:assert";
 import { readdir, readFile, stat } from "node:fs/promises";
 import * as path from "node:path";
+import chalk from "chalk";
 import ignore from "ignore";
 import xxhash from "xxhash-wasm";
 import {
@@ -9,8 +10,10 @@ import {
 	listKVNamespaces,
 	putKVBulkKeyValue,
 	deleteKVBulkKeyValue,
+	BATCH_KEY_MAX,
+	formatNumber,
 } from "./kv/helpers";
-import { logger } from "./logger";
+import { logger, LOGGER_LEVELS } from "./logger";
 import type { Config } from "./config";
 import type { KeyValue } from "./kv/helpers";
 import type { XXHashAPI } from "xxhash-wasm";
@@ -56,7 +59,7 @@ function hashFileContent(hasher: XXHashAPI, content: string): string {
  * Create a hashed asset key for the given asset.
  *
  * The key will change if the file path or content of the asset changes.
- * The algorithm used here matches that of Wrangler 1.
+ * The algorithm used here matches that of Wrangler v1.
  */
 function hashAsset(
 	hasher: XXHashAPI,
@@ -92,6 +95,15 @@ async function createKVNamespaceIfNotAlreadyExisting(
 	};
 }
 
+const MAX_DIFF_LINES = 100;
+const MAX_BUCKET_SIZE = 98 * 1000 * 1000;
+const MAX_BUCKET_KEYS = BATCH_KEY_MAX;
+const MAX_BATCH_OPERATIONS = 5;
+
+function pluralise(count: number) {
+	return count === 1 ? "" : "s";
+}
+
 /**
  * Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
  * the worker given by `scriptName`.
@@ -116,13 +128,13 @@ export async function syncAssets(
 	if (siteAssets === undefined) {
 		return { manifest: undefined, namespace: undefined };
 	}
-
 	if (dryRun) {
 		logger.log("(Note: doing a dry run, not uploading or deleting anything.)");
 		return { manifest: undefined, namespace: undefined };
 	}
 	assert(accountId, "Missing accountId");
 
+	// Create assets namespace if it doesn't exist
 	const title = `__${scriptName}-workers_sites_assets${
 		preview ? "_preview" : ""
 	}`;
@@ -131,55 +143,81 @@ export async function syncAssets(
 		title,
 		accountId
 	);
-
-	// let's get all the keys in this namespace
+	// Get all existing keys in asset namespace
+	logger.info("Fetching list of already uploaded assets...");
 	const namespaceKeysResponse = await listKVNamespaceKeys(accountId, namespace);
 	const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
 
-	const manifest: Record<string, string> = {};
-
-	// A batch of uploads where each bucket has to be less than 98mb
-	const uploadBuckets: KeyValue[][] = [];
-	// The "live" bucket that we'll keep filling until it's just below 98mb
-	let uploadBucket: KeyValue[] = [];
-	// A size counter for the live bucket
-	let uploadBucketSize = 0;
-
-	const include = createPatternMatcher(siteAssets.includePatterns, false);
-	const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
-	const hasher = await xxhash();
-
 	const assetDirectory = path.join(
 		siteAssets.baseDirectory,
 		siteAssets.assetDirectory
 	);
+	const include = createPatternMatcher(siteAssets.includePatterns, false);
+	const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
+	const hasher = await xxhash();
+
+	// Find and validate all assets before we make any changes (can't store base64
+	// contents in memory for upload as users may have *lots* of files, and we
+	// don't want to OOM: https://github.com/cloudflare/workers-sdk/issues/2223)
+
+	const manifest: Record<string, string> = {};
+	type PathKey = [path: string, key: string];
+	// A batch of uploads where each bucket has to be less than 100 MiB and
+	// contain less than 10,000 keys (although we limit to 98 MB and 5000 keys)
+	const uploadBuckets: PathKey[][] = [];
+	// The "live" bucket we'll keep filling until it's just below the size limit
+	let uploadBucket: PathKey[] = [];
+	// Current size of the live bucket in bytes (just base64 encoded values)
+	let uploadBucketSize = 0;
+
+	let uploadCount = 0;
+	let skipCount = 0;
+
+	// Always log the first MAX_DIFF_LINES lines, then require the debug log level
+	let diffCount = 0;
+	function logDiff(line: string) {
+		const level = logger.loggerLevel;
+		if (LOGGER_LEVELS[level] >= LOGGER_LEVELS.debug) {
+			// If we're logging as debug level, we want *all* diff lines to be logged
+			// at debug level, not just the first MAX_DIFF_LINES
+			logger.debug(line);
+		} else if (diffCount < MAX_DIFF_LINES) {
+			// Otherwise, log  the first MAX_DIFF_LINES diffs at info level...
+			logger.info(line);
+		} else if (diffCount === MAX_DIFF_LINES) {
+			// ...and warn when we start to truncate it
+			const msg =
+				"   (truncating changed assets log, set `WRANGLER_LOG=debug` environment variable to see full diff)";
+			logger.info(chalk.dim(msg));
+		}
+		diffCount++;
+	}
+
+	logger.info("Building list of assets to upload...");
 	for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
 		const assetFile = path.relative(assetDirectory, absAssetFile);
-		if (!include(assetFile)) {
-			continue;
-		}
-		if (exclude(assetFile)) {
-			continue;
-		}
+		if (!include(assetFile) || exclude(assetFile)) continue;
 
-		logger.log(`Reading ${assetFile}...`);
 		const content = await readFile(absAssetFile, "base64");
-		await validateAssetSize(absAssetFile, assetFile);
-		// while KV accepts files that are 25 MiB **before** b64 encoding
+		// While KV accepts files that are 25 MiB **before** b64 encoding
 		// the overall bucket size must be below 100 MB **after** b64 encoding
-		const assetSize = Buffer.from(content).length;
+		const assetSize = Buffer.byteLength(content);
+		await validateAssetSize(absAssetFile, assetFile);
 		const assetKey = hashAsset(hasher, assetFile, content);
 		validateAssetKey(assetKey);
 
-		// now put each of the files into kv
 		if (!namespaceKeys.has(assetKey)) {
-			logger.log(`Uploading as ${assetKey}...`);
-
-			// Check if adding this asset to the bucket would
-			// push it over the 98 MiB limit KV bulk API limit
-			if (uploadBucketSize + assetSize > 98 * 1000 * 1000) {
-				// If so, move the current bucket into the batch,
-				// and reset the counter/bucket
+			logDiff(
+				chalk.green(` + ${assetKey} (uploading new version of ${assetFile})`)
+			);
+
+			// Check if adding this asset to the bucket would push it over the KV
+			// bulk API limits
+			if (
+				uploadBucketSize + assetSize > MAX_BUCKET_SIZE ||
+				uploadBucket.length + 1 > MAX_BUCKET_KEYS
+			) {
+				// If so, record the current bucket and reset it
 				uploadBuckets.push(uploadBucket);
 				uploadBucketSize = 0;
 				uploadBucket = [];
@@ -187,13 +225,11 @@ export async function syncAssets(
 
 			// Update the bucket and the size counter
 			uploadBucketSize += assetSize;
-			uploadBucket.push({
-				key: assetKey,
-				value: content,
-				base64: true,
-			});
+			uploadBucket.push([absAssetFile, assetKey]);
+			uploadCount++;
 		} else {
-			logger.log(`Skipping - already uploaded.`);
+			logDiff(chalk.dim(` = ${assetKey} (already uploaded ${assetFile})`));
+			skipCount++;
 		}
 
 		// Remove the key from the set so we know what we've already uploaded
@@ -203,23 +239,99 @@ export async function syncAssets(
 		const manifestKey = urlSafe(path.relative(assetDirectory, absAssetFile));
 		manifest[manifestKey] = assetKey;
 	}
+	// Add the last (potentially only or empty) bucket to the batch
+	if (uploadBucket.length > 0) uploadBuckets.push(uploadBucket);
 
-	// Add the last (potentially only) bucket to the batch
-	uploadBuckets.push(uploadBucket);
-
-	// keys now contains all the files we're deleting
 	for (const key of namespaceKeys) {
-		logger.log(`Deleting ${key} from the asset store...`);
+		logDiff(chalk.red(` - ${key} (removing as stale)`));
+	}
+
+	// Upload new assets, with 5 concurrent uploaders
+	if (uploadCount > 0) {
+		const s = pluralise(uploadCount);
+		logger.info(`Uploading ${formatNumber(uploadCount)} new asset${s}...`);
+	}
+	if (skipCount > 0) {
+		const s = pluralise(skipCount);
+		logger.info(
+			`Skipped uploading ${formatNumber(skipCount)} existing asset${s}.`
+		);
 	}
+	let uploadedCount = 0;
+	const controller = new AbortController();
+	const uploaders = Array.from(Array(MAX_BATCH_OPERATIONS)).map(async () => {
+		while (!controller.signal.aborted) {
+			// Get the next bucket to upload. If there is none, stop this uploader.
+			// JavaScript is single(ish)-threaded, so we don't need to worry about
+			// parallel access here.
+			const nextBucket = uploadBuckets.shift();
+			if (nextBucket === undefined) break;
+
+			// Read all files in the bucket as base64
+			// TODO(perf): consider streaming the bulk upload body, rather than
+			//  buffering all base64 contents then JSON-stringifying. This probably
+			//  doesn't matter *too* much: we know buckets will be about 100MB, so
+			//  with 5 uploaders, we could load about 500MB into memory (+ extra
+			//  object keys/tags/copies/etc).
+			const bucket: KeyValue[] = [];
+			for (const [absAssetFile, assetKey] of nextBucket) {
+				bucket.push({
+					key: assetKey,
+					value: await readFile(absAssetFile, "base64"),
+					base64: true,
+				});
+				if (controller.signal.aborted) break;
+			}
 
-	// upload each bucket in parallel
-	const bucketsToPut = [];
-	for (const bucket of uploadBuckets) {
-		bucketsToPut.push(putKVBulkKeyValue(accountId, namespace, bucket));
+			// Upload the bucket to the KV namespace, suppressing logs, we do our own
+			try {
+				await putKVBulkKeyValue(
+					accountId,
+					namespace,
+					bucket,
+					/* quiet */ true,
+					controller.signal
+				);
+			} catch (e) {
+				// https://developer.mozilla.org/en-US/docs/Web/API/DOMException#error_names
+				// https://github.com/nodejs/undici/blob/a3efc9814447001a43a976f1c64adc41995df7e3/lib/core/errors.js#L89
+				if (
+					typeof e === "object" &&
+					e !== null &&
+					"name" in e &&
+					// @ts-expect-error `e.name` should be typed `unknown`, fixed in
+					//  TypeScript 4.9
+					e.name === "AbortError"
+				) {
+					break;
+				}
+				throw e;
+			}
+			uploadedCount += nextBucket.length;
+			const percent = Math.floor((100 * uploadedCount) / uploadCount);
+			logger.info(
+				`Uploaded ${percent}% [${formatNumber(
+					uploadedCount
+				)} out of ${formatNumber(uploadCount)}]`
+			);
+		}
+	});
+	try {
+		// Wait for all uploaders to complete, or one to fail
+		await Promise.all(uploaders);
+	} catch (e) {
+		// If any uploader fails, abort the others
+		logger.info(`Upload failed, aborting...`);
+		controller.abort();
+		throw e;
 	}
-	await Promise.all(bucketsToPut);
 
-	// then delete all the assets that aren't used anymore
+	// Delete stale assets
+	const deleteCount = namespaceKeys.size;
+	if (deleteCount > 0) {
+		const s = pluralise(deleteCount);
+		logger.info(`Removing ${formatNumber(deleteCount)} stale asset${s}...`);
+	}
 	await deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys));
 
 	logger.log("↗️  Done syncing assets");

package/src/tail/createTail.ts

@@ -7,6 +7,8 @@ export { translateCLICommandToFilterMessage } from "./filters";
 export { jsonPrintLogs, prettyPrintLogs } from "./printing";
 import type { Request } from "undici";
 
+const TRACE_VERSION = "trace-v1";
+
 /**
  * When creating a Tail, the response from the API contains
  * - an ID used for identifying the tail
@@ -59,6 +61,68 @@ function makeDeleteTailUrl(
 		: `/accounts/${accountId}/workers/scripts/${workerName}/tails/${tailId}`;
 }
 
+interface CreatePagesTailOptions {
+	accountId: string;
+	projectName: string;
+	deploymentId: string;
+	filters: TailFilterMessage;
+	debug?: boolean;
+}
+
+/**
+ * Create and connect to a Pages Function Tail.
+ *
+ * Under the hood, this function
+ * - Registers a new Tail with the API
+ * - Connects to the tail worker
+ * - Sends any filters over the connection
+ *
+ * @returns a websocket connection, an expiration, and a function to call to delete the tail
+ */
+export async function createPagesTail({
+	accountId,
+	projectName,
+	deploymentId,
+	filters,
+	debug = false,
+}: CreatePagesTailOptions) {
+	const tailRecord = await fetchResult<TailCreationApiResponse>(
+		`/accounts/${accountId}/pages/projects/${projectName}/deployments/${deploymentId}/tails`,
+		{
+			method: "POST",
+			body: JSON.stringify(filters),
+		}
+	);
+
+	const deleteTail = async () =>
+		fetchResult(
+			`/accounts/${accountId}/pages/projects/${projectName}/deployments/${deploymentId}/tails/${tailRecord.id}`,
+			{ method: "DELETE" }
+		);
+
+	const tail = new WebSocket(tailRecord.url, TRACE_VERSION, {
+		headers: {
+			"Sec-WebSocket-Protocol": TRACE_VERSION, // needs to be `trace-v1` to be accepted
+			"User-Agent": `wrangler-js/${packageVersion}`,
+		},
+	});
+
+	// send filters when we open up
+	tail.on("open", () => {
+		tail.send(
+			JSON.stringify({ debug: debug }),
+			{ binary: false, compress: false, mask: false, fin: true },
+			(err) => {
+				if (err) {
+					throw err;
+				}
+			}
+		);
+	});
+
+	return { tail, deleteTail, expiration: tailRecord.expires_at };
+}
+
 /**
  * Create and connect to a tail.
  *
@@ -102,9 +166,9 @@ export async function createTail(
 	}
 
 	// connect to the tail
-	const tail = new WebSocket(websocketUrl, "trace-v1", {
+	const tail = new WebSocket(websocketUrl, TRACE_VERSION, {
 		headers: {
-			"Sec-WebSocket-Protocol": "trace-v1", // needs to be `trace-v1` to be accepted
+			"Sec-WebSocket-Protocol": TRACE_VERSION, // needs to be `trace-v1` to be accepted
 			"User-Agent": `wrangler-js/${packageVersion}`,
 		},
 	});
@@ -178,11 +242,19 @@ export type TailEventMessage = {
 	 * The event that triggered the worker. In the case of an HTTP request,
 	 * this will be a RequestEvent. If it's a cron trigger, it'll be a
 	 * ScheduledEvent. If it's a durable object alarm, it's an AlarmEvent.
+	 * If it's a email, it'a an EmailEvent
 	 *
 	 * Until workers-types exposes individual types for export, we'll have
 	 * to just re-define these types ourselves.
 	 */
-	event: RequestEvent | ScheduledEvent | AlarmEvent | undefined | null;
+	event:
+		| RequestEvent
+		| ScheduledEvent
+		| AlarmEvent
+		| EmailEvent
+		| TailInfo
+		| undefined
+		| null;
 };
 
 /**
@@ -313,3 +385,31 @@ export type AlarmEvent = {
 	 */
 	scheduledTime: string;
 };
+
+/**
+ * An event that was triggered from an email
+ */
+export type EmailEvent = {
+	/**
+	 * Who sent the email
+	 */
+	mailFrom: string;
+
+	/**
+	 * Who was the email sent to
+	 */
+	rcptTo: string;
+
+	/**
+	 * Size of the email in bytes
+	 */
+	rawSize: number;
+};
+
+/**
+ * Message from tail with information about the tail itself
+ */
+export type TailInfo = {
+	message: string;
+	type: string;
+};

package/src/tail/filters.ts

@@ -7,7 +7,7 @@
 
 /**
  * These are the filters we accept in the CLI. They
- * were copied directly from wrangler 1 in order to
+ * were copied directly from Wrangler v1 in order to
  * maintain compatability, so they aren't actually the exact
  * filters we need to send up to the tail worker. They generally map 1:1,
  * but often require some transformation or

package/src/tail/index.ts

@@ -3,7 +3,7 @@ import onExit from "signal-exit";
 
 import { fetchResult, fetchScriptContent } from "../cfetch";
 import { readConfig } from "../config";
-import { tailDOLogPrompt } from "../dialogs";
+import { confirm } from "../dialogs";
 import {
 	isLegacyEnv,
 	printWranglerBanner,
@@ -20,16 +20,14 @@ import {
 	translateCLICommandToFilterMessage,
 } from "./createTail";
 import type { WorkerMetadata } from "../create-worker-upload-form";
-import type { ConfigPath } from "../index";
 import type {
-	CommonYargsOptions,
-	YargsOptionsToInterface,
+	CommonYargsArgv,
+	StrictYargsOptionsToInterface,
 } from "../yargs-types";
 import type { TailCLIFilters } from "./createTail";
 import type { RawData } from "ws";
-import type { Argv } from "yargs";
 
-export function tailOptions(yargs: Argv<CommonYargsOptions>) {
+export function tailOptions(yargs: CommonYargsArgv) {
 	return yargs
 		.positional("worker", {
 			describe: "Name or route of the worker to tail",
@@ -86,13 +84,13 @@ export function tailOptions(yargs: Argv<CommonYargsOptions>) {
 		});
 }
 
-type TailArgs = YargsOptionsToInterface<typeof tailOptions>;
+type TailArgs = StrictYargsOptionsToInterface<typeof tailOptions>;
 
 export async function tailHandler(args: TailArgs) {
 	if (args.format === "pretty") {
 		await printWranglerBanner();
 	}
-	const config = readConfig(args.config as ConfigPath, args);
+	const config = readConfig(args.config, args);
 	await metrics.sendMetricsEvent("begin log stream", {
 		sendMetrics: config.send_metrics,
 	});
@@ -121,7 +119,7 @@ export async function tailHandler(args: TailArgs) {
 		status: args.status as ("ok" | "error" | "canceled")[] | undefined,
 		header: args.header,
 		method: args.method,
-		samplingRate: args["sampling-rate"],
+		samplingRate: args.samplingRate,
 		search: args.search,
 		clientIp: args.ip,
 	};
@@ -144,8 +142,7 @@ export async function tailHandler(args: TailArgs) {
 			`Beginning log collection requires restarting the Durable Objects associated with ${scriptName}. Any WebSocket connections or other non-persisted state will be lost as part of this restart.`
 		);
 
-		const shouldContinue = await tailDOLogPrompt();
-		if (!shouldContinue) {
+		if (!(await confirm("Would you like to continue?"))) {
 			return;
 		}
 	}

package/src/tail/printing.ts

@@ -1,8 +1,11 @@
+import chalk from "chalk";
 import { logger } from "../logger";
 import type {
 	AlarmEvent,
+	EmailEvent,
 	RequestEvent,
 	ScheduledEvent,
+	TailInfo,
 	TailEventMessage,
 } from "./createTail";
 import type { Outcome } from "./filters";
@@ -30,6 +33,16 @@ export function prettyPrintLogs(data: WebSocket.RawData): void {
 				? `${requestMethod} ${url} - ${outcome} @ ${datetime}`
 				: `[missing request] - ${outcome} @ ${datetime}`
 		);
+	} else if (isEmailEvent(eventMessage.event)) {
+		const outcome = prettifyOutcome(eventMessage.outcome);
+		const datetime = new Date(eventMessage.eventTimestamp).toLocaleString();
+		const mailFrom = eventMessage.event.mailFrom;
+		const rcptTo = eventMessage.event.rcptTo;
+		const rawSize = eventMessage.event.rawSize;
+
+		logger.log(
+			`Email from:${mailFrom} to:${rcptTo} size:${rawSize} @ ${datetime} - ${outcome}`
+		);
 	} else if (isAlarmEvent(eventMessage.event)) {
 		const outcome = prettifyOutcome(eventMessage.outcome);
 		const datetime = new Date(
@@ -37,6 +50,12 @@ export function prettyPrintLogs(data: WebSocket.RawData): void {
 		).toLocaleString();
 
 		logger.log(`Alarm @ ${datetime} - ${outcome}`);
+	} else if (isTailInfo(eventMessage.event)) {
+		if (eventMessage.event.type === "overload") {
+			logger.log(`${chalk.red.bold(eventMessage.event.message)}`);
+		} else if (eventMessage.event.type === "overload-stop") {
+			logger.log(`${chalk.yellow.bold(eventMessage.event.message)}`);
+		}
 	} else {
 		// Unknown event type
 		const outcome = prettifyOutcome(eventMessage.outcome);
@@ -74,6 +93,10 @@ function isScheduledEvent(
 	return Boolean(event && "cron" in event);
 }
 
+function isEmailEvent(event: TailEventMessage["event"]): event is EmailEvent {
+	return Boolean(event && "mailFrom" in event);
+}
+
 /**
  * Check to see if an event sent from a worker is an AlarmEvent.
  *
@@ -88,6 +111,10 @@ function isAlarmEvent(event: TailEventMessage["event"]): event is AlarmEvent {
 	return Boolean(event && "scheduledTime" in event && !("cron" in event));
 }
 
+function isTailInfo(event: TailEventMessage["event"]): event is TailInfo {
+	return Boolean(event && "message" in event && "type" in event);
+}
+
 function prettifyOutcome(outcome: Outcome): string {
 	switch (outcome) {
 		case "ok":

package/src/traverse-module-graph.ts

@@ -0,0 +1,54 @@
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import chalk from "chalk";
+import { logger } from "./logger";
+import { matchFiles, parseRules } from "./module-collection";
+import type { BundleResult } from "./bundle";
+import type { Config } from "./config";
+import type { Entry } from "./entry";
+
+async function getFiles(root: string, relativeTo: string): Promise<string[]> {
+	const files = [];
+	for (const file of await readdir(root, { withFileTypes: true })) {
+		if (file.isDirectory()) {
+			files.push(...(await getFiles(path.join(root, file.name), relativeTo)));
+		} else {
+			files.push(path.relative(relativeTo, path.join(root, file.name)));
+		}
+	}
+	return files;
+}
+
+export default async function traverseModuleGraph(
+	entry: Entry,
+	rules: Config["rules"]
+): Promise<BundleResult> {
+	const files = await getFiles(entry.moduleRoot, entry.moduleRoot);
+	const relativeEntryPoint = path.relative(entry.moduleRoot, entry.file);
+
+	const modules = (await matchFiles(files, entry.moduleRoot, parseRules(rules)))
+		.filter((m) => m.name !== relativeEntryPoint)
+		.map((m) => ({
+			...m,
+			name: m.name,
+		}));
+
+	const bundleType = entry.format === "modules" ? "esm" : "commonjs";
+
+	if (modules.length > 0) {
+		logger.info(`Attaching additional modules:`);
+		modules.forEach(({ name, type }) => {
+			logger.info(`- ${chalk.blue(name)} (${chalk.green(type ?? "")})`);
+		});
+	}
+
+	return {
+		modules,
+		dependencies: {},
+		resolvedEntryPointPath: entry.file,
+		bundleType,
+		stop: undefined,
+		sourceMapPath: undefined,
+		sourceMapMetadata: undefined,
+	};
+}

package/src/tsconfig-sanity.ts

@@ -0,0 +1,16 @@
+// `@types/node` should be included
+Buffer.from("test");
+
+// @ts-expect-error `@types/jest` should NOT be included
+test("test");
+
+// @ts-expect-error `@cloudflare/workers-types` should NOT be included
+const _handler: ExportedHandler = {};
+// @ts-expect-error `@cloudflare/workers-types` should NOT be included
+new HTMLRewriter();
+
+// @ts-expect-error `fetch` should NOT be included as our minimum supported
+//  Node version is 16.13.0 which does not include `fetch` on the global scope
+void fetch("http://localhost/");
+
+export {};