wrangler 0.0.0-e33294b0 → 0.0.0-e351afcf

package/src/sites.ts

@@ -1,6 +1,7 @@
 import assert from "node:assert";
 import { readdir, readFile, stat } from "node:fs/promises";
 import * as path from "node:path";
+import chalk from "chalk";
 import ignore from "ignore";
 import xxhash from "xxhash-wasm";
 import {
@@ -9,8 +10,10 @@ import {
 	listKVNamespaces,
 	putKVBulkKeyValue,
 	deleteKVBulkKeyValue,
+	BATCH_KEY_MAX,
+	formatNumber,
 } from "./kv/helpers";
-import { logger } from "./logger";
+import { logger, LOGGER_LEVELS } from "./logger";
 import type { Config } from "./config";
 import type { KeyValue } from "./kv/helpers";
 import type { XXHashAPI } from "xxhash-wasm";
@@ -92,6 +95,15 @@ async function createKVNamespaceIfNotAlreadyExisting(
 	};
 }
 
+const MAX_DIFF_LINES = 100;
+const MAX_BUCKET_SIZE = 98 * 1000 * 1000;
+const MAX_BUCKET_KEYS = BATCH_KEY_MAX;
+const MAX_BATCH_OPERATIONS = 5;
+
+function pluralise(count: number) {
+	return count === 1 ? "" : "s";
+}
+
 /**
  * Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
  * the worker given by `scriptName`.
@@ -116,13 +128,13 @@ export async function syncAssets(
 	if (siteAssets === undefined) {
 		return { manifest: undefined, namespace: undefined };
 	}
-
 	if (dryRun) {
 		logger.log("(Note: doing a dry run, not uploading or deleting anything.)");
 		return { manifest: undefined, namespace: undefined };
 	}
 	assert(accountId, "Missing accountId");
 
+	// Create assets namespace if it doesn't exist
 	const title = `__${scriptName}-workers_sites_assets${
 		preview ? "_preview" : ""
 	}`;
@@ -131,55 +143,81 @@ export async function syncAssets(
 		title,
 		accountId
 	);
-
-	// let's get all the keys in this namespace
+	// Get all existing keys in asset namespace
+	logger.info("Fetching list of already uploaded assets...");
 	const namespaceKeysResponse = await listKVNamespaceKeys(accountId, namespace);
 	const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
 
-	const manifest: Record<string, string> = {};
-
-	// A batch of uploads where each bucket has to be less than 98mb
-	const uploadBuckets: KeyValue[][] = [];
-	// The "live" bucket that we'll keep filling until it's just below 98mb
-	let uploadBucket: KeyValue[] = [];
-	// A size counter for the live bucket
-	let uploadBucketSize = 0;
-
-	const include = createPatternMatcher(siteAssets.includePatterns, false);
-	const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
-	const hasher = await xxhash();
-
 	const assetDirectory = path.join(
 		siteAssets.baseDirectory,
 		siteAssets.assetDirectory
 	);
+	const include = createPatternMatcher(siteAssets.includePatterns, false);
+	const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
+	const hasher = await xxhash();
+
+	// Find and validate all assets before we make any changes (can't store base64
+	// contents in memory for upload as users may have *lots* of files, and we
+	// don't want to OOM: https://github.com/cloudflare/workers-sdk/issues/2223)
+
+	const manifest: Record<string, string> = {};
+	type PathKey = [path: string, key: string];
+	// A batch of uploads where each bucket has to be less than 100 MiB and
+	// contain less than 10,000 keys (although we limit to 98 MB and 5000 keys)
+	const uploadBuckets: PathKey[][] = [];
+	// The "live" bucket we'll keep filling until it's just below the size limit
+	let uploadBucket: PathKey[] = [];
+	// Current size of the live bucket in bytes (just base64 encoded values)
+	let uploadBucketSize = 0;
+
+	let uploadCount = 0;
+	let skipCount = 0;
+
+	// Always log the first MAX_DIFF_LINES lines, then require the debug log level
+	let diffCount = 0;
+	function logDiff(line: string) {
+		const level = logger.loggerLevel;
+		if (LOGGER_LEVELS[level] >= LOGGER_LEVELS.debug) {
+			// If we're logging as debug level, we want *all* diff lines to be logged
+			// at debug level, not just the first MAX_DIFF_LINES
+			logger.debug(line);
+		} else if (diffCount < MAX_DIFF_LINES) {
+			// Otherwise, log  the first MAX_DIFF_LINES diffs at info level...
+			logger.info(line);
+		} else if (diffCount === MAX_DIFF_LINES) {
+			// ...and warn when we start to truncate it
+			const msg =
+				"   (truncating changed assets log, set `WRANGLER_LOG=debug` environment variable to see full diff)";
+			logger.info(chalk.dim(msg));
+		}
+		diffCount++;
+	}
+
+	logger.info("Building list of assets to upload...");
 	for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
 		const assetFile = path.relative(assetDirectory, absAssetFile);
-		if (!include(assetFile)) {
-			continue;
-		}
-		if (exclude(assetFile)) {
-			continue;
-		}
+		if (!include(assetFile) || exclude(assetFile)) continue;
 
-		logger.log(`Reading ${assetFile}...`);
 		const content = await readFile(absAssetFile, "base64");
-		await validateAssetSize(absAssetFile, assetFile);
-		// while KV accepts files that are 25 MiB **before** b64 encoding
+		// While KV accepts files that are 25 MiB **before** b64 encoding
 		// the overall bucket size must be below 100 MB **after** b64 encoding
-		const assetSize = Buffer.from(content).length;
+		const assetSize = Buffer.byteLength(content);
+		await validateAssetSize(absAssetFile, assetFile);
 		const assetKey = hashAsset(hasher, assetFile, content);
 		validateAssetKey(assetKey);
 
-		// now put each of the files into kv
 		if (!namespaceKeys.has(assetKey)) {
-			logger.log(`Uploading as ${assetKey}...`);
-
-			// Check if adding this asset to the bucket would
-			// push it over the 98 MiB limit KV bulk API limit
-			if (uploadBucketSize + assetSize > 98 * 1000 * 1000) {
-				// If so, move the current bucket into the batch,
-				// and reset the counter/bucket
+			logDiff(
+				chalk.green(` + ${assetKey} (uploading new version of ${assetFile})`)
+			);
+
+			// Check if adding this asset to the bucket would push it over the KV
+			// bulk API limits
+			if (
+				uploadBucketSize + assetSize > MAX_BUCKET_SIZE ||
+				uploadBucket.length + 1 > MAX_BUCKET_KEYS
+			) {
+				// If so, record the current bucket and reset it
 				uploadBuckets.push(uploadBucket);
 				uploadBucketSize = 0;
 				uploadBucket = [];
@@ -187,13 +225,11 @@ export async function syncAssets(
 
 			// Update the bucket and the size counter
 			uploadBucketSize += assetSize;
-			uploadBucket.push({
-				key: assetKey,
-				value: content,
-				base64: true,
-			});
+			uploadBucket.push([absAssetFile, assetKey]);
+			uploadCount++;
 		} else {
-			logger.log(`Skipping - already uploaded.`);
+			logDiff(chalk.dim(` = ${assetKey} (already uploaded ${assetFile})`));
+			skipCount++;
 		}
 
 		// Remove the key from the set so we know what we've already uploaded
@@ -203,23 +239,99 @@ export async function syncAssets(
 		const manifestKey = urlSafe(path.relative(assetDirectory, absAssetFile));
 		manifest[manifestKey] = assetKey;
 	}
+	// Add the last (potentially only or empty) bucket to the batch
+	if (uploadBucket.length > 0) uploadBuckets.push(uploadBucket);
 
-	// Add the last (potentially only) bucket to the batch
-	uploadBuckets.push(uploadBucket);
-
-	// keys now contains all the files we're deleting
 	for (const key of namespaceKeys) {
-		logger.log(`Deleting ${key} from the asset store...`);
+		logDiff(chalk.red(` - ${key} (removing as stale)`));
+	}
+
+	// Upload new assets, with 5 concurrent uploaders
+	if (uploadCount > 0) {
+		const s = pluralise(uploadCount);
+		logger.info(`Uploading ${formatNumber(uploadCount)} new asset${s}...`);
+	}
+	if (skipCount > 0) {
+		const s = pluralise(skipCount);
+		logger.info(
+			`Skipped uploading ${formatNumber(skipCount)} existing asset${s}.`
+		);
 	}
+	let uploadedCount = 0;
+	const controller = new AbortController();
+	const uploaders = Array.from(Array(MAX_BATCH_OPERATIONS)).map(async () => {
+		while (!controller.signal.aborted) {
+			// Get the next bucket to upload. If there is none, stop this uploader.
+			// JavaScript is single(ish)-threaded, so we don't need to worry about
+			// parallel access here.
+			const nextBucket = uploadBuckets.shift();
+			if (nextBucket === undefined) break;
+
+			// Read all files in the bucket as base64
+			// TODO(perf): consider streaming the bulk upload body, rather than
+			//  buffering all base64 contents then JSON-stringifying. This probably
+			//  doesn't matter *too* much: we know buckets will be about 100MB, so
+			//  with 5 uploaders, we could load about 500MB into memory (+ extra
+			//  object keys/tags/copies/etc).
+			const bucket: KeyValue[] = [];
+			for (const [absAssetFile, assetKey] of nextBucket) {
+				bucket.push({
+					key: assetKey,
+					value: await readFile(absAssetFile, "base64"),
+					base64: true,
+				});
+				if (controller.signal.aborted) break;
+			}
 
-	// upload each bucket in parallel
-	const bucketsToPut = [];
-	for (const bucket of uploadBuckets) {
-		bucketsToPut.push(putKVBulkKeyValue(accountId, namespace, bucket));
+			// Upload the bucket to the KV namespace, suppressing logs, we do our own
+			try {
+				await putKVBulkKeyValue(
+					accountId,
+					namespace,
+					bucket,
+					/* quiet */ true,
+					controller.signal
+				);
+			} catch (e) {
+				// https://developer.mozilla.org/en-US/docs/Web/API/DOMException#error_names
+				// https://github.com/nodejs/undici/blob/a3efc9814447001a43a976f1c64adc41995df7e3/lib/core/errors.js#L89
+				if (
+					typeof e === "object" &&
+					e !== null &&
+					"name" in e &&
+					// @ts-expect-error `e.name` should be typed `unknown`, fixed in
+					//  TypeScript 4.9
+					e.name === "AbortError"
+				) {
+					break;
+				}
+				throw e;
+			}
+			uploadedCount += nextBucket.length;
+			const percent = Math.floor((100 * uploadedCount) / uploadCount);
+			logger.info(
+				`Uploaded ${percent}% [${formatNumber(
+					uploadedCount
+				)} out of ${formatNumber(uploadCount)}]`
+			);
+		}
+	});
+	try {
+		// Wait for all uploaders to complete, or one to fail
+		await Promise.all(uploaders);
+	} catch (e) {
+		// If any uploader fails, abort the others
+		logger.info(`Upload failed, aborting...`);
+		controller.abort();
+		throw e;
 	}
-	await Promise.all(bucketsToPut);
 
-	// then delete all the assets that aren't used anymore
+	// Delete stale assets
+	const deleteCount = namespaceKeys.size;
+	if (deleteCount > 0) {
+		const s = pluralise(deleteCount);
+		logger.info(`Removing ${formatNumber(deleteCount)} stale asset${s}...`);
+	}
 	await deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys));
 
 	logger.log("↗️  Done syncing assets");

package/src/traverse-module-graph.ts

@@ -0,0 +1,53 @@
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import chalk from "chalk";
+import { logger } from "./logger";
+import { matchFiles, parseRules } from "./module-collection";
+import type { BundleResult } from "./bundle";
+import type { Config } from "./config";
+import type { Entry } from "./entry";
+
+async function getFiles(root: string, relativeTo: string): Promise<string[]> {
+	const files = [];
+	for (const file of await readdir(root, { withFileTypes: true })) {
+		if (file.isDirectory()) {
+			files.push(...(await getFiles(path.join(root, file.name), relativeTo)));
+		} else {
+			files.push(path.relative(relativeTo, path.join(root, file.name)));
+		}
+	}
+	return files;
+}
+
+export default async function traverseModuleGraph(
+	entry: Entry,
+	rules: Config["rules"]
+): Promise<BundleResult> {
+	const files = await getFiles(entry.moduleRoot, entry.moduleRoot);
+	const relativeEntryPoint = path.relative(entry.moduleRoot, entry.file);
+
+	const modules = (await matchFiles(files, entry.moduleRoot, parseRules(rules)))
+		.filter((m) => m.name !== relativeEntryPoint)
+		.map((m) => ({
+			...m,
+			name: m.name,
+		}));
+
+	const bundleType = entry.format === "modules" ? "esm" : "commonjs";
+
+	if (modules.length > 0) {
+		logger.info(`Uploading additional modules:`);
+		modules.forEach(({ name, type }) => {
+			logger.info(`- ${chalk.blue(name)} (${chalk.green(type ?? "")})`);
+		});
+	}
+
+	return {
+		modules,
+		dependencies: {},
+		resolvedEntryPointPath: entry.file,
+		bundleType,
+		stop: undefined,
+		sourceMapPath: undefined,
+	};
+}

package/src/type-generation.ts

@@ -89,7 +89,7 @@ export async function generateTypes(
 		}
 	}
 
-	if (configToDTS.unsafe) {
+	if (configToDTS.unsafe?.bindings) {
 		for (const unsafe of configToDTS.unsafe.bindings) {
 			envTypeStructure.push(`${unsafe.name}: any;`);
 		}

package/src/user/user.ts

@@ -347,6 +347,7 @@ const Scopes = {
 		"See and change Cloudflare Pages projects, settings and deployments.",
 	"zone:read": "Grants read level access to account zone.",
 	"ssl_certs:write": "See and manage mTLS certificates for your account",
+	"constellation:write": "Manage Constellation AI projects/models",
 } as const;
 
 /**
@@ -1153,7 +1154,7 @@ export async function requireAuth(config: {
 	if (!loggedIn) {
 		if (!isInteractive() || CI.isCI()) {
 			throw new Error(
-				"In a non-interactive environment, it's necessary to set a CLOUDFLARE_API_TOKEN environment variable for wrangler to work. Please go to https://developers.cloudflare.com/api/tokens/create/ for instructions on how to create an api token, and assign its value to CLOUDFLARE_API_TOKEN."
+				"In a non-interactive environment, it's necessary to set a CLOUDFLARE_API_TOKEN environment variable for wrangler to work. Please go to https://developers.cloudflare.com/fundamentals/api/get-started/create-token/ for instructions on how to create an api token, and assign its value to CLOUDFLARE_API_TOKEN."
 			);
 		} else {
 			// didn't login, let's just quit

package/src/utils/render.ts

@@ -0,0 +1,93 @@
+import { EventEmitter } from "events";
+import { render as inkRender } from "ink";
+import type { ReactElement } from "react";
+import type { WriteStream } from "tty";
+
+export function renderToString(tree: ReactElement): string {
+	const { output, cleanup } = render(tree);
+	cleanup();
+	return output;
+}
+
+// The code below is mostly copied from ink-render-string.
+// See https://github.com/zhanwang626/ink-render-string.git.
+
+// MIT License
+
+// Copyright (c) 2021 Zhan Wang
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+interface Instance {
+	output: string;
+	unmount: () => void;
+	cleanup: () => void;
+	stdout: OutputStream;
+	stderr: OutputStream;
+	frames: string[];
+}
+
+const LASTFRAME_UNDEFINED = "value of stdout.lastframe() is undefined";
+
+export class OutputStream extends EventEmitter {
+	readonly frames: string[] = [];
+	private _lastFrame?: string;
+
+	get columns(): number {
+		return this._originalStream.columns;
+	}
+	get rows(): number {
+		return this._originalStream.rows;
+	}
+
+	constructor(private _originalStream: WriteStream) {
+		super();
+	}
+
+	write = (frame: string) => {
+		this.frames.push(frame);
+		this._lastFrame = frame;
+	};
+
+	lastFrame = () => {
+		return this._lastFrame;
+	};
+}
+
+export const render = (tree: ReactElement): Instance => {
+	const stdout = new OutputStream(process.stdout);
+	const stderr = new OutputStream(process.stderr);
+
+	const instance = inkRender(tree, {
+		stdout: stdout as unknown as WriteStream,
+		stderr: stderr as unknown as WriteStream,
+		debug: true,
+		exitOnCtrlC: false,
+		patchConsole: false,
+	});
+
+	return {
+		output: stdout.lastFrame() || LASTFRAME_UNDEFINED,
+		stdout,
+		stderr,
+		cleanup: instance.cleanup,
+		unmount: instance.unmount,
+		frames: stdout.frames,
+	};
+};

package/src/worker.ts

@@ -79,6 +79,15 @@ export interface CfKvNamespace {
 	id: string;
 }
 
+/**
+ * A binding to send email.
+ */
+export interface CfSendEmailBindings {
+	name: string;
+	destination_address?: string;
+	allowed_destination_addresses?: string[];
+}
+
 /**
  * A binding to a wasm module (in service-worker format)
  */
@@ -174,6 +183,13 @@ interface CfUnsafeBinding {
 	type: string;
 }
 
+type CfUnsafeMetadata = Record<string, unknown>;
+
+interface CfUnsafe {
+	bindings: CfUnsafeBinding[] | undefined;
+	metadata: CfUnsafeMetadata | undefined;
+}
+
 export interface CfDurableObjectMigrations {
 	old_tag?: string;
 	new_tag: string;
@@ -209,6 +225,7 @@ export interface CfWorkerInit {
 	bindings: {
 		vars: CfVars | undefined;
 		kv_namespaces: CfKvNamespace[] | undefined;
+		send_email: CfSendEmailBindings[] | undefined;
 		wasm_modules: CfWasmModuleBindings | undefined;
 		text_blobs: CfTextBlobBindings | undefined;
 		data_blobs: CfDataBlobBindings | undefined;
@@ -221,7 +238,7 @@ export interface CfWorkerInit {
 		dispatch_namespaces: CfDispatchNamespace[] | undefined;
 		mtls_certificates: CfMTlsCertificate[] | undefined;
 		logfwdr: CfLogfwdr | undefined;
-		unsafe: CfUnsafeBinding[] | undefined;
+		unsafe: CfUnsafe | undefined;
 	};
 	migrations: CfDurableObjectMigrations | undefined;
 	compatibility_date: string | undefined;

package/templates/d1-beta-facade.js

@@ -11,7 +11,7 @@ var D1Database = class {
 		return new D1PreparedStatement(this, query);
 	}
 	async dump() {
-		const response = await this.binding.fetch("/dump", {
+		const response = await this.binding.fetch("http://d1/dump", {
 			method: "POST",
 			headers: {
 				"content-type": "application/json",
@@ -79,7 +79,7 @@ var D1Database = class {
 						params,
 				  }
 		);
-		const response = await this.binding.fetch(endpoint, {
+		const response = await this.binding.fetch(new URL(endpoint, "http://d1"), {
 			method: "POST",
 			headers: {
 				"content-type": "application/json",
@@ -237,5 +237,8 @@ var shim_default = {
 	async trace(traces, env, ctx) {
 		return worker.trace(traces, getMaskedEnv(env), ctx);
 	},
+	async email(message, env, ctx) {
+		return worker.email(message, getMaskedEnv(env), ctx);
+	},
 };
 export { shim_default as default };

package/templates/gitignore

@@ -169,3 +169,4 @@ dist
 # wrangler project
 
 .dev.vars
+.wrangler/

package/templates/middleware/middleware-miniflare3-json-error.ts

@@ -1,15 +1,27 @@
 import type { Middleware } from "./common";
 
+interface JsonError {
+	message?: string;
+	name?: string;
+	stack?: string;
+	cause?: JsonError;
+}
+
+function reduceError(e: any): JsonError {
+	return {
+		name: e?.name,
+		message: e?.message ?? String(e),
+		stack: e?.stack,
+		cause: e?.cause === undefined ? undefined : reduceError(e.cause),
+	};
+}
+
 // See comment in `bundle.ts` for details on why this is needed
 const jsonError: Middleware = async (request, env, _ctx, middlewareCtx) => {
 	try {
 		return await middlewareCtx.next(request, env);
 	} catch (e: any) {
-		const error = {
-			name: e?.name,
-			message: e?.message ?? String(e),
-			stack: e?.stack,
-		};
+		const error = reduceError(e);
 		return Response.json(error, {
 			status: 500,
 			headers: { "MF-Experimental-Error-Stack": "true" },

package/templates/pages-template-plugin.ts

@@ -133,7 +133,11 @@ export default function (pluginArgs: unknown) {
 		const handlerIterator = executeRequest(request, relativePathname);
 		const pluginNext = async (input?: RequestInfo, init?: RequestInit) => {
 			if (input !== undefined) {
-				request = new Request(input, init);
+				let url = input;
+				if (typeof input === "string") {
+					url = new URL(input, request.url).toString();
+				}
+				request = new Request(url, init);
 			}
 
 			const result = handlerIterator.next();
@@ -141,7 +145,7 @@ export default function (pluginArgs: unknown) {
 			if (result.done === false) {
 				const { handler, params, path } = result.value;
 				const context = {
-					request,
+					request: new Request(request.clone()),
 					functionPath: workerContext.functionPath + path,
 					next: pluginNext,
 					params,
@@ -157,7 +161,7 @@ export default function (pluginArgs: unknown) {
 
 				return cloneResponse(response);
 			} else {
-				return next();
+				return next(request);
 			}
 		};
 

package/templates/tsconfig.init.json

@@ -34,8 +34,7 @@
 		// "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
 		// "typeRoots": [],                                  /* Specify multiple folders that act like `./node_modules/@types`. */
 		"types": [
-			"@cloudflare/workers-types",
-			"vitest"
+			"@cloudflare/workers-types"
 		] /* Specify type package names to be included without being referenced in a source file. */,
 		// "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
 		"resolveJsonModule": true /* Enable importing .json files */,