wrangler 0.0.0-e178d6fb → 0.0.0-e1c2f5b9

package/src/pages/publish.tsx

@@ -16,22 +16,20 @@ import * as metrics from "../metrics";
 import { requireAuth } from "../user";
 import { buildFunctions } from "./build";
 import { PAGES_CONFIG_CACHE_FILENAME } from "./constants";
+import { FunctionsNoRoutesError, getFunctionsNoRoutesWarning } from "./errors";
+import { validateRoutes } from "./functions/routes-validation";
 import { listProjects } from "./projects";
+import { promptSelectProject } from "./prompt-select-project";
 import { upload } from "./upload";
 import { pagesBetaWarning } from "./utils";
-import type { Deployment, PagesConfigCache, Project } from "./types";
-import type { ArgumentsCamelCase, Argv } from "yargs";
-
-type PublishArgs = {
-	directory: string;
-	"project-name"?: string;
-	branch?: string;
-	"commit-hash"?: string;
-	"commit-message"?: string;
-	"commit-dirty"?: boolean;
-};
+import type { YargsOptionsToInterface } from "../yargs-types";
+import type { PagesConfigCache } from "./types";
+import type { Project, Deployment } from "@cloudflare/types";
+import type { Argv } from "yargs";
+
+type PublishArgs = YargsOptionsToInterface<typeof Options>;
 
-export function Options(yargs: Argv): Argv<PublishArgs> {
+export function Options(yargs: Argv) {
 	return yargs
 		.positional("directory", {
 			type: "string",
@@ -60,6 +58,10 @@ export function Options(yargs: Argv): Argv<PublishArgs> {
 				description:
 					"Whether or not the workspace should be considered dirty for this deployment",
 			},
+			"skip-caching": {
+				type: "boolean",
+				description: "Skip asset caching which speeds up builds",
+			},
 			config: {
 				describe: "Pages does not support wrangler.toml",
 				type: "string",
@@ -76,8 +78,9 @@ export const Handler = async ({
 	commitHash,
 	commitMessage,
 	commitDirty,
+	skipCaching,
 	config: wranglerConfig,
-}: ArgumentsCamelCase<PublishArgs>) => {
+}: PublishArgs) => {
 	if (wranglerConfig) {
 		throw new FatalError("Pages does not support wrangler.toml", 1);
 	}
@@ -132,24 +135,7 @@ export const Handler = async ({
 
 		switch (existingOrNew) {
 			case "existing": {
-				projectName = await new Promise((resolve) => {
-					const { unmount } = render(
-						<>
-							<Text>Select a project:</Text>
-							<SelectInput
-								items={projects.map((project) => ({
-									key: project.name,
-									label: project.name,
-									value: project,
-								}))}
-								onSelect={async (selected) => {
-									resolve(selected.value.name);
-									unmount();
-								}}
-							/>
-						</>
-					);
-				});
+				projectName = await promptSelectProject({ accountId });
 				break;
 			}
 			case "new": {
@@ -205,7 +191,6 @@ export const Handler = async ({
 	}
 
 	// We infer git info by default is not passed in
-
 	let isGitDir = true;
 	try {
 		execSync(`git rev-parse --is-inside-work-tree`, {
@@ -249,24 +234,96 @@ export const Handler = async ({
 		}
 	}
 
+	let _headers: string | undefined,
+		_redirects: string | undefined,
+		_routesGenerated: string | undefined,
+		_routesCustom: string | undefined,
+		_workerJS: string | undefined;
+
+	try {
+		_headers = readFileSync(join(directory, "_headers"), "utf-8");
+	} catch {}
+
+	try {
+		_redirects = readFileSync(join(directory, "_redirects"), "utf-8");
+	} catch {}
+
+	try {
+		/**
+		 * Developers can specify a custom _routes.json file, for projects with Pages
+		 * Functions or projects in Advanced Mode
+		 */
+		_routesCustom = readFileSync(join(directory, "_routes.json"), "utf-8");
+	} catch {}
+
+	try {
+		_workerJS = readFileSync(join(directory, "_worker.js"), "utf-8");
+	} catch {}
+
+	// Grab the bindings from the API, we need these for shims and other such hacky inserts
+	const project = await fetchResult<Project>(
+		`/accounts/${accountId}/pages/projects/${projectName}`
+	);
+	let isProduction = true;
+	if (branch) {
+		isProduction = project.production_branch === branch;
+	}
+
+	/**
+	 * Evaluate if this is an Advanced Mode or Pages Functions project. If Advanced Mode, we'll
+	 * go ahead and upload `_worker.js` as is, but if Pages Functions, we need to attempt to build
+	 * Functions first and exit if it failed
+	 */
 	let builtFunctions: string | undefined = undefined;
 	const functionsDirectory = join(cwd(), "functions");
-	if (existsSync(functionsDirectory)) {
+	const routesOutputPath = !existsSync(join(directory, "_routes.json"))
+		? join(tmpdir(), `_routes-${Math.random()}.json`)
+		: undefined;
+
+	// Routing configuration displayed in the Functions tab of a deployment in Dash
+	let filepathRoutingConfig: string | undefined;
+
+	if (!_workerJS && existsSync(functionsDirectory)) {
 		const outfile = join(tmpdir(), `./functionsWorker-${Math.random()}.js`);
+		const outputConfigPath = join(
+			tmpdir(),
+			`functions-filepath-routing-config-${Math.random()}.json`
+		);
 
-		await new Promise((resolve) =>
-			buildFunctions({
+		try {
+			await buildFunctions({
 				outfile,
+				outputConfigPath,
 				functionsDirectory,
-				onEnd: () => resolve(null),
+				onEnd: () => {},
 				buildOutputDirectory: dirname(outfile),
-			})
-		);
+				routesOutputPath,
+				local: false,
+				d1Databases: Object.keys(
+					project.deployment_configs[isProduction ? "production" : "preview"]
+						.d1_databases ?? {}
+				),
+			});
 
-		builtFunctions = readFileSync(outfile, "utf-8");
+			builtFunctions = readFileSync(outfile, "utf-8");
+			filepathRoutingConfig = readFileSync(outputConfigPath, "utf-8");
+		} catch (e) {
+			if (e instanceof FunctionsNoRoutesError) {
+				logger.warn(
+					getFunctionsNoRoutesWarning(functionsDirectory, "skipping")
+				);
+			} else {
+				throw e;
+			}
+		}
 	}
 
-	const manifest = await upload({ directory, accountId, projectName });
+	const manifest = await upload({
+		directory,
+		accountId,
+		projectName,
+		skipCaching: skipCaching ?? false,
+	});
 
 	const formData = new FormData();
 
@@ -288,34 +345,100 @@ export const Handler = async ({
 		formData.append("commit_dirty", commitDirty);
 	}
 
-	let _headers: string | undefined,
-		_redirects: string | undefined,
-		_workerJS: string | undefined;
-
-	try {
-		_headers = readFileSync(join(directory, "_headers"), "utf-8");
-	} catch {}
-
-	try {
-		_redirects = readFileSync(join(directory, "_redirects"), "utf-8");
-	} catch {}
-
-	try {
-		_workerJS = readFileSync(join(directory, "_worker.js"), "utf-8");
-	} catch {}
-
 	if (_headers) {
 		formData.append("_headers", new File([_headers], "_headers"));
+		logger.log(`✨ Uploading _headers`);
 	}
 
 	if (_redirects) {
 		formData.append("_redirects", new File([_redirects], "_redirects"));
+		logger.log(`✨ Uploading _redirects`);
 	}
 
-	if (builtFunctions) {
-		formData.append("_worker.js", new File([builtFunctions], "_worker.js"));
-	} else if (_workerJS) {
+	if (filepathRoutingConfig) {
+		formData.append(
+			"functions-filepath-routing-config.json",
+			new File(
+				[filepathRoutingConfig],
+				"functions-filepath-routing-config.json"
+			)
+		);
+	}
+
+	/**
+	 * Advanced Mode
+	 * https://developers.cloudflare.com/pages/platform/functions/#advanced-mode
+	 *
+	 * When using a _worker.js file, the entire /functions directory is ignored
+	 * – this includes its routing and middleware characteristics.
+	 */
+	if (_workerJS) {
 		formData.append("_worker.js", new File([_workerJS], "_worker.js"));
+		logger.log(`✨ Uploading _worker.js`);
+
+		if (_routesCustom) {
+			// user provided a custom _routes.json file
+			try {
+				const routesCustomJSON = JSON.parse(_routesCustom);
+				validateRoutes(routesCustomJSON, join(directory, "_routes.json"));
+
+				formData.append(
+					"_routes.json",
+					new File([_routesCustom], "_routes.json")
+				);
+				logger.log(`✨ Uploading _routes.json`);
+				logger.warn(
+					`_routes.json is an experimental feature and is subject to change. Please use with care.`
+				);
+			} catch (err) {
+				if (err instanceof FatalError) {
+					throw err;
+				}
+			}
+		}
+	}
+
+	/**
+	 * Pages Functions
+	 * https://developers.cloudflare.com/pages/platform/functions/
+	 */
+	if (builtFunctions && !_workerJS) {
+		// if Functions were build successfully, proceed to uploading the build file
+		formData.append("_worker.js", new File([builtFunctions], "_worker.js"));
+		logger.log(`✨ Uploading Functions`);
+
+		if (_routesCustom) {
+			// user provided a custom _routes.json file
+			try {
+				const routesCustomJSON = JSON.parse(_routesCustom);
+				validateRoutes(routesCustomJSON, join(directory, "_routes.json"));
+
+				formData.append(
+					"_routes.json",
+					new File([_routesCustom], "_routes.json")
+				);
+				logger.log(`✨ Uploading _routes.json`);
+				logger.warn(
+					`_routes.json is an experimental feature and is subject to change. Please use with care.`
+				);
+			} catch (err) {
+				if (err instanceof FatalError) {
+					throw err;
+				}
+			}
+		} else if (routesOutputPath) {
+			// no custom _routes.json file found, so fallback to the generated one
+			try {
+				_routesGenerated = readFileSync(routesOutputPath, "utf-8");
+
+				if (_routesGenerated) {
+					formData.append(
+						"_routes.json",
+						new File([_routesGenerated], "_routes.json")
+					);
+				}
+			} catch {}
+		}
 	}
 
 	const deploymentResponse = await fetchResult<Deployment>(
@@ -334,5 +457,5 @@ export const Handler = async ({
 	logger.log(
 		`✨ Deployment complete! Take a peek over at ${deploymentResponse.url}`
 	);
-	await metrics.sendMetricsEvent("deploy pages project");
+	await metrics.sendMetricsEvent("create pages deployment");
 };

package/src/pages/types.ts

@@ -13,6 +13,7 @@ export type Project = {
 };
 export type Deployment = {
 	id: string;
+	created_on: string;
 	environment: string;
 	deployment_trigger: {
 		metadata: {

package/src/pages/upload.tsx

@@ -1,39 +1,34 @@
 import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
-import {
-	basename,
-	dirname,
-	extname,
-	join,
-	relative,
-	resolve,
-	sep,
-} from "node:path";
-import { hash as blake3hash } from "blake3-wasm";
+import { dirname, join, relative, resolve, sep } from "node:path";
 import { render, Text } from "ink";
 import Spinner from "ink-spinner";
 import { getType } from "mime";
+import { Minimatch } from "minimatch";
 import PQueue from "p-queue";
 import prettyBytes from "pretty-bytes";
 import React from "react";
 import { fetchResult } from "../cfetch";
 import { FatalError } from "../errors";
+import isInteractive from "../is-interactive";
 import { logger } from "../logger";
 import {
+	MAX_ASSET_COUNT,
+	MAX_ASSET_SIZE,
 	BULK_UPLOAD_CONCURRENCY,
 	MAX_BUCKET_FILE_COUNT,
 	MAX_BUCKET_SIZE,
+	MAX_CHECK_MISSING_ATTEMPTS,
 	MAX_UPLOAD_ATTEMPTS,
 } from "./constants";
+import { hashFile } from "./hash";
 import { pagesBetaWarning } from "./utils";
+import type { YargsOptionsToInterface } from "../yargs-types";
 import type { UploadPayloadFile } from "./types";
-import type { ArgumentsCamelCase, Argv } from "yargs";
+import type { Argv } from "yargs";
 
-type UploadArgs = {
-	directory: string;
-	"output-manifest-path"?: string;
-};
+type UploadArgs = YargsOptionsToInterface<typeof Options>;
 
-export function Options(yargs: Argv): Argv<UploadArgs> {
+export function Options(yargs: Argv) {
 	return yargs
 		.positional("directory", {
 			type: "string",
@@ -45,6 +40,10 @@ export function Options(yargs: Argv): Argv<UploadArgs> {
 				type: "string",
 				description: "The name of the project you want to deploy to",
 			},
+			"skip-caching": {
+				type: "boolean",
+				description: "Skip asset caching which speeds up builds",
+			},
 		})
 		.epilogue(pagesBetaWarning);
 }
@@ -52,7 +51,8 @@ export function Options(yargs: Argv): Argv<UploadArgs> {
 export const Handler = async ({
 	directory,
 	outputManifestPath,
-}: ArgumentsCamelCase<UploadArgs>) => {
+	skipCaching,
+}: UploadArgs) => {
 	if (!directory) {
 		throw new FatalError("Must specify a directory.", 1);
 	}
@@ -64,6 +64,7 @@ export const Handler = async ({
 	const manifest = await upload({
 		directory,
 		jwt: process.env.CF_PAGES_UPLOAD_JWT,
+		skipCaching: skipCaching ?? false,
 	});
 
 	if (outputManifestPath) {
@@ -79,8 +80,14 @@ export const upload = async (
 		| {
 				directory: string;
 				jwt: string;
+				skipCaching: boolean;
+		  }
+		| {
+				directory: string;
+				accountId: string;
+				projectName: string;
+				skipCaching: boolean;
 		  }
-		| { directory: string; accountId: string; projectName: string }
 ) => {
 	async function fetchJwt(): Promise<string> {
 		if ("jwt" in args) {
@@ -105,10 +112,12 @@ export const upload = async (
 		"_worker.js",
 		"_redirects",
 		"_headers",
-		".DS_Store",
-		"node_modules",
-		".git",
-	];
+		"_routes.json",
+		"functions",
+		"**/.DS_Store",
+		"**/node_modules",
+		"**/.git",
+	].map((pattern) => new Minimatch(pattern));
 
 	const directory = resolve(args.directory);
 
@@ -130,10 +139,13 @@ export const upload = async (
 		await Promise.all(
 			files.map(async (file) => {
 				const filepath = join(dir, file);
+				const relativeFilepath = relative(startingDir, filepath);
 				const filestat = await stat(filepath);
 
-				if (IGNORE_LIST.includes(file)) {
-					return;
+				for (const minimatch of IGNORE_LIST) {
+					if (minimatch.match(relativeFilepath)) {
+						return;
+					}
 				}
 
 				if (filestat.isSymbolicLink()) {
@@ -143,17 +155,12 @@ export const upload = async (
 				if (filestat.isDirectory()) {
 					fileMap = await walk(filepath, fileMap, startingDir);
 				} else {
-					const name = relative(startingDir, filepath).split(sep).join("/");
-
-					const fileContent = await readFile(filepath);
+					const name = relativeFilepath.split(sep).join("/");
 
-					const base64Content = fileContent.toString("base64");
-					const extension = extname(basename(name)).substring(1);
-
-					if (filestat.size > 25 * 1024 * 1024) {
+					if (filestat.size > MAX_ASSET_SIZE) {
 						throw new FatalError(
 							`Error: Pages only supports files up to ${prettyBytes(
-								25 * 1024 * 1024
+								MAX_ASSET_SIZE
 							)} in size\n${name} is ${prettyBytes(filestat.size)} in size`,
 							1
 						);
@@ -164,9 +171,7 @@ export const upload = async (
 						path: filepath,
 						contentType: getType(name) || "application/octet-stream",
 						sizeInBytes: filestat.size,
-						hash: blake3hash(base64Content + extension)
-							.toString("hex")
-							.slice(0, 32),
+						hash: hashFile(filepath),
 					});
 				}
 			})
@@ -177,9 +182,9 @@ export const upload = async (
 
 	const fileMap = await walk(directory);
 
-	if (fileMap.size > 20000) {
+	if (fileMap.size > MAX_ASSET_COUNT) {
 		throw new FatalError(
-			`Error: Pages only supports up to 20,000 files in a deployment. Ensure you have specified your build output directory correctly.`,
+			`Error: Pages only supports up to ${MAX_ASSET_COUNT.toLocaleString()} files in a deployment. Ensure you have specified your build output directory correctly.`,
 			1
 		);
 	}
@@ -190,19 +195,42 @@ export const upload = async (
 
 	const start = Date.now();
 
-	const missingHashes = await fetchResult<string[]>(
-		`/pages/assets/check-missing`,
-		{
-			method: "POST",
-			headers: {
-				"Content-Type": "application/json",
-				Authorization: `Bearer ${jwt}`,
-			},
-			body: JSON.stringify({
-				hashes: files.map(({ hash }) => hash),
-			}),
+	let attempts = 0;
+	const getMissingHashes = async (skipCaching: boolean): Promise<string[]> => {
+		if (skipCaching) {
+			console.debug("Force skipping cache");
+			return files.map(({ hash }) => hash);
 		}
-	);
+
+		try {
+			return await fetchResult<string[]>(`/pages/assets/check-missing`, {
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+					Authorization: `Bearer ${jwt}`,
+				},
+				body: JSON.stringify({
+					hashes: files.map(({ hash }) => hash),
+				}),
+			});
+		} catch (e) {
+			if (attempts < MAX_CHECK_MISSING_ATTEMPTS) {
+				// Exponential backoff, 1 second first time, then 2 second, then 4 second etc.
+				await new Promise((resolvePromise) =>
+					setTimeout(resolvePromise, Math.pow(2, attempts++) * 1000)
+				);
+
+				if ((e as { code: number }).code === 8000013) {
+					// Looks like the JWT expired, fetch another one
+					jwt = await fetchJwt();
+				}
+				return getMissingHashes(skipCaching);
+			} else {
+				throw e;
+			}
+		}
+	};
+	const missingHashes = await getMissingHashes(args.skipCaching);
 
 	const sortedFiles = files
 		.filter((file) => missingHashes.includes(file.hash))
@@ -256,7 +284,7 @@ export const upload = async (
 		// Don't upload empty buckets (can happen for tiny projects)
 		if (bucket.files.length === 0) continue;
 
-		let attempts = 0;
+		attempts = 0;
 		const doUpload = async (): Promise<void> => {
 			// Populate the payload only when actually uploading (this is limited to 3 concurrent uploads at 50 MiB per bucket meaning we'd only load in a max of ~150 MiB)
 			// This is so we don't run out of memory trying to upload the files.
@@ -272,7 +300,8 @@ export const upload = async (
 			);
 
 			try {
-				return await fetchResult(`/pages/assets/upload`, {
+				console.debug("POST /pages/assets/upload");
+				const res = await fetchResult(`/pages/assets/upload`, {
 					method: "POST",
 					headers: {
 						"Content-Type": "application/json",
@@ -280,25 +309,28 @@ export const upload = async (
 					},
 					body: JSON.stringify(payload),
 				});
+				console.debug("result:", res);
 			} catch (e) {
 				if (attempts < MAX_UPLOAD_ATTEMPTS) {
+					console.debug("failed:", e, "retrying...");
 					// Exponential backoff, 1 second first time, then 2 second, then 4 second etc.
 					await new Promise((resolvePromise) =>
 						setTimeout(resolvePromise, Math.pow(2, attempts++) * 1000)
 					);
 
-					if ((e as { code: number }).code === 8000013) {
+					if ((e as { code: number }).code === 8000013 || isJwtExpired(jwt)) {
 						// Looks like the JWT expired, fetch another one
 						jwt = await fetchJwt();
 					}
 					return doUpload();
 				} else {
+					console.debug("failed:", e);
 					throw e;
 				}
 			}
 		};
 
-		queue.add(() =>
+		void queue.add(() =>
 			doUpload().then(
 				() => {
 					counter += bucket.files.length;
@@ -307,7 +339,9 @@ export const upload = async (
 				(error) => {
 					return Promise.reject(
 						new FatalError(
-							"Failed to upload files. Please try again.",
+							`Failed to upload files. Please try again. Error: ${JSON.stringify(
+								error
+							)})`,
 							error.code || 1
 						)
 					);
@@ -346,7 +380,7 @@ export const upload = async (
 		} catch (e) {
 			await new Promise((resolvePromise) => setTimeout(resolvePromise, 1000));
 
-			if ((e as { code: number }).code === 8000013) {
+			if ((e as { code: number }).code === 8000013 || isJwtExpired(jwt)) {
 				// Looks like the JWT expired, fetch another one
 				jwt = await fetchJwt();
 			}
@@ -380,6 +414,23 @@ export const upload = async (
 	);
 };
 
+// Decode and check that the current JWT has not expired
+function isJwtExpired(token: string): boolean | undefined {
+	try {
+		const decodedJwt = JSON.parse(
+			Buffer.from(token.split(".")[1], "base64").toString()
+		);
+
+		const dateNow = new Date().getTime() / 1000;
+
+		return decodedJwt.exp <= dateNow;
+	} catch (e) {
+		if (e instanceof Error) {
+			throw new Error(`Invalid token: ${e.message}`);
+		}
+	}
+}
+
 function formatTime(duration: number) {
 	return `(${(duration / 1000).toFixed(2)} sec)`;
 }
@@ -388,7 +439,7 @@ function Progress({ done, total }: { done: number; total: number }) {
 	return (
 		<>
 			<Text>
-				<Spinner type="earth" />
+				{isInteractive() ? <Spinner type="earth" /> : null}
 				{` Uploading... (${done}/${total})\n`}
 			</Text>
 		</>

package/src/pages/utils.ts

@@ -1,6 +1,6 @@
-import type { BuildResult } from "esbuild";
+import type { BundleResult } from "../bundle";
 
-export const RUNNING_BUILDERS: BuildResult[] = [];
+export const RUNNING_BUILDERS: BundleResult[] = [];
 
 export const CLEANUP_CALLBACKS: (() => void)[] = [];
 export const CLEANUP = () => {
@@ -10,3 +10,14 @@ export const CLEANUP = () => {
 
 export const pagesBetaWarning =
 	"🚧 'wrangler pages <command>' is a beta command. Please report any issues to https://github.com/cloudflare/wrangler2/issues/new/choose";
+
+export function isUrl(maybeUrl?: string): maybeUrl is string {
+	if (!maybeUrl) return false;
+
+	try {
+		new URL(maybeUrl);
+		return true;
+	} catch (e) {
+		return false;
+	}
+}