wrangler 0.0.0-e928f94 → 0.0.0-e9307365

package/src/pages/upload.tsx

@@ -0,0 +1,415 @@
+import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import {
+	basename,
+	dirname,
+	extname,
+	join,
+	relative,
+	resolve,
+	sep,
+} from "node:path";
+import { hash as blake3hash } from "blake3-wasm";
+import { render, Text } from "ink";
+import Spinner from "ink-spinner";
+import { getType } from "mime";
+import PQueue from "p-queue";
+import prettyBytes from "pretty-bytes";
+import React from "react";
+import { fetchResult } from "../cfetch";
+import { FatalError } from "../errors";
+import { logger } from "../logger";
+import {
+	BULK_UPLOAD_CONCURRENCY,
+	MAX_BUCKET_FILE_COUNT,
+	MAX_BUCKET_SIZE,
+	MAX_CHECK_MISSING_ATTEMPTS,
+	MAX_UPLOAD_ATTEMPTS,
+} from "./constants";
+import { pagesBetaWarning } from "./utils";
+import type { UploadPayloadFile } from "./types";
+import type { ArgumentsCamelCase, Argv } from "yargs";
+
+type UploadArgs = {
+	directory: string;
+	"output-manifest-path"?: string;
+};
+
+export function Options(yargs: Argv): Argv<UploadArgs> {
+	return yargs
+		.positional("directory", {
+			type: "string",
+			demandOption: true,
+			description: "The directory of static files to upload",
+		})
+		.options({
+			"output-manifest-path": {
+				type: "string",
+				description: "The name of the project you want to deploy to",
+			},
+		})
+		.epilogue(pagesBetaWarning);
+}
+
+export const Handler = async ({
+	directory,
+	outputManifestPath,
+}: ArgumentsCamelCase<UploadArgs>) => {
+	if (!directory) {
+		throw new FatalError("Must specify a directory.", 1);
+	}
+
+	if (!process.env.CF_PAGES_UPLOAD_JWT) {
+		throw new FatalError("No JWT given.", 1);
+	}
+
+	const manifest = await upload({
+		directory,
+		jwt: process.env.CF_PAGES_UPLOAD_JWT,
+	});
+
+	if (outputManifestPath) {
+		await mkdir(dirname(outputManifestPath), { recursive: true });
+		await writeFile(outputManifestPath, JSON.stringify(manifest));
+	}
+
+	logger.log(`✨ Upload complete!`);
+};
+
+export const upload = async (
+	args:
+		| {
+				directory: string;
+				jwt: string;
+		  }
+		| { directory: string; accountId: string; projectName: string }
+) => {
+	async function fetchJwt(): Promise<string> {
+		if ("jwt" in args) {
+			return args.jwt;
+		} else {
+			return (
+				await fetchResult<{ jwt: string }>(
+					`/accounts/${args.accountId}/pages/projects/${args.projectName}/upload-token`
+				)
+			).jwt;
+		}
+	}
+
+	type FileContainer = {
+		path: string;
+		contentType: string;
+		sizeInBytes: number;
+		hash: string;
+	};
+
+	const IGNORE_LIST = [
+		"_worker.js",
+		"_redirects",
+		"_headers",
+		".DS_Store",
+		"node_modules",
+		".git",
+	];
+
+	const directory = resolve(args.directory);
+
+	// TODO(future): Use this to more efficiently load files in and speed up uploading
+	// Limit memory to 1 GB unless more is specified
+	// let maxMemory = 1_000_000_000;
+	// if (process.env.NODE_OPTIONS && (process.env.NODE_OPTIONS.includes('--max-old-space-size=') || process.env.NODE_OPTIONS.includes('--max_old_space_size='))) {
+	// 	const parsed = parser(process.env.NODE_OPTIONS);
+	// 	maxMemory = (parsed['max-old-space-size'] ? parsed['max-old-space-size'] : parsed['max_old_space_size']) * 1000 * 1000; // Turn MB into bytes
+	// }
+
+	const walk = async (
+		dir: string,
+		fileMap: Map<string, FileContainer> = new Map(),
+		startingDir: string = dir
+	) => {
+		const files = await readdir(dir);
+
+		await Promise.all(
+			files.map(async (file) => {
+				const filepath = join(dir, file);
+				const filestat = await stat(filepath);
+
+				if (IGNORE_LIST.includes(file)) {
+					return;
+				}
+
+				if (filestat.isSymbolicLink()) {
+					return;
+				}
+
+				if (filestat.isDirectory()) {
+					fileMap = await walk(filepath, fileMap, startingDir);
+				} else {
+					const name = relative(startingDir, filepath).split(sep).join("/");
+
+					const fileContent = await readFile(filepath);
+
+					const base64Content = fileContent.toString("base64");
+					const extension = extname(basename(name)).substring(1);
+
+					if (filestat.size > 25 * 1024 * 1024) {
+						throw new FatalError(
+							`Error: Pages only supports files up to ${prettyBytes(
+								25 * 1024 * 1024
+							)} in size\n${name} is ${prettyBytes(filestat.size)} in size`,
+							1
+						);
+					}
+
+					// We don't want to hold the content in memory. We instead only want to read it when it's needed
+					fileMap.set(name, {
+						path: filepath,
+						contentType: getType(name) || "application/octet-stream",
+						sizeInBytes: filestat.size,
+						hash: blake3hash(base64Content + extension)
+							.toString("hex")
+							.slice(0, 32),
+					});
+				}
+			})
+		);
+
+		return fileMap;
+	};
+
+	const fileMap = await walk(directory);
+
+	if (fileMap.size > 20000) {
+		throw new FatalError(
+			`Error: Pages only supports up to 20,000 files in a deployment. Ensure you have specified your build output directory correctly.`,
+			1
+		);
+	}
+
+	const files = [...fileMap.values()];
+
+	let jwt = await fetchJwt();
+
+	const start = Date.now();
+
+	let attempts = 0;
+	const getMissingHashes = async (): Promise<string[]> => {
+		try {
+			return await fetchResult<string[]>(`/pages/assets/check-missing`, {
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+					Authorization: `Bearer ${jwt}`,
+				},
+				body: JSON.stringify({
+					hashes: files.map(({ hash }) => hash),
+				}),
+			});
+		} catch (e) {
+			if (attempts < MAX_CHECK_MISSING_ATTEMPTS) {
+				// Exponential backoff, 1 second first time, then 2 second, then 4 second etc.
+				await new Promise((resolvePromise) =>
+					setTimeout(resolvePromise, Math.pow(2, attempts++) * 1000)
+				);
+
+				if ((e as { code: number }).code === 8000013) {
+					// Looks like the JWT expired, fetch another one
+					jwt = await fetchJwt();
+				}
+				return getMissingHashes();
+			} else {
+				throw e;
+			}
+		}
+	};
+	const missingHashes = await getMissingHashes();
+
+	const sortedFiles = files
+		.filter((file) => missingHashes.includes(file.hash))
+		.sort((a, b) => b.sizeInBytes - a.sizeInBytes);
+
+	// Start with a few buckets so small projects still get
+	// the benefit of multiple upload streams
+	const buckets: {
+		files: FileContainer[];
+		remainingSize: number;
+	}[] = new Array(BULK_UPLOAD_CONCURRENCY).fill(null).map(() => ({
+		files: [],
+		remainingSize: MAX_BUCKET_SIZE,
+	}));
+
+	let bucketOffset = 0;
+	for (const file of sortedFiles) {
+		let inserted = false;
+
+		for (let i = 0; i < buckets.length; i++) {
+			// Start at a different bucket for each new file
+			const bucket = buckets[(i + bucketOffset) % buckets.length];
+			if (
+				bucket.remainingSize >= file.sizeInBytes &&
+				bucket.files.length < MAX_BUCKET_FILE_COUNT
+			) {
+				bucket.files.push(file);
+				bucket.remainingSize -= file.sizeInBytes;
+				inserted = true;
+				break;
+			}
+		}
+
+		if (!inserted) {
+			buckets.push({
+				files: [file],
+				remainingSize: MAX_BUCKET_SIZE - file.sizeInBytes,
+			});
+		}
+		bucketOffset++;
+	}
+
+	let counter = fileMap.size - sortedFiles.length;
+	const { rerender, unmount } = render(
+		<Progress done={counter} total={fileMap.size} />
+	);
+
+	const queue = new PQueue({ concurrency: BULK_UPLOAD_CONCURRENCY });
+
+	for (const bucket of buckets) {
+		// Don't upload empty buckets (can happen for tiny projects)
+		if (bucket.files.length === 0) continue;
+
+		attempts = 0;
+		const doUpload = async (): Promise<void> => {
+			// Populate the payload only when actually uploading (this is limited to 3 concurrent uploads at 50 MiB per bucket meaning we'd only load in a max of ~150 MiB)
+			// This is so we don't run out of memory trying to upload the files.
+			const payload: UploadPayloadFile[] = await Promise.all(
+				bucket.files.map(async (file) => ({
+					key: file.hash,
+					value: (await readFile(file.path)).toString("base64"),
+					metadata: {
+						contentType: file.contentType,
+					},
+					base64: true,
+				}))
+			);
+
+			try {
+				return await fetchResult(`/pages/assets/upload`, {
+					method: "POST",
+					headers: {
+						"Content-Type": "application/json",
+						Authorization: `Bearer ${jwt}`,
+					},
+					body: JSON.stringify(payload),
+				});
+			} catch (e) {
+				if (attempts < MAX_UPLOAD_ATTEMPTS) {
+					// Exponential backoff, 1 second first time, then 2 second, then 4 second etc.
+					await new Promise((resolvePromise) =>
+						setTimeout(resolvePromise, Math.pow(2, attempts++) * 1000)
+					);
+
+					if ((e as { code: number }).code === 8000013) {
+						// Looks like the JWT expired, fetch another one
+						jwt = await fetchJwt();
+					}
+					return doUpload();
+				} else {
+					throw e;
+				}
+			}
+		};
+
+		queue.add(() =>
+			doUpload().then(
+				() => {
+					counter += bucket.files.length;
+					rerender(<Progress done={counter} total={fileMap.size} />);
+				},
+				(error) => {
+					return Promise.reject(
+						new FatalError(
+							"Failed to upload files. Please try again.",
+							error.code || 1
+						)
+					);
+				}
+			)
+		);
+	}
+
+	await queue.onIdle();
+
+	unmount();
+
+	const uploadMs = Date.now() - start;
+
+	const skipped = fileMap.size - missingHashes.length;
+	const skippedMessage = skipped > 0 ? `(${skipped} already uploaded) ` : "";
+
+	logger.log(
+		`✨ Success! Uploaded ${
+			sortedFiles.length
+		} files ${skippedMessage}${formatTime(uploadMs)}\n`
+	);
+
+	const doUpsertHashes = async (): Promise<void> => {
+		try {
+			return await fetchResult(`/pages/assets/upsert-hashes`, {
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+					Authorization: `Bearer ${jwt}`,
+				},
+				body: JSON.stringify({
+					hashes: files.map(({ hash }) => hash),
+				}),
+			});
+		} catch (e) {
+			await new Promise((resolvePromise) => setTimeout(resolvePromise, 1000));
+
+			if ((e as { code: number }).code === 8000013) {
+				// Looks like the JWT expired, fetch another one
+				jwt = await fetchJwt();
+			}
+
+			return await fetchResult(`/pages/assets/upsert-hashes`, {
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+					Authorization: `Bearer ${jwt}`,
+				},
+				body: JSON.stringify({
+					hashes: files.map(({ hash }) => hash),
+				}),
+			});
+		}
+	};
+
+	try {
+		await doUpsertHashes();
+	} catch {
+		logger.warn(
+			"Failed to update file hashes. Every upload appeared to succeed for this deployment, but you might need to re-upload for future deployments. This shouldn't have any impact other than slowing the upload speed of your next deployment."
+		);
+	}
+
+	return Object.fromEntries(
+		[...fileMap.entries()].map(([fileName, file]) => [
+			`/${fileName}`,
+			file.hash,
+		])
+	);
+};
+
+function formatTime(duration: number) {
+	return `(${(duration / 1000).toFixed(2)} sec)`;
+}
+
+function Progress({ done, total }: { done: number; total: number }) {
+	return (
+		<>
+			<Text>
+				<Spinner type="earth" />
+				{` Uploading... (${done}/${total})\n`}
+			</Text>
+		</>
+	);
+}

package/src/pages/utils.ts

@@ -0,0 +1,12 @@
+import type { BuildResult } from "esbuild";
+
+export const RUNNING_BUILDERS: BuildResult[] = [];
+
+export const CLEANUP_CALLBACKS: (() => void)[] = [];
+export const CLEANUP = () => {
+	CLEANUP_CALLBACKS.forEach((callback) => callback());
+	RUNNING_BUILDERS.forEach((builder) => builder.stop?.());
+};
+
+export const pagesBetaWarning =
+	"🚧 'wrangler pages <command>' is a beta command. Please report any issues to https://github.com/cloudflare/wrangler2/issues/new/choose";

package/src/parse.ts

@@ -0,0 +1,286 @@
+import * as fs from "node:fs";
+import { resolve } from "node:path";
+import TOML from "@iarna/toml";
+import { formatMessagesSync } from "esbuild";
+import { logger } from "./logger";
+
+export type Message = {
+	text: string;
+	location?: Location;
+	notes?: Message[];
+	kind?: "warning" | "error";
+};
+
+export type Location = File & {
+	line: number;
+	column: number;
+	length?: number;
+	lineText?: string;
+	suggestion?: string;
+};
+
+export type File = {
+	file?: string;
+	fileText?: string;
+};
+
+/**
+ * Formats a `Message` using esbuild's pretty-printing algorithm.
+ */
+export function formatMessage(
+	{ text, notes, location, kind = "error" }: Message,
+	color = true
+): string {
+	const input = { text, notes, location };
+	delete input.location?.fileText;
+	for (const note of notes ?? []) {
+		delete note.location?.fileText;
+	}
+	const lines = formatMessagesSync([input], {
+		color,
+		kind: kind,
+		terminalWidth: logger.columns,
+	});
+	return lines.join("\n");
+}
+
+/**
+ * An error that's thrown when something fails to parse.
+ */
+export class ParseError extends Error implements Message {
+	readonly text: string;
+	readonly notes: Message[];
+	readonly location?: Location;
+	readonly kind: "warning" | "error";
+
+	constructor({ text, notes, location, kind }: Message) {
+		super(text);
+		this.name = this.constructor.name;
+		this.text = text;
+		this.notes = notes ?? [];
+		this.location = location;
+		this.kind = kind ?? "error";
+	}
+}
+
+const TOML_ERROR_NAME = "TomlError";
+const TOML_ERROR_SUFFIX = " at row ";
+
+type TomlError = Error & {
+	line: number;
+	col: number;
+};
+
+/**
+ * A wrapper around `TOML.parse` that throws a `ParseError`.
+ */
+export function parseTOML(input: string, file?: string): TOML.JsonMap | never {
+	try {
+		// Normalize CRLF to LF to avoid hitting https://github.com/iarna/iarna-toml/issues/33.
+		const normalizedInput = input.replace(/\r\n/g, "\n");
+		return TOML.parse(normalizedInput);
+	} catch (err) {
+		const { name, message, line, col } = err as TomlError;
+		if (name !== TOML_ERROR_NAME) {
+			throw err;
+		}
+		const text = message.substring(0, message.lastIndexOf(TOML_ERROR_SUFFIX));
+		const lineText = input.split("\n")[line];
+		const location = {
+			lineText,
+			line: line + 1,
+			column: col - 1,
+			file,
+			fileText: input,
+		};
+		throw new ParseError({ text, location });
+	}
+}
+
+const JSON_ERROR_SUFFIX = " in JSON at position ";
+
+/**
+ * A minimal type describing a package.json file.
+ */
+export type PackageJSON = {
+	devDependencies?: Record<string, unknown>;
+	dependencies?: Record<string, unknown>;
+	scripts?: Record<string, unknown>;
+};
+
+/**
+ * A typed version of `parseJSON()`.
+ */
+export function parsePackageJSON<T extends PackageJSON = PackageJSON>(
+	input: string,
+	file?: string
+): T {
+	return parseJSON<T>(input, file);
+}
+
+/**
+ * A wrapper around `JSON.parse` that throws a `ParseError`.
+ */
+export function parseJSON<T>(input: string, file?: string): T {
+	try {
+		return JSON.parse(input);
+	} catch (err) {
+		const { message } = err as Error;
+		const index = message.lastIndexOf(JSON_ERROR_SUFFIX);
+		if (index < 0) {
+			throw err;
+		}
+		const text = message.substring(0, index);
+		const position = parseInt(
+			message.substring(index + JSON_ERROR_SUFFIX.length)
+		);
+		const location = indexLocation({ file, fileText: input }, position);
+		throw new ParseError({ text, location });
+	}
+}
+
+/**
+ * Reads a file into a node Buffer.
+ */
+export function readFileSyncToBuffer(file: string): Buffer {
+	try {
+		return fs.readFileSync(file);
+	} catch (err) {
+		const { message } = err as Error;
+		throw new ParseError({
+			text: `Could not read file: ${file}`,
+			notes: [
+				{
+					text: message.replace(file, resolve(file)),
+				},
+			],
+		});
+	}
+}
+
+/**
+ * Reads a file and parses it based on its type.
+ */
+export function readFileSync(file: string): string {
+	try {
+		return fs.readFileSync(file, { encoding: "utf-8" });
+	} catch (err) {
+		const { message } = err as Error;
+		throw new ParseError({
+			text: `Could not read file: ${file}`,
+			notes: [
+				{
+					text: message.replace(file, resolve(file)),
+				},
+			],
+		});
+	}
+}
+
+/**
+ * Calculates the line and column location from an index.
+ */
+export function indexLocation(file: File, index: number): Location {
+	let lineText,
+		line = 0,
+		column = 0,
+		cursor = 0;
+	const { fileText = "" } = file;
+	for (const row of fileText.split("\n")) {
+		line++;
+		cursor += row.length + 1;
+		if (cursor >= index) {
+			lineText = row;
+			column = row.length - (cursor - index);
+			break;
+		}
+	}
+	return { lineText, line, column, ...file };
+}
+
+/**
+ * Guesses the line and column location of a search query.
+ */
+export function searchLocation(file: File, query: unknown): Location {
+	let lineText,
+		length,
+		line = 0,
+		column = 0;
+	const queryText = String(query);
+	const { fileText = "" } = file;
+	for (const content of fileText.split("\n")) {
+		line++;
+		const index = content.indexOf(queryText);
+		if (index >= 0) {
+			lineText = content;
+			column = index;
+			length = queryText.length;
+			break;
+		}
+	}
+	return { lineText, line, column, length, ...file };
+}
+
+const units = {
+	nanoseconds: 0.000000001,
+	nanosecond: 0.000000001,
+	microseconds: 0.000001,
+	microsecond: 0.000001,
+	milliseconds: 0.001,
+	millisecond: 0.001,
+	seconds: 1,
+	second: 1,
+	minutes: 60,
+	minute: 60,
+	hours: 3600,
+	hour: 3600,
+	days: 86400,
+	day: 86400,
+	weeks: 604800,
+	week: 604800,
+	month: 18144000,
+	year: 220752000,
+
+	nsecs: 0.000000001,
+	nsec: 0.000000001,
+	usecs: 0.000001,
+	usec: 0.000001,
+	msecs: 0.001,
+	msec: 0.001,
+	secs: 1,
+	sec: 1,
+	mins: 60,
+	min: 60,
+
+	ns: 0.000000001,
+	us: 0.000001,
+	ms: 0.001,
+	mo: 18144000,
+	yr: 220752000,
+
+	s: 1,
+	m: 60,
+	h: 3600,
+	d: 86400,
+	w: 604800,
+	y: 220752000,
+};
+
+/**
+ * Parse a human-readable time duration in seconds (including fractional)
+ *
+ * Invalid values will return NaN
+ */
+export function parseHumanDuration(s: string): number {
+	const unitsMap = new Map(Object.entries(units));
+	s = s.trim().toLowerCase();
+	let base = 1;
+	for (const [name, _] of unitsMap) {
+		if (s.endsWith(name)) {
+			s = s.substring(0, s.length - name.length);
+			base = unitsMap.get(name) || 1;
+			break;
+		}
+	}
+	return Number(s) * base;
+}