wrangler 0.0.0-e7a0cd5b → 0.0.0-e7c7235fd

package/src/pages/upload.tsx

@@ -1,425 +0,0 @@
-import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
-import { dirname, join, relative, resolve, sep } from "node:path";
-import { render, Text } from "ink";
-import Spinner from "ink-spinner";
-import { getType } from "mime";
-import { Minimatch } from "minimatch";
-import PQueue from "p-queue";
-import prettyBytes from "pretty-bytes";
-import React from "react";
-import { fetchResult } from "../cfetch";
-import { FatalError } from "../errors";
-import isInteractive from "../is-interactive";
-import { logger } from "../logger";
-import {
-	MAX_ASSET_COUNT,
-	MAX_ASSET_SIZE,
-	BULK_UPLOAD_CONCURRENCY,
-	MAX_BUCKET_FILE_COUNT,
-	MAX_BUCKET_SIZE,
-	MAX_CHECK_MISSING_ATTEMPTS,
-	MAX_UPLOAD_ATTEMPTS,
-} from "./constants";
-import { hashFile } from "./hash";
-import { pagesBetaWarning } from "./utils";
-import type { UploadPayloadFile, YargsOptionsToInterface } from "./types";
-import type { Argv } from "yargs";
-
-type UploadArgs = YargsOptionsToInterface<typeof Options>;
-
-export function Options(yargs: Argv) {
-	return yargs
-		.positional("directory", {
-			type: "string",
-			demandOption: true,
-			description: "The directory of static files to upload",
-		})
-		.options({
-			"output-manifest-path": {
-				type: "string",
-				description: "The name of the project you want to deploy to",
-			},
-		})
-		.epilogue(pagesBetaWarning);
-}
-
-export const Handler = async ({
-	directory,
-	outputManifestPath,
-}: UploadArgs) => {
-	if (!directory) {
-		throw new FatalError("Must specify a directory.", 1);
-	}
-
-	if (!process.env.CF_PAGES_UPLOAD_JWT) {
-		throw new FatalError("No JWT given.", 1);
-	}
-
-	const manifest = await upload({
-		directory,
-		jwt: process.env.CF_PAGES_UPLOAD_JWT,
-	});
-
-	if (outputManifestPath) {
-		await mkdir(dirname(outputManifestPath), { recursive: true });
-		await writeFile(outputManifestPath, JSON.stringify(manifest));
-	}
-
-	logger.log(`✨ Upload complete!`);
-};
-
-export const upload = async (
-	args:
-		| {
-				directory: string;
-				jwt: string;
-		  }
-		| { directory: string; accountId: string; projectName: string }
-) => {
-	async function fetchJwt(): Promise<string> {
-		if ("jwt" in args) {
-			return args.jwt;
-		} else {
-			return (
-				await fetchResult<{ jwt: string }>(
-					`/accounts/${args.accountId}/pages/projects/${args.projectName}/upload-token`
-				)
-			).jwt;
-		}
-	}
-
-	type FileContainer = {
-		path: string;
-		contentType: string;
-		sizeInBytes: number;
-		hash: string;
-	};
-
-	const IGNORE_LIST = [
-		"_worker.js",
-		"_redirects",
-		"_headers",
-		"_routes.json",
-		"functions",
-		"**/.DS_Store",
-		"**/node_modules",
-		"**/.git",
-	].map((pattern) => new Minimatch(pattern));
-
-	const directory = resolve(args.directory);
-
-	// TODO(future): Use this to more efficiently load files in and speed up uploading
-	// Limit memory to 1 GB unless more is specified
-	// let maxMemory = 1_000_000_000;
-	// if (process.env.NODE_OPTIONS && (process.env.NODE_OPTIONS.includes('--max-old-space-size=') || process.env.NODE_OPTIONS.includes('--max_old_space_size='))) {
-	// 	const parsed = parser(process.env.NODE_OPTIONS);
-	// 	maxMemory = (parsed['max-old-space-size'] ? parsed['max-old-space-size'] : parsed['max_old_space_size']) * 1000 * 1000; // Turn MB into bytes
-	// }
-
-	const walk = async (
-		dir: string,
-		fileMap: Map<string, FileContainer> = new Map(),
-		startingDir: string = dir
-	) => {
-		const files = await readdir(dir);
-
-		await Promise.all(
-			files.map(async (file) => {
-				const filepath = join(dir, file);
-				const relativeFilepath = relative(startingDir, filepath);
-				const filestat = await stat(filepath);
-
-				for (const minimatch of IGNORE_LIST) {
-					if (minimatch.match(relativeFilepath)) {
-						return;
-					}
-				}
-
-				if (filestat.isSymbolicLink()) {
-					return;
-				}
-
-				if (filestat.isDirectory()) {
-					fileMap = await walk(filepath, fileMap, startingDir);
-				} else {
-					const name = relativeFilepath.split(sep).join("/");
-
-					if (filestat.size > MAX_ASSET_SIZE) {
-						throw new FatalError(
-							`Error: Pages only supports files up to ${prettyBytes(
-								MAX_ASSET_SIZE
-							)} in size\n${name} is ${prettyBytes(filestat.size)} in size`,
-							1
-						);
-					}
-
-					// We don't want to hold the content in memory. We instead only want to read it when it's needed
-					fileMap.set(name, {
-						path: filepath,
-						contentType: getType(name) || "application/octet-stream",
-						sizeInBytes: filestat.size,
-						hash: hashFile(filepath),
-					});
-				}
-			})
-		);
-
-		return fileMap;
-	};
-
-	const fileMap = await walk(directory);
-
-	if (fileMap.size > MAX_ASSET_COUNT) {
-		throw new FatalError(
-			`Error: Pages only supports up to ${MAX_ASSET_COUNT.toLocaleString()} files in a deployment. Ensure you have specified your build output directory correctly.`,
-			1
-		);
-	}
-
-	const files = [...fileMap.values()];
-
-	let jwt = await fetchJwt();
-
-	const start = Date.now();
-
-	let attempts = 0;
-	const getMissingHashes = async (): Promise<string[]> => {
-		try {
-			return await fetchResult<string[]>(`/pages/assets/check-missing`, {
-				method: "POST",
-				headers: {
-					"Content-Type": "application/json",
-					Authorization: `Bearer ${jwt}`,
-				},
-				body: JSON.stringify({
-					hashes: files.map(({ hash }) => hash),
-				}),
-			});
-		} catch (e) {
-			if (attempts < MAX_CHECK_MISSING_ATTEMPTS) {
-				// Exponential backoff, 1 second first time, then 2 second, then 4 second etc.
-				await new Promise((resolvePromise) =>
-					setTimeout(resolvePromise, Math.pow(2, attempts++) * 1000)
-				);
-
-				if ((e as { code: number }).code === 8000013) {
-					// Looks like the JWT expired, fetch another one
-					jwt = await fetchJwt();
-				}
-				return getMissingHashes();
-			} else {
-				throw e;
-			}
-		}
-	};
-	const missingHashes = await getMissingHashes();
-
-	const sortedFiles = files
-		.filter((file) => missingHashes.includes(file.hash))
-		.sort((a, b) => b.sizeInBytes - a.sizeInBytes);
-
-	// Start with a few buckets so small projects still get
-	// the benefit of multiple upload streams
-	const buckets: {
-		files: FileContainer[];
-		remainingSize: number;
-	}[] = new Array(BULK_UPLOAD_CONCURRENCY).fill(null).map(() => ({
-		files: [],
-		remainingSize: MAX_BUCKET_SIZE,
-	}));
-
-	let bucketOffset = 0;
-	for (const file of sortedFiles) {
-		let inserted = false;
-
-		for (let i = 0; i < buckets.length; i++) {
-			// Start at a different bucket for each new file
-			const bucket = buckets[(i + bucketOffset) % buckets.length];
-			if (
-				bucket.remainingSize >= file.sizeInBytes &&
-				bucket.files.length < MAX_BUCKET_FILE_COUNT
-			) {
-				bucket.files.push(file);
-				bucket.remainingSize -= file.sizeInBytes;
-				inserted = true;
-				break;
-			}
-		}
-
-		if (!inserted) {
-			buckets.push({
-				files: [file],
-				remainingSize: MAX_BUCKET_SIZE - file.sizeInBytes,
-			});
-		}
-		bucketOffset++;
-	}
-
-	let counter = fileMap.size - sortedFiles.length;
-	const { rerender, unmount } = render(
-		<Progress done={counter} total={fileMap.size} />
-	);
-
-	const queue = new PQueue({ concurrency: BULK_UPLOAD_CONCURRENCY });
-
-	for (const bucket of buckets) {
-		// Don't upload empty buckets (can happen for tiny projects)
-		if (bucket.files.length === 0) continue;
-
-		attempts = 0;
-		const doUpload = async (): Promise<void> => {
-			// Populate the payload only when actually uploading (this is limited to 3 concurrent uploads at 50 MiB per bucket meaning we'd only load in a max of ~150 MiB)
-			// This is so we don't run out of memory trying to upload the files.
-			const payload: UploadPayloadFile[] = await Promise.all(
-				bucket.files.map(async (file) => ({
-					key: file.hash,
-					value: (await readFile(file.path)).toString("base64"),
-					metadata: {
-						contentType: file.contentType,
-					},
-					base64: true,
-				}))
-			);
-
-			try {
-				return await fetchResult(`/pages/assets/upload`, {
-					method: "POST",
-					headers: {
-						"Content-Type": "application/json",
-						Authorization: `Bearer ${jwt}`,
-					},
-					body: JSON.stringify(payload),
-				});
-			} catch (e) {
-				if (attempts < MAX_UPLOAD_ATTEMPTS) {
-					// Exponential backoff, 1 second first time, then 2 second, then 4 second etc.
-					await new Promise((resolvePromise) =>
-						setTimeout(resolvePromise, Math.pow(2, attempts++) * 1000)
-					);
-
-					if ((e as { code: number }).code === 8000013 || isJwtExpired(jwt)) {
-						// Looks like the JWT expired, fetch another one
-						jwt = await fetchJwt();
-					}
-					return doUpload();
-				} else {
-					throw e;
-				}
-			}
-		};
-
-		queue.add(() =>
-			doUpload().then(
-				() => {
-					counter += bucket.files.length;
-					rerender(<Progress done={counter} total={fileMap.size} />);
-				},
-				(error) => {
-					return Promise.reject(
-						new FatalError(
-							`Failed to upload files. Please try again. Error: ${JSON.stringify(
-								error
-							)})`,
-							error.code || 1
-						)
-					);
-				}
-			)
-		);
-	}
-
-	await queue.onIdle();
-
-	unmount();
-
-	const uploadMs = Date.now() - start;
-
-	const skipped = fileMap.size - missingHashes.length;
-	const skippedMessage = skipped > 0 ? `(${skipped} already uploaded) ` : "";
-
-	logger.log(
-		`✨ Success! Uploaded ${
-			sortedFiles.length
-		} files ${skippedMessage}${formatTime(uploadMs)}\n`
-	);
-
-	const doUpsertHashes = async (): Promise<void> => {
-		try {
-			return await fetchResult(`/pages/assets/upsert-hashes`, {
-				method: "POST",
-				headers: {
-					"Content-Type": "application/json",
-					Authorization: `Bearer ${jwt}`,
-				},
-				body: JSON.stringify({
-					hashes: files.map(({ hash }) => hash),
-				}),
-			});
-		} catch (e) {
-			await new Promise((resolvePromise) => setTimeout(resolvePromise, 1000));
-
-			if ((e as { code: number }).code === 8000013 || isJwtExpired(jwt)) {
-				// Looks like the JWT expired, fetch another one
-				jwt = await fetchJwt();
-			}
-
-			return await fetchResult(`/pages/assets/upsert-hashes`, {
-				method: "POST",
-				headers: {
-					"Content-Type": "application/json",
-					Authorization: `Bearer ${jwt}`,
-				},
-				body: JSON.stringify({
-					hashes: files.map(({ hash }) => hash),
-				}),
-			});
-		}
-	};
-
-	try {
-		await doUpsertHashes();
-	} catch {
-		logger.warn(
-			"Failed to update file hashes. Every upload appeared to succeed for this deployment, but you might need to re-upload for future deployments. This shouldn't have any impact other than slowing the upload speed of your next deployment."
-		);
-	}
-
-	return Object.fromEntries(
-		[...fileMap.entries()].map(([fileName, file]) => [
-			`/${fileName}`,
-			file.hash,
-		])
-	);
-};
-
-// Decode and check that the current JWT has not expired
-function isJwtExpired(token: string): boolean | undefined {
-	try {
-		const decodedJwt = JSON.parse(
-			Buffer.from(token.split(".")[1], "base64").toString()
-		);
-
-		const dateNow = new Date().getTime() / 1000;
-
-		return decodedJwt.exp <= dateNow;
-	} catch (e) {
-		if (e instanceof Error) {
-			throw new Error(`Invalid token: ${e.message}`);
-		}
-	}
-}
-
-function formatTime(duration: number) {
-	return `(${(duration / 1000).toFixed(2)} sec)`;
-}
-
-function Progress({ done, total }: { done: number; total: number }) {
-	return (
-		<>
-			<Text>
-				{isInteractive() ? <Spinner type="earth" /> : null}
-				{` Uploading... (${done}/${total})\n`}
-			</Text>
-		</>
-	);
-}

package/src/pages/utils.ts

@@ -1,12 +0,0 @@
-import type { BuildResult } from "esbuild";
-
-export const RUNNING_BUILDERS: BuildResult[] = [];
-
-export const CLEANUP_CALLBACKS: (() => void)[] = [];
-export const CLEANUP = () => {
-	CLEANUP_CALLBACKS.forEach((callback) => callback());
-	RUNNING_BUILDERS.forEach((builder) => builder.stop?.());
-};
-
-export const pagesBetaWarning =
-	"🚧 'wrangler pages <command>' is a beta command. Please report any issues to https://github.com/cloudflare/wrangler2/issues/new/choose";

package/src/parse.ts

@@ -1,286 +0,0 @@
-import * as fs from "node:fs";
-import { resolve } from "node:path";
-import TOML from "@iarna/toml";
-import { formatMessagesSync } from "esbuild";
-import { logger } from "./logger";
-
-export type Message = {
-	text: string;
-	location?: Location;
-	notes?: Message[];
-	kind?: "warning" | "error";
-};
-
-export type Location = File & {
-	line: number;
-	column: number;
-	length?: number;
-	lineText?: string;
-	suggestion?: string;
-};
-
-export type File = {
-	file?: string;
-	fileText?: string;
-};
-
-/**
- * Formats a `Message` using esbuild's pretty-printing algorithm.
- */
-export function formatMessage(
-	{ text, notes, location, kind = "error" }: Message,
-	color = true
-): string {
-	const input = { text, notes, location };
-	delete input.location?.fileText;
-	for (const note of notes ?? []) {
-		delete note.location?.fileText;
-	}
-	const lines = formatMessagesSync([input], {
-		color,
-		kind: kind,
-		terminalWidth: logger.columns,
-	});
-	return lines.join("\n");
-}
-
-/**
- * An error that's thrown when something fails to parse.
- */
-export class ParseError extends Error implements Message {
-	readonly text: string;
-	readonly notes: Message[];
-	readonly location?: Location;
-	readonly kind: "warning" | "error";
-
-	constructor({ text, notes, location, kind }: Message) {
-		super(text);
-		this.name = this.constructor.name;
-		this.text = text;
-		this.notes = notes ?? [];
-		this.location = location;
-		this.kind = kind ?? "error";
-	}
-}
-
-const TOML_ERROR_NAME = "TomlError";
-const TOML_ERROR_SUFFIX = " at row ";
-
-type TomlError = Error & {
-	line: number;
-	col: number;
-};
-
-/**
- * A wrapper around `TOML.parse` that throws a `ParseError`.
- */
-export function parseTOML(input: string, file?: string): TOML.JsonMap | never {
-	try {
-		// Normalize CRLF to LF to avoid hitting https://github.com/iarna/iarna-toml/issues/33.
-		const normalizedInput = input.replace(/\r\n/g, "\n");
-		return TOML.parse(normalizedInput);
-	} catch (err) {
-		const { name, message, line, col } = err as TomlError;
-		if (name !== TOML_ERROR_NAME) {
-			throw err;
-		}
-		const text = message.substring(0, message.lastIndexOf(TOML_ERROR_SUFFIX));
-		const lineText = input.split("\n")[line];
-		const location = {
-			lineText,
-			line: line + 1,
-			column: col - 1,
-			file,
-			fileText: input,
-		};
-		throw new ParseError({ text, location });
-	}
-}
-
-const JSON_ERROR_SUFFIX = " in JSON at position ";
-
-/**
- * A minimal type describing a package.json file.
- */
-export type PackageJSON = {
-	devDependencies?: Record<string, unknown>;
-	dependencies?: Record<string, unknown>;
-	scripts?: Record<string, unknown>;
-};
-
-/**
- * A typed version of `parseJSON()`.
- */
-export function parsePackageJSON<T extends PackageJSON = PackageJSON>(
-	input: string,
-	file?: string
-): T {
-	return parseJSON<T>(input, file);
-}
-
-/**
- * A wrapper around `JSON.parse` that throws a `ParseError`.
- */
-export function parseJSON<T>(input: string, file?: string): T {
-	try {
-		return JSON.parse(input);
-	} catch (err) {
-		const { message } = err as Error;
-		const index = message.lastIndexOf(JSON_ERROR_SUFFIX);
-		if (index < 0) {
-			throw err;
-		}
-		const text = message.substring(0, index);
-		const position = parseInt(
-			message.substring(index + JSON_ERROR_SUFFIX.length)
-		);
-		const location = indexLocation({ file, fileText: input }, position);
-		throw new ParseError({ text, location });
-	}
-}
-
-/**
- * Reads a file into a node Buffer.
- */
-export function readFileSyncToBuffer(file: string): Buffer {
-	try {
-		return fs.readFileSync(file);
-	} catch (err) {
-		const { message } = err as Error;
-		throw new ParseError({
-			text: `Could not read file: ${file}`,
-			notes: [
-				{
-					text: message.replace(file, resolve(file)),
-				},
-			],
-		});
-	}
-}
-
-/**
- * Reads a file and parses it based on its type.
- */
-export function readFileSync(file: string): string {
-	try {
-		return fs.readFileSync(file, { encoding: "utf-8" });
-	} catch (err) {
-		const { message } = err as Error;
-		throw new ParseError({
-			text: `Could not read file: ${file}`,
-			notes: [
-				{
-					text: message.replace(file, resolve(file)),
-				},
-			],
-		});
-	}
-}
-
-/**
- * Calculates the line and column location from an index.
- */
-export function indexLocation(file: File, index: number): Location {
-	let lineText,
-		line = 0,
-		column = 0,
-		cursor = 0;
-	const { fileText = "" } = file;
-	for (const row of fileText.split("\n")) {
-		line++;
-		cursor += row.length + 1;
-		if (cursor >= index) {
-			lineText = row;
-			column = row.length - (cursor - index);
-			break;
-		}
-	}
-	return { lineText, line, column, ...file };
-}
-
-/**
- * Guesses the line and column location of a search query.
- */
-export function searchLocation(file: File, query: unknown): Location {
-	let lineText,
-		length,
-		line = 0,
-		column = 0;
-	const queryText = String(query);
-	const { fileText = "" } = file;
-	for (const content of fileText.split("\n")) {
-		line++;
-		const index = content.indexOf(queryText);
-		if (index >= 0) {
-			lineText = content;
-			column = index;
-			length = queryText.length;
-			break;
-		}
-	}
-	return { lineText, line, column, length, ...file };
-}
-
-const units = {
-	nanoseconds: 0.000000001,
-	nanosecond: 0.000000001,
-	microseconds: 0.000001,
-	microsecond: 0.000001,
-	milliseconds: 0.001,
-	millisecond: 0.001,
-	seconds: 1,
-	second: 1,
-	minutes: 60,
-	minute: 60,
-	hours: 3600,
-	hour: 3600,
-	days: 86400,
-	day: 86400,
-	weeks: 604800,
-	week: 604800,
-	month: 18144000,
-	year: 220752000,
-
-	nsecs: 0.000000001,
-	nsec: 0.000000001,
-	usecs: 0.000001,
-	usec: 0.000001,
-	msecs: 0.001,
-	msec: 0.001,
-	secs: 1,
-	sec: 1,
-	mins: 60,
-	min: 60,
-
-	ns: 0.000000001,
-	us: 0.000001,
-	ms: 0.001,
-	mo: 18144000,
-	yr: 220752000,
-
-	s: 1,
-	m: 60,
-	h: 3600,
-	d: 86400,
-	w: 604800,
-	y: 220752000,
-};
-
-/**
- * Parse a human-readable time duration in seconds (including fractional)
- *
- * Invalid values will return NaN
- */
-export function parseHumanDuration(s: string): number {
-	const unitsMap = new Map(Object.entries(units));
-	s = s.trim().toLowerCase();
-	let base = 1;
-	for (const [name, _] of unitsMap) {
-		if (s.endsWith(name)) {
-			s = s.substring(0, s.length - name.length);
-			base = unitsMap.get(name) || 1;
-			break;
-		}
-	}
-	return Number(s) * base;
-}