wrangler 2.2.2 → 2.2.3

package/src/d1/migrations.tsx

@@ -0,0 +1,446 @@
+import fs from "node:fs";
+import path from "path";
+import { Box, render, Text } from "ink";
+import Table from "ink-table";
+import React from "react";
+import { withConfig } from "../config";
+import { confirm } from "../dialogs";
+import { logger } from "../logger";
+import { requireAuth } from "../user";
+import { createBackup } from "./backups";
+import { DEFAULT_MIGRATION_PATH } from "./constants";
+import { executeSql } from "./execute";
+import { Database } from "./options";
+import { d1BetaWarning, getDatabaseInfoFromConfig } from "./utils";
+import type { ConfigFields, DevConfig, Environment } from "../config";
+import type { ParseError } from "../parse";
+import type { BaseSqlExecuteArgs, QueryResult } from "./execute";
+import type { Migration } from "./types";
+import type { Argv } from "yargs";
+
+async function getMigrationsPath(
+	projectPath: string,
+	migrationsFolderPath: string,
+	createIfMissing: boolean
+): Promise<string> {
+	const dir = path.resolve(projectPath, migrationsFolderPath);
+	if (fs.existsSync(dir)) return dir;
+
+	const warning = `No migrations folder found.${
+		migrationsFolderPath === DEFAULT_MIGRATION_PATH
+			? " Set `migrations_dir` in wrangler.toml to choose a different path."
+			: ""
+	}`;
+
+	if (createIfMissing && (await confirm(`${warning}\nOk to create ${dir}?`))) {
+		fs.mkdirSync(dir, { recursive: true });
+		return dir;
+	} else {
+		logger.warn(warning);
+	}
+
+	throw new Error(`No migrations present at ${dir}.`);
+}
+
+async function getUnappliedMigrations(
+	migrationsTableName: string,
+	migrationsPath: string,
+	local: undefined | boolean,
+	config: ConfigFields<DevConfig> & Environment,
+	name: string,
+	persistTo: undefined | string
+): Promise<Array<string>> {
+	const appliedMigrations = (
+		await listAppliedMigrations(
+			migrationsTableName,
+			local,
+			config,
+			name,
+			persistTo
+		)
+	).map((migration) => {
+		return migration.name;
+	});
+	const projectMigrations = getMigrationNames(migrationsPath);
+
+	const unappliedMigrations: Array<string> = [];
+
+	for (const migration of projectMigrations) {
+		if (!appliedMigrations.includes(migration)) {
+			unappliedMigrations.push(migration);
+		}
+	}
+
+	return unappliedMigrations;
+}
+
+export function ListOptions(yargs: Argv): Argv<BaseSqlExecuteArgs> {
+	return Database(yargs);
+}
+
+export const ListHandler = withConfig<BaseSqlExecuteArgs>(
+	async ({ config, database, local, persistTo }): Promise<void> => {
+		await requireAuth({});
+		logger.log(d1BetaWarning);
+
+		const databaseInfo = await getDatabaseInfoFromConfig(config, database);
+		if (!databaseInfo) {
+			throw new Error(
+				`Can't find a DB with name/binding '${database}' in local config. Check info in wrangler.toml...`
+			);
+		}
+
+		if (!config.configPath) {
+			return;
+		}
+		const { migrationsTableName, migrationsFolderPath } = databaseInfo;
+
+		const migrationsPath = await getMigrationsPath(
+			path.dirname(config.configPath),
+			migrationsFolderPath,
+			false
+		);
+		await initMigrationsTable(
+			migrationsTableName,
+			local,
+			config,
+			database,
+			persistTo
+		);
+
+		const unappliedMigrations = (
+			await getUnappliedMigrations(
+				migrationsTableName,
+				migrationsPath,
+				local,
+				config,
+				database,
+				persistTo
+			)
+		).map((migration) => {
+			return {
+				Name: migration,
+			};
+		});
+
+		if (unappliedMigrations.length === 0) {
+			render(<Text>✅ No migrations to apply!</Text>);
+			return;
+		}
+
+		render(
+			<Box flexDirection="column">
+				<Text>Migrations to be applied:</Text>
+				<Table data={unappliedMigrations} columns={["Name"]}></Table>
+			</Box>
+		);
+	}
+);
+
+export function ApplyOptions(yargs: Argv): Argv<BaseSqlExecuteArgs> {
+	return Database(yargs);
+}
+
+export const ApplyHandler = withConfig<BaseSqlExecuteArgs>(
+	async ({ config, database, local, persistTo }): Promise<void> => {
+		const accountId = await requireAuth({});
+		logger.log(d1BetaWarning);
+
+		const databaseInfo = await getDatabaseInfoFromConfig(config, database);
+		if (!databaseInfo) {
+			throw new Error(
+				`Can't find a DB with name/binding '${database}' in local config. Check info in wrangler.toml...`
+			);
+		}
+
+		if (!config.configPath) {
+			return;
+		}
+
+		const migrationsPath = await getMigrationsPath(
+			path.dirname(config.configPath),
+			databaseInfo.migrationsFolderPath,
+			false
+		);
+		await initMigrationsTable(
+			databaseInfo.migrationsTableName,
+			local,
+			config,
+			database,
+			persistTo
+		);
+
+		const unappliedMigrations = (
+			await getUnappliedMigrations(
+				databaseInfo.migrationsTableName,
+				migrationsPath,
+				local,
+				config,
+				database,
+				persistTo
+			)
+		)
+			.map((migration) => {
+				return {
+					Name: migration,
+					Status: "🕒️",
+				};
+			})
+			.sort((a, b) => {
+				const migrationNumberA = parseInt(a.Name.split("_")[0]);
+				const migrationNumberB = parseInt(b.Name.split("_")[0]);
+				if (migrationNumberA < migrationNumberB) {
+					return -1;
+				}
+				if (migrationNumberA > migrationNumberB) {
+					return 1;
+				}
+
+				// numbers must be equal
+				return 0;
+			});
+
+		if (unappliedMigrations.length === 0) {
+			render(<Text>✅ No migrations to apply!</Text>);
+			return;
+		}
+
+		const isInteractive = process.stdout.isTTY;
+		if (isInteractive) {
+			const ok = await confirm(
+				`About to apply ${unappliedMigrations.length} migration(s)\n` +
+					"Your database may not be available to serve requests during the migration, continue?",
+				<Box flexDirection="column">
+					<Text>Migrations to be applied:</Text>
+					<Table data={unappliedMigrations} columns={["Name"]}></Table>
+				</Box>
+			);
+			if (!ok) return;
+		}
+
+		render(<Text>🕒 Creating backup...</Text>);
+		await createBackup(accountId, databaseInfo.uuid);
+
+		for (const migration of unappliedMigrations) {
+			let query = fs.readFileSync(
+				`${migrationsPath}/${migration.Name}`,
+				"utf8"
+			);
+			query += `
+								INSERT INTO ${databaseInfo.migrationsTableName} (name)
+								values ('${migration.Name}');
+						`;
+
+			let success = true;
+			let errorNotes: Array<string> = [];
+			try {
+				const response = await executeSql(
+					local,
+					config,
+					database,
+					undefined,
+					persistTo,
+					undefined,
+					query
+				);
+
+				if (response === null) {
+					// TODO:  return error
+					return;
+				}
+
+				for (const result of response) {
+					// When executing more than 1 statement, response turns into an array of QueryResult
+					if (Array.isArray(result)) {
+						for (const subResult of result) {
+							if (!subResult.success) {
+								success = false;
+							}
+						}
+					} else {
+						if (!result.success) {
+							success = false;
+						}
+					}
+				}
+			} catch (e) {
+				const err = e as ParseError;
+
+				success = false;
+				errorNotes = err.notes.map((msg) => msg.text);
+			}
+
+			migration.Status = success ? "✅" : "❌";
+
+			render(
+				<Box flexDirection="column">
+					<Table
+						data={unappliedMigrations}
+						columns={["Name", "Status"]}
+					></Table>
+					{errorNotes.length > 0 && (
+						<Box flexDirection="column">
+							<Text>&nbsp;</Text>
+							<Text>
+								❌ Migration {migration.Name} failed with following Errors
+							</Text>
+							<Table
+								data={errorNotes.map((err) => {
+									return { Error: err };
+								})}
+							></Table>
+						</Box>
+					)}
+				</Box>
+			);
+
+			if (errorNotes.length > 0) return;
+		}
+	}
+);
+
+export const listAppliedMigrations = async (
+	migrationsTableName: string,
+	local: undefined | boolean,
+	config: ConfigFields<DevConfig> & Environment,
+	name: string,
+	persistTo: undefined | string
+): Promise<Migration[]> => {
+	const Query = `SELECT *
+									 FROM ${migrationsTableName}
+									 ORDER BY id`;
+
+	const response: QueryResult[] | null = await executeSql(
+		local,
+		config,
+		name,
+		undefined,
+		persistTo,
+		undefined,
+		Query
+	);
+
+	if (!response || response[0].results.length === 0) return [];
+
+	return response[0].results as Migration[];
+};
+
+const initMigrationsTable = async (
+	migrationsTableName: string,
+	local: undefined | boolean,
+	config: ConfigFields<DevConfig> & Environment,
+	name: string,
+	persistTo: undefined | string
+) => {
+	return executeSql(
+		local,
+		config,
+		name,
+		undefined,
+		persistTo,
+		undefined,
+		`
+						CREATE TABLE IF NOT EXISTS ${migrationsTableName}
+						(
+								id         INTEGER PRIMARY KEY AUTOINCREMENT,
+								name       TEXT UNIQUE,
+								applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
+						);
+				`
+	);
+};
+
+function getMigrationNames(migrationsPath: string): Array<string> {
+	const migrations = [];
+
+	const dir = fs.opendirSync(migrationsPath);
+
+	let dirent;
+	while ((dirent = dir.readSync()) !== null) {
+		if (dirent.name.endsWith(".sql")) migrations.push(dirent.name);
+	}
+
+	dir.closeSync();
+
+	return migrations;
+}
+
+function getNextMigrationNumber(migrationsPath: string): number {
+	let highestMigrationNumber = -1;
+
+	for (const migration in getMigrationNames(migrationsPath)) {
+		const migrationNumber = parseInt(migration.split("_")[0]);
+
+		if (migrationNumber > highestMigrationNumber) {
+			highestMigrationNumber = migrationNumber;
+		}
+	}
+
+	return highestMigrationNumber + 1;
+}
+
+function pad(num: number, size: number): string {
+	let newNum = num.toString();
+	while (newNum.length < size) newNum = "0" + newNum;
+	return newNum;
+}
+
+type MigrationsCreateArgs = {
+	config?: string;
+	database: string;
+	message: string;
+};
+
+export function CreateOptions(yargs: Argv): Argv<MigrationsCreateArgs> {
+	return Database(yargs).positional("message", {
+		describe: "The Migration message",
+		type: "string",
+		demandOption: true,
+	});
+}
+
+export const CreateHandler = withConfig<MigrationsCreateArgs>(
+	async ({ config, database, message }): Promise<void> => {
+		await requireAuth({});
+		logger.log(d1BetaWarning);
+
+		const databaseInfo = await getDatabaseInfoFromConfig(config, database);
+		if (!databaseInfo) {
+			throw new Error(
+				`Can't find a DB with name/binding '${database}' in local config. Check info in wrangler.toml...`
+			);
+		}
+
+		if (!config.configPath) {
+			return;
+		}
+
+		const migrationsPath = await getMigrationsPath(
+			path.dirname(config.configPath),
+			databaseInfo.migrationsFolderPath,
+			true
+		);
+		const nextMigrationNumber = pad(getNextMigrationNumber(migrationsPath), 4);
+		const migrationName = message.replaceAll(" ", "_");
+
+		const newMigrationName = `${nextMigrationNumber}_${migrationName}.sql`;
+
+		fs.writeFileSync(
+			`${migrationsPath}/${newMigrationName}`,
+			`-- Migration number: ${nextMigrationNumber} \t ${new Date().toISOString()}\n`
+		);
+
+		render(
+			<Box flexDirection="column">
+				<Text>
+					✅ Successfully created Migration &apos;{newMigrationName}&apos;!
+				</Text>
+				<Text>&nbsp;</Text>
+				<Text>The migration is available for editing here</Text>
+				<Text>
+					{migrationsPath}/{newMigrationName}
+				</Text>
+			</Box>
+		);
+	}
+);

package/src/d1/options.ts

@@ -10,3 +10,13 @@ export function Name(yargs: Argv) {
 		})
 		.epilogue(d1BetaWarning);
 }
+
+export function Database(yargs: Argv) {
+	return yargs
+		.positional("database", {
+			describe: "The name or binding of the DB",
+			type: "string",
+			demandOption: true,
+		})
+		.epilogue(d1BetaWarning);
+}

package/src/d1/types.tsx

@@ -1,6 +1,10 @@
 export type Database = {
 	uuid: string;
 	name: string;
+	binding: string;
+	internal_env?: string;
+	migrationsTableName: string;
+	migrationsFolderPath: string;
 };
 
 export type Backup = {
@@ -12,3 +16,9 @@ export type Backup = {
 	file_size: number;
 	size?: string;
 };
+
+export type Migration = {
+	id: string;
+	name: string;
+	applied_at: string;
+};

package/src/d1/utils.ts

@@ -1,8 +1,12 @@
+import { DEFAULT_MIGRATION_PATH, DEFAULT_MIGRATION_TABLE } from "./constants";
 import { listDatabases } from "./list";
 import type { Config } from "../config";
 import type { Database } from "./types";
 
-export function getDatabaseInfoFromConfig(config: Config, name: string) {
+export function getDatabaseInfoFromConfig(
+	config: Config,
+	name: string
+): Database | null {
 	for (const d1Database of config.d1_databases) {
 		if (
 			d1Database.database_id &&
@@ -12,6 +16,11 @@ export function getDatabaseInfoFromConfig(config: Config, name: string) {
 				uuid: d1Database.database_id,
 				binding: d1Database.binding,
 				name: d1Database.database_name,
+				migrationsTableName:
+					d1Database.migrations_table || DEFAULT_MIGRATION_TABLE,
+				migrationsFolderPath:
+					d1Database.migrations_dir || DEFAULT_MIGRATION_PATH,
+				internal_env: d1Database.database_internal_env,
 			};
 		}
 	}

package/src/dev/local.tsx

@@ -4,6 +4,7 @@ import { realpathSync } from "node:fs";
 import { readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import chalk from "chalk";
+import getPort from "get-port";
 import { npxImport } from "npx-import";
 import { useState, useEffect, useRef } from "react";
 import onExit from "signal-exit";
@@ -109,6 +110,7 @@ function useLocalWorker({
 	const local = useRef<ChildProcess>();
 	const experimentalLocalRef = useRef<Miniflare3Type>();
 	const removeSignalExitListener = useRef<() => void>();
+	const removeExperimentalLocalSignalExitListener = useRef<() => void>();
 	const [inspectorUrl, setInspectorUrl] = useState<string | undefined>();
 
 	useEffect(() => {
@@ -174,7 +176,7 @@ function useLocalWorker({
 				bundle,
 			});
 
-			const { forkOptions, miniflareCLIPath } = setupMiniflareOptions({
+			const { forkOptions, miniflareCLIPath, options } = setupMiniflareOptions({
 				workerName,
 				port,
 				scriptPath,
@@ -207,16 +209,27 @@ function useLocalWorker({
 			});
 
 			if (experimentalLocal) {
-				// TODO: refactor setupMiniflareOptions so we don't need to parse here
-				const mf2Options: MiniflareOptions = JSON.parse(forkOptions[0]);
-				const mf = await setupExperimentalLocal(mf2Options, format, bundle);
-				await mf.ready;
-				experimentalLocalRef.current = mf;
-				removeSignalExitListener.current = onExit(() => {
-					logger.log("⎔ Shutting down experimental local server.");
-					mf.dispose();
-					experimentalLocalRef.current = undefined;
-				});
+				const mf3Options = await transformLocalOptions(options, format, bundle);
+				const current = experimentalLocalRef.current;
+				if (current === undefined) {
+					// If we don't have an active Miniflare instance, create a new one
+					const Miniflare = await getMiniflare3Constructor();
+					if (abortController.signal.aborted) return;
+					const mf = new Miniflare(mf3Options);
+					experimentalLocalRef.current = mf;
+					removeExperimentalLocalSignalExitListener.current = onExit(() => {
+						logger.log("⎔ Shutting down experimental local server.");
+						mf.dispose();
+						experimentalLocalRef.current = undefined;
+					});
+					await mf.ready;
+				} else {
+					// Otherwise, reuse the existing instance with its loopback server
+					// and just update the options
+					if (abortController.signal.aborted) return;
+					logger.log("⎔ Reloading experimental local server.");
+					await current.setOptions(mf3Options);
+				}
 				return;
 			}
 
@@ -317,13 +330,6 @@ function useLocalWorker({
 				local.current?.kill();
 				local.current = undefined;
 			}
-			if (experimentalLocalRef.current) {
-				logger.log("⎔ Shutting down experimental local server.");
-				// Initialisation errors are also thrown asynchronously by dispose().
-				// The catch() above should've caught them though.
-				experimentalLocalRef.current?.dispose().catch(() => {});
-				experimentalLocalRef.current = undefined;
-			}
 			removeSignalExitListener.current?.();
 			removeSignalExitListener.current = undefined;
 		};
@@ -362,6 +368,26 @@ function useLocalWorker({
 		enablePagesAssetsServiceBinding,
 		experimentalLocal,
 	]);
+
+	// Rather than disposing the Miniflare instance on every reload, only dispose
+	// it if local mode is disabled and the `Local` component is unmounted. This
+	// allows us to use the more efficient `Miniflare#setOptions` on reload which
+	// retains internal state (e.g. the Miniflare loopback server).
+	useEffect(
+		() => () => {
+			if (experimentalLocalRef.current) {
+				logger.log("⎔ Shutting down experimental local server.");
+				// Initialisation errors are also thrown asynchronously by dispose().
+				// The catch() above should've caught them though.
+				experimentalLocalRef.current?.dispose().catch(() => {});
+				experimentalLocalRef.current = undefined;
+			}
+			removeExperimentalLocalSignalExitListener.current?.();
+			removeExperimentalLocalSignalExitListener.current = undefined;
+		},
+		[]
+	);
+
 	return { inspectorUrl };
 }
 
@@ -512,9 +538,10 @@ export function setupMiniflareOptions({
 }: SetupMiniflareOptionsProps): {
 	miniflareCLIPath: string;
 	forkOptions: string[];
+	options: MiniflareOptions;
 } {
 	// It's now getting _really_ messy now with Pages ASSETS binding outside and the external Durable Objects inside.
-	const options = {
+	const options: MiniflareOptions = {
 		name: workerName,
 		port,
 		scriptPath,
@@ -633,7 +660,7 @@ export function setupMiniflareOptions({
 	if (enablePagesAssetsServiceBinding) {
 		forkOptions.push(JSON.stringify(enablePagesAssetsServiceBinding));
 	}
-	return { miniflareCLIPath, forkOptions };
+	return { miniflareCLIPath, forkOptions, options };
 }
 
 export function setupNodeOptions({
@@ -656,19 +683,15 @@ export function setupNodeOptions({
 	return nodeOptions;
 }
 
-// Caching of the `npx-import`ed `@miniflare/tre` package
-// eslint-disable-next-line @typescript-eslint/consistent-type-imports
-let Miniflare: typeof import("@miniflare/tre").Miniflare;
-
 function arrayToObject(values: string[] = []): Record<string, string> {
 	return Object.fromEntries(values.map((value) => [value, value]));
 }
 
-export async function setupExperimentalLocal(
+export async function transformLocalOptions(
 	mf2Options: MiniflareOptions,
 	format: CfScriptFormat,
 	bundle: EsbuildBundle
-): Promise<Miniflare3Type> {
+): Promise<Miniflare3Options> {
 	const options: Miniflare3Options = {
 		...mf2Options,
 		// Miniflare 3 distinguishes between binding name and namespace/bucket IDs.
@@ -676,6 +699,8 @@ export async function setupExperimentalLocal(
 		// TODO: use defined KV preview ID if any
 		kvNamespaces: arrayToObject(mf2Options.kvNamespaces),
 		r2Buckets: arrayToObject(mf2Options.r2Buckets),
+		inspectorPort: await getPort({ port: 9229 }),
+		verbose: true,
 	};
 
 	if (format === "modules") {
@@ -704,14 +729,18 @@ export async function setupExperimentalLocal(
 		];
 	}
 
-	logger.log("⎔ Starting an experimental local server...");
+	return options;
+}
 
+// Caching of the `npx-import`ed `@miniflare/tre` package
+// eslint-disable-next-line @typescript-eslint/consistent-type-imports
+let Miniflare: typeof import("@miniflare/tre").Miniflare;
+export async function getMiniflare3Constructor(): Promise<typeof Miniflare> {
 	if (Miniflare === undefined) {
 		({ Miniflare } = await npxImport<
 			// eslint-disable-next-line @typescript-eslint/consistent-type-imports
 			typeof import("@miniflare/tre")
-		>("@miniflare/tre@next"));
+		>("@miniflare/tre@3.0.0-next.5"));
 	}
-
-	return new Miniflare(options);
+	return Miniflare;
 }