wrangler 0.0.0-e7a0cd5b → 0.0.0-e87ad068

package/src/d1/execute.tsx

@@ -1,67 +1,59 @@
 import { existsSync } from "node:fs";
 import { mkdir } from "node:fs/promises";
 import path from "node:path";
-import { render, Static, Text } from "ink";
+import chalk from "chalk";
+import { Static, Text } from "ink";
 import Table from "ink-table";
 import { npxImport } from "npx-import";
 import React from "react";
 import { fetchResult } from "../cfetch";
-import { withConfig } from "../config";
+import { readConfig } from "../config";
 import { getLocalPersistencePath } from "../dev/get-local-persistence-path";
-import { confirm, logDim } from "../dialogs";
+import { confirm } from "../dialogs";
 import { logger } from "../logger";
+import { readFileSync } from "../parse";
 import { readableRelative } from "../paths";
 import { requireAuth } from "../user";
-import { Name } from "./options";
+import { renderToString } from "../utils/render";
+import { DEFAULT_BATCH_SIZE } from "./constants";
+import * as options from "./options";
+import splitSqlQuery from "./splitter";
 import {
 	d1BetaWarning,
 	getDatabaseByNameOrBinding,
 	getDatabaseInfoFromConfig,
 } from "./utils";
-import type { Config } from "../config";
+import type { Config, ConfigFields, DevConfig, Environment } from "../config";
+import type {
+	CommonYargsArgv,
+	StrictYargsOptionsToInterface,
+} from "../yargs-types";
 import type { Database } from "./types";
-import type splitSqlQuery from "@databases/split-sql-query";
-import type { SQL, SQLQuery } from "@databases/sql";
-import type { Statement as StatementType } from "@miniflare/d1";
-import type { createSQLiteDB as createSQLiteDBType } from "@miniflare/shared";
-import type { Argv } from "yargs";
 
-type MiniflareNpxImportTypes = [
-	{
-		Statement: typeof StatementType;
-	},
-	{
-		createSQLiteDB: typeof createSQLiteDBType;
-	}
-];
-
-type ExecuteArgs = {
-	config?: string;
-	name: string;
-	file?: string;
-	command?: string;
-	local?: boolean;
-	"persist-to"?: string;
-};
-
-type QueryResult = {
+export type QueryResult = {
 	results: Record<string, string | number | boolean>[];
 	success: boolean;
-	duration: number;
+	meta?: {
+		duration?: number;
+	};
 	query?: string;
 };
-// Max number of bytes to send in a single /execute call
-const QUERY_LIMIT = 10_000;
 
-export function Options(yargs: Argv): Argv<ExecuteArgs> {
-	return Name(yargs)
+export function Options(yargs: CommonYargsArgv) {
+	return options
+		.Database(yargs)
+		.option("yes", {
+			describe: 'Answer "yes" to any prompts',
+			type: "boolean",
+			alias: "y",
+		})
 		.option("local", {
 			describe:
 				"Execute commands/files against a local DB for use with wrangler dev",
 			type: "boolean",
 		})
 		.option("file", {
-			describe: "A .sql file to injest",
+			describe: "A .sql file to ingest",
 			type: "string",
 		})
 		.option("command", {
@@ -72,54 +64,69 @@ export function Options(yargs: Argv): Argv<ExecuteArgs> {
 			describe: "Specify directory to use for local persistence (for --local)",
 			type: "string",
 			requiresArg: true,
+		})
+		.option("json", {
+			describe: "Return output as clean JSON",
+			type: "boolean",
+			default: false,
+		})
+		.option("preview", {
+			describe: "Execute commands/files against a preview D1 DB",
+			type: "boolean",
+			default: false,
+		})
+		.option("batch-size", {
+			describe: "Number of queries to send in a single batch",
+			type: "number",
+			default: DEFAULT_BATCH_SIZE,
 		});
 }
 
-function shorten(query: string | undefined, length: number) {
-	return query && query.length > length
-		? query.slice(0, length) + "..."
-		: query;
-}
-
-export const Handler = withConfig<ExecuteArgs>(
-	async ({ config, name, file, command, local, persistTo }): Promise<void> => {
-		logger.log(d1BetaWarning);
-		if (file && command)
-			return console.error(`Error: can't provide both --command and --file.`);
-		const { parser, splitter } = await loadSqlUtils();
-
-		const sql = file
-			? parser.file(file)
-			: command
-			? parser.__dangerous__rawValue(command)
-			: null;
+type HandlerOptions = StrictYargsOptionsToInterface<typeof Options>;
 
-		if (!sql) throw new Error(`Error: must provide --command or --file.`);
-		if (persistTo && !local)
-			throw new Error(`Error: can't use --persist-to without --local`);
+export const Handler = async (args: HandlerOptions): Promise<void> => {
+	const {
+		local,
+		database,
+		yes,
+		persistTo,
+		file,
+		command,
+		json,
+		preview,
+		batchSize,
+	} = args;
+	const existingLogLevel = logger.loggerLevel;
+	if (json) {
+		// set loggerLevel to error to avoid readConfig warnings appearing in JSON output
+		logger.loggerLevel = "error";
+	}
+	const config = readConfig(args.config, args);
+	logger.log(d1BetaWarning);
+	if (file && command)
+		return logger.error(`Error: can't provide both --command and --file.`);
 
-		const isInteractive = process.stdout.isTTY;
-		const response: QueryResult[] | null = local
-			? await executeLocally(
-					config,
-					name,
-					isInteractive,
-					splitSql(splitter, sql),
-					persistTo
-			  )
-			: await executeRemotely(
-					config,
-					name,
-					isInteractive,
-					batchSplit(splitter, sql)
-			  );
+	const isInteractive = process.stdout.isTTY;
+	const response: QueryResult[] | null = await executeSql({
+		local,
+		config,
+		name: database,
+		shouldPrompt: isInteractive && !yes,
+		persistTo,
+		file,
+		command,
+		json,
+		preview,
+		batchSize,
+	});
 
-		// Early exit if prompt rejected
-		if (!response) return;
+	// Early exit if prompt rejected
+	if (!response) return;
 
-		if (isInteractive) {
-			// Render table if single result
-			render(
+	if (isInteractive && !json) {
+		// Render table if single result
+		logger.log(
+			renderToString(
 				<Static items={response}>
 					{(result) => {
 						// batch results
@@ -138,20 +145,90 @@ export const Handler = withConfig<ExecuteArgs>(
 						}
 					}}
 				</Static>
+			)
+		);
+	} else {
+		// set loggerLevel back to what it was before to actually output the JSON in stdout
+		logger.loggerLevel = existingLogLevel;
+		logger.log(JSON.stringify(response, null, 2));
+	}
+};
+
+export async function executeSql({
+	local,
+	config,
+	name,
+	shouldPrompt,
+	persistTo,
+	file,
+	command,
+	json,
+	preview,
+	batchSize,
+}: {
+	local: boolean | undefined;
+	config: ConfigFields<DevConfig> & Environment;
+	name: string;
+	shouldPrompt: boolean | undefined;
+	persistTo: string | undefined;
+	file: string | undefined;
+	command: string | undefined;
+	json: boolean | undefined;
+	preview: boolean | undefined;
+	batchSize: number;
+}) {
+	const sql = file ? readFileSync(file) : command;
+	if (!sql) throw new Error(`Error: must provide --command or --file.`);
+	if (preview && local)
+		throw new Error(`Error: can't use --preview with --local`);
+	if (persistTo && !local)
+		throw new Error(`Error: can't use --persist-to without --local`);
+	logger.log(`🌀 Mapping SQL input into an array of statements`);
+	const queries = splitSqlQuery(sql);
+
+	if (file && sql) {
+		if (queries[0].startsWith("SQLite format 3")) {
+			//TODO: update this error to recommend using `wrangler d1 restore` when it exists
+			throw new Error(
+				"Provided file is a binary SQLite database file instead of an SQL text file.\nThe execute command can only process SQL text files.\nPlease export an SQL file from your SQLite database and try again."
 			);
-		} else {
-			console.log(JSON.stringify(response, null, 2));
 		}
 	}
-);
 
-async function executeLocally(
-	config: Config,
-	name: string,
-	isInteractive: boolean,
-	queries: string[],
-	persistTo: string | undefined
-) {
+	return local
+		? await executeLocally({
+				config,
+				name,
+				shouldPrompt,
+				queries,
+				persistTo,
+				json,
+		  })
+		: await executeRemotely({
+				config,
+				name,
+				shouldPrompt,
+				batches: batchSplit(queries, batchSize),
+				json,
+				preview,
+		  });
+}
+
+async function executeLocally({
+	config,
+	name,
+	shouldPrompt,
+	queries,
+	persistTo,
+	json,
+}: {
+	config: Config;
+	name: string;
+	shouldPrompt: boolean | undefined;
+	queries: string[];
+	persistTo: string | undefined;
+	json: boolean | undefined;
+}) {
 	const localDB = getDatabaseInfoFromConfig(config, name);
 	if (!localDB) {
 		throw new Error(
@@ -167,79 +244,92 @@ async function executeLocally(
 
 	const dbDir = path.join(persistencePath, "d1");
 	const dbPath = path.join(dbDir, `${localDB.binding}.sqlite3`);
-	const [{ Statement }, { createSQLiteDB }] =
-		await npxImport<MiniflareNpxImportTypes>(
-			["@miniflare/d1", "@miniflare/shared"],
-			logDim
-		);
+	const [{ D1Database, D1DatabaseAPI }, { createSQLiteDB }] = await npxImport<
+		// eslint-disable-next-line @typescript-eslint/consistent-type-imports
+		[typeof import("@miniflare/d1"), typeof import("@miniflare/shared")]
+	>(["@miniflare/d1", "@miniflare/shared"], logger.log);
 
-	if (!existsSync(dbDir) && isInteractive) {
-		const ok = await confirm(
-			`About to create ${readableRelative(dbPath)}, ok?`
-		);
+	if (!existsSync(dbDir)) {
+		const ok =
+			json ||
+			!shouldPrompt ||
+			(await confirm(`About to create ${readableRelative(dbPath)}, ok?`));
 		if (!ok) return null;
 		await mkdir(dbDir, { recursive: true });
 	}
 
 	logger.log(`🌀 Loading DB at ${readableRelative(dbPath)}`);
-	const db = await createSQLiteDB(dbPath);
 
-	const results: QueryResult[] = [];
-	for (const sql of queries) {
-		const statement = new Statement(db, sql);
-		results.push((await statement.all()) as QueryResult);
-	}
-
-	return results;
+	const sqliteDb = await createSQLiteDB(dbPath);
+	const db = new D1Database(new D1DatabaseAPI(sqliteDb));
+	const stmts = queries.map((query) => db.prepare(query));
+	return (await db.batch(stmts)) as QueryResult[];
 }
 
-async function executeRemotely(
-	config: Config,
-	name: string,
-	isInteractive: boolean,
-	batches: string[]
-) {
-	if (batches.length > 1) {
+async function executeRemotely({
+	config,
+	name,
+	shouldPrompt,
+	batches,
+	json,
+	preview,
+}: {
+	config: Config;
+	name: string;
+	shouldPrompt: boolean | undefined;
+	batches: string[];
+	json: boolean | undefined;
+	preview: boolean | undefined;
+}) {
+	const multiple_batches = batches.length > 1;
+	// in JSON mode, we don't want a prompt here
+	if (multiple_batches && !json) {
 		const warning = `⚠️  Too much SQL to send at once, this execution will be sent as ${batches.length} batches.`;
 
-		if (isInteractive) {
+		if (shouldPrompt) {
 			const ok = await confirm(
 				`${warning}\nℹ️  Each batch is sent individually and may leave your DB in an unexpected state if a later batch fails.\n⚠️  Make sure you have a recent backup. Ok to proceed?`
 			);
 			if (!ok) return null;
 			logger.log(`🌀 Let's go`);
 		} else {
-			console.error(warning);
+			logger.error(warning);
 		}
 	}
 
-	const accountId = await requireAuth({});
+	const accountId = await requireAuth(config);
 	const db: Database = await getDatabaseByNameOrBinding(
 		config,
 		accountId,
 		name
 	);
-
-	if (isInteractive) {
-		logger.log(`🌀 Executing on ${name} (${db.uuid}):`);
-	} else {
-		// Pipe to error so we don't break jq
-		console.error(`Executing on ${name} (${db.uuid}):`);
+	if (preview && !db.previewDatabaseUuid) {
+		throw logger.error(
+			"Please define a `preview_database_id` in your wrangler.toml to execute your queries against a preview database"
+		);
 	}
+	const dbUuid = preview ? db.previewDatabaseUuid : db.uuid;
+	logger.log(`🌀 Executing on ${name} (${dbUuid}):`);
 
 	const results: QueryResult[] = [];
 	for (const sql of batches) {
+		if (multiple_batches)
+			logger.log(
+				chalk.gray(`  ${sql.slice(0, 70)}${sql.length > 70 ? "..." : ""}`)
+			);
+
 		const result = await fetchResult<QueryResult[]>(
-			`/accounts/${accountId}/d1/database/${db.uuid}/query`,
+			`/accounts/${accountId}/d1/database/${dbUuid}/query`,
 			{
 				method: "POST",
 				headers: {
 					"Content-Type": "application/json",
+					...(db.internal_env ? { "x-d1-internal-env": db.internal_env } : {}),
 				},
 				body: JSON.stringify({ sql }),
 			}
 		);
-		result.map(logResult);
+		logResult(result);
 		results.push(...result);
 	}
 	return results;
@@ -247,53 +337,35 @@ async function executeRemotely(
 
 function logResult(r: QueryResult | QueryResult[]) {
 	logger.log(
-		`🚣 Executed ${Array.isArray(r) ? r.length : "1"} command(s) in ${
+		`🚣 Executed ${
+			Array.isArray(r) ? `${r.length} commands` : "1 command"
+		} in ${
 			Array.isArray(r)
 				? r
-						.map((d: QueryResult) => d.duration)
+						.map((d: QueryResult) => d.meta?.duration || 0)
 						.reduce((a: number, b: number) => a + b, 0)
-				: r.duration
+				: r.meta?.duration
 		}ms`
 	);
 }
 
-function splitSql(splitter: (query: SQLQuery) => SQLQuery[], sql: SQLQuery) {
-	// We have no interpolations, so convert everything to text
-	logger.log(`🌀 Mapping SQL input into an array of statements`);
-	return splitter(sql).map(
-		(q) =>
-			q.format({
-				escapeIdentifier: (_) => "",
-				formatValue: (_, __) => ({ placeholder: "", value: "" }),
-			}).text
-	);
-}
-
-function batchSplit(splitter: typeof splitSqlQuery, sql: SQLQuery) {
-	const queries = splitSql(splitter, sql);
+function batchSplit(queries: string[], batchSize: number) {
 	logger.log(`🌀 Parsing ${queries.length} statements`);
+	const num_batches = Math.ceil(queries.length / batchSize);
 	const batches: string[] = [];
-	const nbatches = Math.floor(queries.length / QUERY_LIMIT);
-	for (let i = 0; i <= nbatches; i++) {
-		batches.push(
-			queries.slice(i * QUERY_LIMIT, (i + 1) * QUERY_LIMIT).join("; ")
+	for (let i = 0; i < num_batches; i++) {
+		batches.push(queries.slice(i * batchSize, (i + 1) * batchSize).join("; "));
+	}
+	if (num_batches > 1) {
+		logger.log(
+			`🌀 We are sending ${num_batches} batch(es) to D1 (limited to ${batchSize} statements per batch. Use --batch-size to override.)`
 		);
 	}
-	logger.log(
-		`🌀 We are sending ${batches.length} batch(es) to D1 (limited to ${QUERY_LIMIT} statements per batch)`
-	);
 	return batches;
 }
 
-async function loadSqlUtils() {
-	const [
-		{ default: parser },
-		{
-			// No idea why this is doubly-nested, see https://github.com/ForbesLindesay/atdatabases/issues/255
-			default: { default: splitter },
-		},
-	] = await npxImport<
-		[{ default: SQL }, { default: { default: typeof splitSqlQuery } }]
-	>(["@databases/sql@3.2.0", "@databases/split-sql-query@1.0.3"], logDim);
-	return { parser, splitter };
+function shorten(query: string | undefined, length: number) {
+	return query && query.length > length
+		? query.slice(0, length) + "..."
+		: query;
 }

package/src/d1/index.ts

@@ -3,11 +3,11 @@ import * as Create from "./create";
 import * as Delete from "./delete";
 import * as Execute from "./execute";
 import * as List from "./list";
+import * as Migrations from "./migrations";
 import { d1BetaWarning } from "./utils";
-import type { CommonYargsOptions } from "../yargs-types";
-import type { Argv } from "yargs";
+import type { CommonYargsArgv } from "../yargs-types";
 
-export const d1 = (yargs: Argv<CommonYargsOptions>) => {
+export function d1(yargs: CommonYargsArgv) {
 	return (
 		yargs
 			.command("list", "List D1 databases", List.Options, List.Handler)
@@ -23,8 +23,9 @@ export const d1 = (yargs: Argv<CommonYargsOptions>) => {
 				Delete.Options,
 				Delete.Handler
 			)
-			.command("backup", "Interact with D1 Backups", (yargs2) =>
-				yargs2
+			.command("backup", "Interact with D1 Backups", (backupArgs) =>
+				backupArgs
+					.demandCommand()
 					.command(
 						"list <name>",
 						"List your D1 backups",
@@ -66,11 +67,34 @@ export const d1 = (yargs: Argv<CommonYargsOptions>) => {
 			//   }
 			// )
 			.command(
-				"execute <name>",
+				"execute <database>",
 				"Executed command or SQL file",
 				Execute.Options,
 				Execute.Handler
 			)
+			.command("migrations", "Interact with D1 Migrations", (yargs2) =>
+				yargs2
+					.demandCommand()
+					.command(
+						"list <database>",
+						"List your D1 migrations",
+						Migrations.ListOptions,
+						Migrations.ListHandler
+					)
+					.command(
+						"create <database> <message>",
+						"Create a new Migration",
+						Migrations.CreateOptions,
+						Migrations.CreateHandler
+					)
+					.command(
+						"apply <database>",
+						"Apply D1 Migrations",
+						Migrations.ApplyOptions,
+						Migrations.ApplyHandler
+					)
+					.epilogue(d1BetaWarning)
+			)
 			.epilogue(d1BetaWarning)
 	);
-};
+}

package/src/d1/list.tsx

@@ -1,27 +1,41 @@
-import { render } from "ink";
 import Table from "ink-table";
 import React from "react";
 import { fetchResult } from "../cfetch";
+import { withConfig } from "../config";
 import { logger } from "../logger";
 import { requireAuth } from "../user";
+import { renderToString } from "../utils/render";
 import { d1BetaWarning } from "./utils";
+import type {
+	CommonYargsArgv,
+	StrictYargsOptionsToInterface,
+} from "../yargs-types";
 import type { Database } from "./types";
-import type { ArgumentsCamelCase, Argv } from "yargs";
 
-type ListArgs = Record<string, never>;
-
-export function Options(d1ListYargs: Argv): Argv<ListArgs> {
-	return d1ListYargs.epilogue(d1BetaWarning);
+export function Options(d1ListYargs: CommonYargsArgv) {
+	return d1ListYargs
+		.option("json", {
+			describe: "return output as clean JSON",
+			type: "boolean",
+			default: false,
+		})
+		.epilogue(d1BetaWarning);
 }
 
-export async function Handler(_: ArgumentsCamelCase<ListArgs>): Promise<void> {
-	const accountId = await requireAuth({});
-	logger.log(d1BetaWarning);
-
-	const dbs: Array<Database> = await listDatabases(accountId);
+type HandlerOptions = StrictYargsOptionsToInterface<typeof Options>;
+export const Handler = withConfig<HandlerOptions>(
+	async ({ json, config }): Promise<void> => {
+		const accountId = await requireAuth(config);
+		const dbs: Array<Database> = await listDatabases(accountId);
 
-	render(<Table data={dbs}></Table>);
-}
+		if (json) {
+			logger.log(JSON.stringify(dbs, null, 2));
+		} else {
+			logger.log(d1BetaWarning);
+			logger.log(renderToString(<Table data={dbs}></Table>));
+		}
+	}
+);
 
 export const listDatabases = async (
 	accountId: string