wrangler 0.0.0-e178d6fb → 0.0.0-e1c2f5b9

package/src/d1/execute.tsx

@@ -0,0 +1,352 @@
+import { existsSync } from "node:fs";
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import chalk from "chalk";
+import { render, Static, Text } from "ink";
+import Table from "ink-table";
+import { npxImport } from "npx-import";
+import React from "react";
+import { fetchResult } from "../cfetch";
+import { withConfig } from "../config";
+import { getLocalPersistencePath } from "../dev/get-local-persistence-path";
+import { confirm, logDim } from "../dialogs";
+import { logger } from "../logger";
+import { readableRelative } from "../paths";
+import { requireAuth } from "../user";
+import * as options from "./options";
+import {
+	d1BetaWarning,
+	getDatabaseByNameOrBinding,
+	getDatabaseInfoFromConfig,
+} from "./utils";
+import type { Config, ConfigFields, DevConfig, Environment } from "../config";
+import type { Database } from "./types";
+import type splitSqlQuery from "@databases/split-sql-query";
+import type { SQL, SQLQuery } from "@databases/sql";
+import type { Statement as StatementType } from "@miniflare/d1";
+import type { createSQLiteDB as createSQLiteDBType } from "@miniflare/shared";
+import type { Argv } from "yargs";
+
+type MiniflareNpxImportTypes = [
+	{
+		Statement: typeof StatementType;
+	},
+	{
+		createSQLiteDB: typeof createSQLiteDBType;
+	}
+];
+
+export type BaseSqlExecuteArgs = {
+	config?: string;
+	database: string;
+	local?: boolean;
+	"persist-to"?: string;
+	yes?: boolean;
+};
+
+type ExecuteArgs = BaseSqlExecuteArgs & {
+	file?: string;
+	command?: string;
+};
+
+export type QueryResult = {
+	results: Record<string, string | number | boolean>[];
+	success: boolean;
+	meta?: {
+		duration?: number;
+	};
+	query?: string;
+};
+// Max number of bytes to send in a single /execute call
+const QUERY_LIMIT = 10_000;
+
+export function Options(yargs: Argv): Argv<ExecuteArgs> {
+	return options
+		.Database(yargs)
+		.option("yes", {
+			describe: 'Answer "yes" to any prompts',
+			type: "boolean",
+			alias: "y",
+		})
+		.option("local", {
+			describe:
+				"Execute commands/files against a local DB for use with wrangler dev",
+			type: "boolean",
+		})
+		.option("file", {
+			describe: "A .sql file to injest",
+			type: "string",
+		})
+		.option("command", {
+			describe: "A single SQL statement to execute",
+			type: "string",
+		})
+		.option("persist-to", {
+			describe: "Specify directory to use for local persistence (for --local)",
+			type: "string",
+			requiresArg: true,
+		});
+}
+
+function shorten(query: string | undefined, length: number) {
+	return query && query.length > length
+		? query.slice(0, length) + "..."
+		: query;
+}
+
+export async function executeSql(
+	local: undefined | boolean,
+	config: ConfigFields<DevConfig> & Environment,
+	name: string,
+	shouldPrompt: boolean | undefined,
+	persistTo: undefined | string,
+	file?: string,
+	command?: string
+) {
+	const { parser, splitter } = await loadSqlUtils();
+
+	const sql = file
+		? parser.file(file)
+		: command
+		? parser.__dangerous__rawValue(command)
+		: null;
+
+	if (!sql) throw new Error(`Error: must provide --command or --file.`);
+	if (persistTo && !local)
+		throw new Error(`Error: can't use --persist-to without --local`);
+
+	const queries = splitSql(splitter, sql);
+	if (file && sql) {
+		if (queries[0].startsWith("SQLite format 3")) {
+			//TODO: update this error to recommend using `wrangler d1 restore` when it exists
+			throw new Error(
+				"Provided file is a binary SQLite database file instead of an SQL text file.\nThe execute command can only process SQL text files.\nPlease export an SQL file from your SQLite database and try again."
+			);
+		}
+	}
+
+	return local
+		? await executeLocally(config, name, shouldPrompt, queries, persistTo)
+		: await executeRemotely(config, name, shouldPrompt, batchSplit(queries));
+}
+
+export const Handler = withConfig<ExecuteArgs>(
+	async ({
+		config,
+		database,
+		file,
+		command,
+		local,
+		persistTo,
+		yes,
+	}): Promise<void> => {
+		logger.log(d1BetaWarning);
+		if (file && command)
+			return logger.error(`Error: can't provide both --command and --file.`);
+
+		const isInteractive = process.stdout.isTTY;
+		const response: QueryResult[] | null = await executeSql(
+			local,
+			config,
+			database,
+			isInteractive && !yes,
+			persistTo,
+			file,
+			command
+		);
+
+		// Early exit if prompt rejected
+		if (!response) return;
+
+		if (isInteractive) {
+			// Render table if single result
+			render(
+				<Static items={response}>
+					{(result) => {
+						// batch results
+						if (!Array.isArray(result)) {
+							const { results, query } = result;
+
+							if (Array.isArray(results) && results.length > 0) {
+								const shortQuery = shorten(query, 48);
+								return (
+									<>
+										{shortQuery ? <Text dimColor>{shortQuery}</Text> : null}
+										<Table data={results}></Table>
+									</>
+								);
+							}
+						}
+					}}
+				</Static>
+			);
+		} else {
+			logger.log(JSON.stringify(response, null, 2));
+		}
+	}
+);
+
+async function executeLocally(
+	config: Config,
+	name: string,
+	shouldPrompt: boolean | undefined,
+	queries: string[],
+	persistTo: string | undefined
+) {
+	const localDB = getDatabaseInfoFromConfig(config, name);
+	if (!localDB) {
+		throw new Error(
+			`Can't find a DB with name/binding '${name}' in local config. Check info in wrangler.toml...`
+		);
+	}
+
+	const persistencePath = getLocalPersistencePath(
+		persistTo,
+		true,
+		config.configPath
+	);
+
+	const dbDir = path.join(persistencePath, "d1");
+	const dbPath = path.join(dbDir, `${localDB.binding}.sqlite3`);
+	const [{ Statement }, { createSQLiteDB }] =
+		await npxImport<MiniflareNpxImportTypes>(
+			["@miniflare/d1", "@miniflare/shared"],
+			logDim
+		);
+
+	if (!existsSync(dbDir) && shouldPrompt) {
+		const ok = await confirm(
+			`About to create ${readableRelative(dbPath)}, ok?`
+		);
+		if (!ok) return null;
+		await mkdir(dbDir, { recursive: true });
+	}
+
+	logger.log(`🌀 Loading DB at ${readableRelative(dbPath)}`);
+	const db = await createSQLiteDB(dbPath);
+
+	const results: QueryResult[] = [];
+	for (const sql of queries) {
+		const statement = new Statement(db, sql);
+		results.push((await statement.all()) as QueryResult);
+	}
+
+	return results;
+}
+
+async function executeRemotely(
+	config: Config,
+	name: string,
+	shouldPrompt: boolean | undefined,
+	batches: string[]
+) {
+	const multiple_batches = batches.length > 1;
+	if (multiple_batches) {
+		const warning = `⚠️  Too much SQL to send at once, this execution will be sent as ${batches.length} batches.`;
+
+		if (shouldPrompt) {
+			const ok = await confirm(
+				`${warning}\nℹ️  Each batch is sent individually and may leave your DB in an unexpected state if a later batch fails.\n⚠️  Make sure you have a recent backup. Ok to proceed?`
+			);
+			if (!ok) return null;
+			logger.log(`🌀 Let's go`);
+		} else {
+			logger.error(warning);
+		}
+	}
+
+	const accountId = await requireAuth({});
+	const db: Database = await getDatabaseByNameOrBinding(
+		config,
+		accountId,
+		name
+	);
+
+	if (shouldPrompt) {
+		logger.log(`🌀 Executing on ${name} (${db.uuid}):`);
+
+		// Don't output if shouldPrompt is undefined
+	} else if (shouldPrompt !== undefined) {
+		// Pipe to error so we don't break jq
+		logger.error(`Executing on ${name} (${db.uuid}):`);
+	}
+
+	const results: QueryResult[] = [];
+	for (const sql of batches) {
+		if (multiple_batches)
+			logger.log(
+				chalk.gray(`  ${sql.slice(0, 70)}${sql.length > 70 ? "..." : ""}`)
+			);
+
+		const result = await fetchResult<QueryResult[]>(
+			`/accounts/${accountId}/d1/database/${db.uuid}/query`,
+			{
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+					...(db.internal_env ? { "x-d1-internal-env": db.internal_env } : {}),
+				},
+				body: JSON.stringify({ sql }),
+			}
+		);
+		result.map(logResult);
+		results.push(...result);
+	}
+	return results;
+}
+
+function logResult(r: QueryResult | QueryResult[]) {
+	logger.log(
+		`🚣 Executed ${
+			Array.isArray(r) ? `${r.length} commands` : "1 command"
+		} in ${
+			Array.isArray(r)
+				? r
+						.map((d: QueryResult) => d.meta?.duration || 0)
+						.reduce((a: number, b: number) => a + b, 0)
+				: r.meta?.duration
+		}ms`
+	);
+}
+
+function splitSql(splitter: (query: SQLQuery) => SQLQuery[], sql: SQLQuery) {
+	// We have no interpolations, so convert everything to text
+	logger.log(`🌀 Mapping SQL input into an array of statements`);
+	return splitter(sql).map(
+		(q) =>
+			q.format({
+				escapeIdentifier: (_) => "",
+				formatValue: (_, __) => ({ placeholder: "", value: "" }),
+			}).text
+	);
+}
+
+function batchSplit(queries: string[]) {
+	logger.log(`🌀 Parsing ${queries.length} statements`);
+	const num_batches = Math.ceil(queries.length / QUERY_LIMIT);
+	const batches: string[] = [];
+	for (let i = 0; i < num_batches; i++) {
+		batches.push(
+			queries.slice(i * QUERY_LIMIT, (i + 1) * QUERY_LIMIT).join("; ")
+		);
+	}
+	if (num_batches > 1) {
+		logger.log(
+			`🌀 We are sending ${num_batches} batch(es) to D1 (limited to ${QUERY_LIMIT} statements per batch)`
+		);
+	}
+	return batches;
+}
+
+async function loadSqlUtils() {
+	const [
+		{ default: parser },
+		{
+			// No idea why this is doubly-nested, see https://github.com/ForbesLindesay/atdatabases/issues/255
+			default: { default: splitter },
+		},
+	] = await npxImport<
+		[{ default: SQL }, { default: { default: typeof splitSqlQuery } }]
+	>(["@databases/sql@3.2.0", "@databases/split-sql-query@1.0.3"], logDim);
+	return { parser, splitter };
+}

package/src/d1/formatTimeAgo.ts

@@ -0,0 +1,14 @@
+import TimeAgo from "javascript-time-ago";
+import en from "javascript-time-ago/locale/en";
+import prettyBytes from "pretty-bytes";
+TimeAgo.addDefaultLocale(en);
+const timeAgo = new TimeAgo("en-US");
+
+export const formatTimeAgo = (date: Date): string => {
+	const result = timeAgo.format(date);
+	return Array.isArray(result) ? result[0] : result;
+};
+
+export const formatBytes = (bytes: number): string => {
+	return prettyBytes(bytes);
+};

package/src/d1/index.ts

@@ -0,0 +1,101 @@
+import * as Backups from "./backups";
+import * as Create from "./create";
+import * as Delete from "./delete";
+import * as Execute from "./execute";
+import * as List from "./list";
+import * as Migrations from "./migrations";
+import { d1BetaWarning } from "./utils";
+import type { CommonYargsOptions } from "../yargs-types";
+import type { Argv } from "yargs";
+
+export const d1 = (yargs: Argv<CommonYargsOptions>) => {
+	return (
+		yargs
+			.command("list", "List D1 databases", List.Options, List.Handler)
+			.command(
+				"create <name>",
+				"Create D1 database",
+				Create.Options,
+				Create.Handler
+			)
+			.command(
+				"delete <name>",
+				"Delete D1 database",
+				Delete.Options,
+				Delete.Handler
+			)
+			.command("backup", "Interact with D1 Backups", (yargs2) =>
+				yargs2
+					.demandCommand()
+					.command(
+						"list <name>",
+						"List your D1 backups",
+						Backups.ListOptions,
+						Backups.ListHandler
+					)
+					.command(
+						"create <name>",
+						"Create a new D1 backup",
+						Backups.CreateOptions,
+						Backups.CreateHandler
+					)
+					.command(
+						"restore <name> <backup-id>",
+						"Restore a DB backup",
+						Backups.RestoreOptions,
+						Backups.RestoreHandler
+					)
+					.command(
+						"download <name> <backup-id>",
+						"Download a DB backup",
+						Backups.DownloadOptions,
+						Backups.DownloadHandler
+					)
+					.epilogue(d1BetaWarning)
+			)
+			// .command(
+			//   "console <name>",
+			//   "Open a Console on a D1 database",
+			//   (d1CreateYargs) => {
+			//     return d1CreateYargs.positional("name", {
+			//       describe: "The name of the DB",
+			//       type: "string",
+			//       demandOption: true,
+			//     });
+			//   },
+			//   async (_) => {
+			//     // TODO
+			//   }
+			// )
+			.command(
+				"execute <database>",
+				"Executed command or SQL file",
+				Execute.Options,
+				Execute.Handler
+			)
+			.command("migrations", "Interact with D1 Migrations", (yargs2) =>
+				yargs2
+					.demandCommand()
+					.command(
+						"list <database>",
+						"List your D1 migrations",
+						Migrations.ListOptions,
+						Migrations.ListHandler
+					)
+					.command(
+						"create <database> <message>",
+						"Create a new Migration",
+						Migrations.CreateOptions,
+						Migrations.CreateHandler
+					)
+					.command(
+						"apply <database>",
+						"Apply D1 Migrations",
+						Migrations.ApplyOptions,
+						Migrations.ApplyHandler
+					)
+					.epilogue(d1BetaWarning)
+			)
+			.epilogue(d1BetaWarning)
+	);
+};

package/src/d1/list.tsx

@@ -0,0 +1,48 @@
+import { render } from "ink";
+import Table from "ink-table";
+import React from "react";
+import { fetchResult } from "../cfetch";
+import { logger } from "../logger";
+import { requireAuth } from "../user";
+import { d1BetaWarning } from "./utils";
+import type { Database } from "./types";
+import type { ArgumentsCamelCase, Argv } from "yargs";
+
+type ListArgs = Record<string, never>;
+
+export function Options(d1ListYargs: Argv): Argv<ListArgs> {
+	return d1ListYargs.epilogue(d1BetaWarning);
+}
+
+export async function Handler(_: ArgumentsCamelCase<ListArgs>): Promise<void> {
+	const accountId = await requireAuth({});
+	logger.log(d1BetaWarning);
+
+	const dbs: Array<Database> = await listDatabases(accountId);
+
+	render(<Table data={dbs}></Table>);
+}
+
+export const listDatabases = async (
+	accountId: string
+): Promise<Array<Database>> => {
+	const pageSize = 10;
+	let page = 1;
+	const results = [];
+	while (results.length % pageSize === 0) {
+		const json: Array<Database> = await fetchResult(
+			`/accounts/${accountId}/d1/database`,
+			{},
+			new URLSearchParams({
+				per_page: pageSize.toString(),
+				page: page.toString(),
+			})
+		);
+		page++;
+		results.push(...json);
+		if (json.length < pageSize) {
+			break;
+		}
+	}
+	return results;
+};

package/src/d1/migrations/apply.tsx

@@ -0,0 +1,196 @@
+import assert from "node:assert";
+import fs from "node:fs";
+import path from "path";
+import { Box, render, Text } from "ink";
+import Table from "ink-table";
+import React from "react";
+import { withConfig } from "../../config";
+import { confirm } from "../../dialogs";
+import { CI } from "../../is-ci";
+import isInteractive from "../../is-interactive";
+import { logger } from "../../logger";
+import { requireAuth } from "../../user";
+import { createBackup } from "../backups";
+import { DEFAULT_MIGRATION_PATH, DEFAULT_MIGRATION_TABLE } from "../constants";
+import { executeSql } from "../execute";
+import { d1BetaWarning, getDatabaseInfoFromConfig } from "../utils";
+import {
+	getMigrationsPath,
+	getUnappliedMigrations,
+	initMigrationsTable,
+} from "./helpers";
+import { DatabaseWithLocal } from "./options";
+import type { ParseError } from "../../parse";
+import type { BaseSqlExecuteArgs } from "../execute";
+import type { Argv } from "yargs";
+
+export function ApplyOptions(yargs: Argv): Argv<BaseSqlExecuteArgs> {
+	return DatabaseWithLocal(yargs);
+}
+
+export const ApplyHandler = withConfig<BaseSqlExecuteArgs>(
+	async ({ config, database, local, persistTo }): Promise<void> => {
+		logger.log(d1BetaWarning);
+
+		const databaseInfo = await getDatabaseInfoFromConfig(config, database);
+		if (!databaseInfo && !local) {
+			throw new Error(
+				`Can't find a DB with name/binding '${database}' in local config. Check info in wrangler.toml...`
+			);
+		}
+
+		if (!config.configPath) {
+			return;
+		}
+
+		const migrationsPath = await getMigrationsPath(
+			path.dirname(config.configPath),
+			databaseInfo?.migrationsFolderPath ?? DEFAULT_MIGRATION_PATH,
+			false
+		);
+
+		const migrationTableName =
+			databaseInfo?.migrationsTableName ?? DEFAULT_MIGRATION_TABLE;
+		await initMigrationsTable(
+			migrationTableName,
+			local,
+			config,
+			database,
+			persistTo
+		);
+
+		const unappliedMigrations = (
+			await getUnappliedMigrations(
+				migrationTableName,
+				migrationsPath,
+				local,
+				config,
+				database,
+				persistTo
+			)
+		)
+			.map((migration) => {
+				return {
+					Name: migration,
+					Status: "🕒️",
+				};
+			})
+			.sort((a, b) => {
+				const migrationNumberA = parseInt(a.Name.split("_")[0]);
+				const migrationNumberB = parseInt(b.Name.split("_")[0]);
+				if (migrationNumberA < migrationNumberB) {
+					return -1;
+				}
+				if (migrationNumberA > migrationNumberB) {
+					return 1;
+				}
+
+				// numbers must be equal
+				return 0;
+			});
+
+		if (unappliedMigrations.length === 0) {
+			render(<Text>✅ No migrations to apply!</Text>);
+			return;
+		}
+
+		if (isInteractive() && !CI.isCI()) {
+			const ok = await confirm(
+				`About to apply ${unappliedMigrations.length} migration(s)\n` +
+					"Your database may not be available to serve requests during the migration, continue?",
+				<Box flexDirection="column">
+					<Text>Migrations to be applied:</Text>
+					<Table data={unappliedMigrations} columns={["Name"]}></Table>
+				</Box>
+			);
+			if (!ok) return;
+		}
+
+		// don't backup prod db when applying migrations locally
+		if (!local) {
+			assert(
+				databaseInfo,
+				"In non-local mode `databaseInfo` should be defined."
+			);
+			render(<Text>🕒 Creating backup...</Text>);
+			const accountId = await requireAuth({});
+			await createBackup(accountId, databaseInfo.uuid);
+		}
+
+		for (const migration of unappliedMigrations) {
+			let query = fs.readFileSync(
+				`${migrationsPath}/${migration.Name}`,
+				"utf8"
+			);
+			query += `
+								INSERT INTO ${migrationTableName} (name)
+								values ('${migration.Name}');
+						`;
+
+			let success = true;
+			let errorNotes: Array<string> = [];
+			try {
+				const response = await executeSql(
+					local,
+					config,
+					database,
+					isInteractive() && !CI.isCI(),
+					persistTo,
+					undefined,
+					query
+				);
+
+				if (response === null) {
+					// TODO:  return error
+					return;
+				}
+
+				for (const result of response) {
+					// When executing more than 1 statement, response turns into an array of QueryResult
+					if (Array.isArray(result)) {
+						for (const subResult of result) {
+							if (!subResult.success) {
+								success = false;
+							}
+						}
+					} else {
+						if (!result.success) {
+							success = false;
+						}
+					}
+				}
+			} catch (e) {
+				const err = e as ParseError;
+
+				success = false;
+				errorNotes = err.notes.map((msg) => msg.text);
+			}
+
+			migration.Status = success ? "✅" : "❌";
+
+			render(
+				<Box flexDirection="column">
+					<Table
+						data={unappliedMigrations}
+						columns={["Name", "Status"]}
+					></Table>
+					{errorNotes.length > 0 && (
+						<Box flexDirection="column">
+							<Text>&nbsp;</Text>
+							<Text>
+								❌ Migration {migration.Name} failed with following Errors
+							</Text>
+							<Table
+								data={errorNotes.map((err) => {
+									return { Error: err };
+								})}
+							></Table>
+						</Box>
+					)}
+				</Box>
+			);
+
+			if (errorNotes.length > 0) return;
+		}
+	}
+);