wrangler 2.1.6 → 2.1.8

package/src/d1/execute.tsx

@@ -0,0 +1,294 @@
+import { existsSync } from "node:fs";
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import chalk from "chalk";
+import { render, Static, Text } from "ink";
+import Table from "ink-table";
+import { npxImport } from "npx-import";
+import React from "react";
+import { fetchResult } from "../cfetch";
+import { withConfig } from "../config";
+import { getLocalPersistencePath } from "../dev/get-local-persistence-path";
+import { confirm, logDim } from "../dialogs";
+import { logger } from "../logger";
+import { readableRelative } from "../paths";
+import { requireAuth } from "../user";
+import { Name } from "./options";
+import {
+	d1BetaWarning,
+	getDatabaseByNameOrBinding,
+	getDatabaseInfoFromConfig,
+} from "./utils";
+import type { Config } from "../config";
+import type { Database } from "./types";
+import type splitSqlQuery from "@databases/split-sql-query";
+import type { SQL, SQLQuery } from "@databases/sql";
+import type { Statement as StatementType } from "@miniflare/d1";
+import type { createSQLiteDB as createSQLiteDBType } from "@miniflare/shared";
+import type { Argv } from "yargs";
+
+type MiniflareNpxImportTypes = [
+	{
+		Statement: typeof StatementType;
+	},
+	{
+		createSQLiteDB: typeof createSQLiteDBType;
+	}
+];
+
+type ExecuteArgs = {
+	config?: string;
+	name: string;
+	file?: string;
+	command?: string;
+	local?: boolean;
+	"persist-to"?: string;
+};
+
+type QueryResult = {
+	results: Record<string, string | number | boolean>[];
+	success: boolean;
+	duration: number;
+	query?: string;
+};
+// Max number of bytes to send in a single /execute call
+const QUERY_LIMIT = 1_000_000; // 1MB
+
+export function Options(yargs: Argv): Argv<ExecuteArgs> {
+	return Name(yargs)
+		.option("local", {
+			describe:
+				"Execute commands/files against a local DB for use with wrangler dev",
+			type: "boolean",
+		})
+		.option("file", {
+			describe: "A .sql file to injest",
+			type: "string",
+		})
+		.option("command", {
+			describe: "A single SQL statement to execute",
+			type: "string",
+		})
+		.option("persist-to", {
+			describe: "Specify directory to use for local persistence (for --local)",
+			type: "string",
+			requiresArg: true,
+		});
+}
+
+function shorten(query: string | undefined, length: number) {
+	return query && query.length > length
+		? query.slice(0, length) + "..."
+		: query;
+}
+
+export const Handler = withConfig<ExecuteArgs>(
+	async ({ config, name, file, command, local, persistTo }): Promise<void> => {
+		logger.log(d1BetaWarning);
+		if (file && command)
+			return console.error(`Error: can't provide both --command and --file.`);
+		const { parser, splitter } = await loadSqlUtils();
+
+		const sql = file
+			? parser.file(file)
+			: command
+			? parser.__dangerous__rawValue(command)
+			: null;
+
+		if (!sql) throw new Error(`Error: must provide --command or --file.`);
+		if (persistTo && !local)
+			throw new Error(`Error: can't use --persist-to without --local`);
+
+		const isInteractive = process.stdout.isTTY;
+		const response: QueryResult[] | null = local
+			? await executeLocally(
+					config,
+					name,
+					isInteractive,
+					splitSql(splitter, sql),
+					persistTo
+			  )
+			: await executeRemotely(
+					config,
+					name,
+					isInteractive,
+					batchSplit(splitter, sql)
+			  );
+
+		// Early exit if prompt rejected
+		if (!response) return;
+
+		if (isInteractive) {
+			render(
+				<Static items={response}>
+					{(result) => {
+						const { results, duration, query } = result;
+
+						if (Array.isArray(results) && results.length > 0) {
+							const shortQuery = shorten(query, 48);
+							return (
+								<>
+									{shortQuery ? <Text dimColor>{shortQuery}</Text> : null}
+									<Table data={results}></Table>
+								</>
+							);
+						} else {
+							const shortQuery = shorten(query, 24);
+							return (
+								<Text>
+									Executed{" "}
+									{shortQuery ? <Text dimColor>{shortQuery}</Text> : "command"}{" "}
+									in {duration}ms.
+								</Text>
+							);
+						}
+					}}
+				</Static>
+			);
+		} else {
+			console.log(JSON.stringify(response, null, 2));
+		}
+	}
+);
+
+async function executeLocally(
+	config: Config,
+	name: string,
+	isInteractive: boolean,
+	queries: string[],
+	persistTo: string | undefined
+) {
+	const localDB = getDatabaseInfoFromConfig(config, name);
+	if (!localDB) {
+		throw new Error(
+			`Can't find a DB with name/binding '${name}' in local config. Check info in wrangler.toml...`
+		);
+	}
+
+	const persistencePath = getLocalPersistencePath(
+		persistTo,
+		true,
+		config.configPath
+	);
+
+	const dbDir = path.join(persistencePath, "d1");
+	const dbPath = path.join(dbDir, `${localDB.binding}.sqlite3`);
+	const [{ Statement }, { createSQLiteDB }] =
+		await npxImport<MiniflareNpxImportTypes>(
+			["@miniflare/d1", "@miniflare/shared"],
+			logDim
+		);
+
+	if (!existsSync(dbDir) && isInteractive) {
+		const ok = await confirm(
+			`About to create ${readableRelative(dbPath)}, ok?`
+		);
+		if (!ok) return null;
+		await mkdir(dbDir, { recursive: true });
+	}
+
+	console.log(`Loading DB at ${readableRelative(dbPath)}`);
+	const db = await createSQLiteDB(dbPath);
+
+	const results: QueryResult[] = [];
+	for (const sql of queries) {
+		const statement = new Statement(db, sql);
+		results.push((await statement.all()) as QueryResult);
+	}
+
+	return results;
+}
+
+async function executeRemotely(
+	config: Config,
+	name: string,
+	isInteractive: boolean,
+	batches: string[]
+) {
+	if (batches.length > 1) {
+		const warning =
+			chalk.red(`WARNING! `) +
+			`Too much SQL to send at once, this execution will be sent as ${batches.length} batches.`;
+
+		if (isInteractive) {
+			const ok = await confirm(
+				`${warning}\nNOTE: each batch is sent individually and may leave your DB in an unexpected state if a later batch fails.\n${chalk.green(
+					`Make sure you have a recent backup.`
+				)}\nOk to proceed?`
+			);
+			if (!ok) return null;
+		} else {
+			console.error(warning);
+		}
+	}
+
+	const accountId = await requireAuth({});
+	const db: Database = await getDatabaseByNameOrBinding(
+		config,
+		accountId,
+		name
+	);
+
+	if (isInteractive) {
+		console.log(`Executing on ${name} (${db.uuid}):`);
+	} else {
+		// Pipe to error so we don't break jq
+		console.error(`Executing on ${name} (${db.uuid}):`);
+	}
+
+	const results: QueryResult[] = [];
+	for (const sql of batches) {
+		results.push(
+			...(await fetchResult<QueryResult[]>(
+				`/accounts/${accountId}/d1/database/${db.uuid}/query`,
+				{
+					method: "POST",
+					headers: {
+						"Content-Type": "application/json",
+					},
+					body: JSON.stringify({ sql }),
+				}
+			))
+		);
+	}
+	return results;
+}
+
+function splitSql(splitter: (query: SQLQuery) => SQLQuery[], sql: SQLQuery) {
+	// We have no interpolations, so convert everything to text
+	return splitter(sql).map(
+		(q) =>
+			q.format({
+				escapeIdentifier: (_) => "",
+				formatValue: (_, __) => ({ placeholder: "", value: "" }),
+			}).text
+	);
+}
+
+function batchSplit(splitter: typeof splitSqlQuery, sql: SQLQuery) {
+	const queries = splitSql(splitter, sql);
+
+	const batches: string[] = [];
+	for (const query of queries) {
+		const last = batches.at(-1);
+		if (!last || last.length + query.length > QUERY_LIMIT) {
+			batches.push(query);
+		} else {
+			batches.splice(-1, 1, [last, query].join("; "));
+		}
+	}
+	return batches;
+}
+
+async function loadSqlUtils() {
+	const [
+		{ default: parser },
+		{
+			// No idea why this is doubly-nested, see https://github.com/ForbesLindesay/atdatabases/issues/255
+			default: { default: splitter },
+		},
+	] = await npxImport<
+		[{ default: SQL }, { default: { default: typeof splitSqlQuery } }]
+	>(["@databases/sql@3.2.0", "@databases/split-sql-query@1.0.3"], logDim);
+	return { parser, splitter };
+}

package/src/d1/formatTimeAgo.ts

@@ -0,0 +1,14 @@
+import TimeAgo from "javascript-time-ago";
+import en from "javascript-time-ago/locale/en";
+import prettyBytes from "pretty-bytes";
+TimeAgo.addDefaultLocale(en);
+const timeAgo = new TimeAgo("en-US");
+
+export const formatTimeAgo = (date: Date): string => {
+	const result = timeAgo.format(date);
+	return Array.isArray(result) ? result[0] : result;
+};
+
+export const formatBytes = (bytes: number): string => {
+	return prettyBytes(bytes);
+};

package/src/d1/index.ts

@@ -0,0 +1,75 @@
+import * as Backups from "./backups";
+import * as Create from "./create";
+import * as Delete from "./delete";
+import * as Execute from "./execute";
+import * as List from "./list";
+import { d1BetaWarning } from "./utils";
+import type { Argv } from "yargs";
+
+export const d1api = (yargs: Argv) => {
+	return (
+		yargs
+			.command("list", "List D1 databases", List.Options, List.Handler)
+			.command(
+				"create <name>",
+				"Create D1 database",
+				Create.Options,
+				Create.Handler
+			)
+			.command(
+				"delete <name>",
+				"Delete D1 database",
+				Delete.Options,
+				Delete.Handler
+			)
+			.command("backup", "Interact with D1 Backups", (yargs2) =>
+				yargs2
+					.command(
+						"list <name>",
+						"List your D1 backups",
+						Backups.ListOptions,
+						Backups.ListHandler
+					)
+					.command(
+						"create <name>",
+						"Create a new D1 backup",
+						Backups.CreateOptions,
+						Backups.CreateHandler
+					)
+					.command(
+						"restore <name> <backup-id>",
+						"Restore a DB backup",
+						Backups.RestoreOptions,
+						Backups.RestoreHandler
+					)
+					.command(
+						"download <name> <backup-id>",
+						"Download a DB backup",
+						Backups.DownloadOptions,
+						Backups.DownloadHandler
+					)
+					.epilogue(d1BetaWarning)
+			)
+			// .command(
+			//   "console <name>",
+			//   "Open a Console on a D1 database",
+			//   (d1CreateYargs) => {
+			//     return d1CreateYargs.positional("name", {
+			//       describe: "The name of the DB",
+			//       type: "string",
+			//       demandOption: true,
+			//     });
+			//   },
+			//   async (_) => {
+			//     // TODO
+			//   }
+			// )
+			.command(
+				"execute <name>",
+				"Executed command or SQL file",
+				Execute.Options,
+				Execute.Handler
+			)
+			.epilogue(d1BetaWarning)
+	);
+};

package/src/d1/list.tsx

@@ -0,0 +1,48 @@
+import { render } from "ink";
+import Table from "ink-table";
+import React from "react";
+import { fetchResult } from "../cfetch";
+import { logger } from "../logger";
+import { requireAuth } from "../user";
+import { d1BetaWarning } from "./utils";
+import type { Database } from "./types";
+import type { ArgumentsCamelCase, Argv } from "yargs";
+
+type ListArgs = Record<string, never>;
+
+export function Options(d1ListYargs: Argv): Argv<ListArgs> {
+	return d1ListYargs.epilogue(d1BetaWarning);
+}
+
+export async function Handler(_: ArgumentsCamelCase<ListArgs>): Promise<void> {
+	const accountId = await requireAuth({});
+	logger.log(d1BetaWarning);
+
+	const dbs: Array<Database> = await listDatabases(accountId);
+
+	render(<Table data={dbs}></Table>);
+}
+
+export const listDatabases = async (
+	accountId: string
+): Promise<Array<Database>> => {
+	const pageSize = 10;
+	let page = 1;
+	const results = [];
+	while (results.length % pageSize === 0) {
+		const json: Array<Database> = await fetchResult(
+			`/accounts/${accountId}/d1/database`,
+			{},
+			new URLSearchParams({
+				per_page: pageSize.toString(),
+				page: page.toString(),
+			})
+		);
+		page++;
+		results.push(...json);
+		if (json.length < pageSize) {
+			break;
+		}
+	}
+	return results;
+};

package/src/d1/options.ts

@@ -0,0 +1,12 @@
+import { d1BetaWarning } from "./utils";
+import type { Argv } from "yargs";
+
+export function Name(yargs: Argv) {
+	return yargs
+		.positional("name", {
+			describe: "The name or binding of the DB",
+			type: "string",
+			demandOption: true,
+		})
+		.epilogue(d1BetaWarning);
+}

package/src/d1/types.tsx

@@ -0,0 +1,14 @@
+export type Database = {
+	uuid: string;
+	name: string;
+};
+
+export type Backup = {
+	id: string;
+	database_id: string;
+	created_at: string;
+	state: "progress" | "done";
+	num_tables: number;
+	file_size: number;
+	size?: string;
+};

package/src/d1/utils.ts

@@ -0,0 +1,39 @@
+import { listDatabases } from "./list";
+import type { Config } from "../config";
+import type { Database } from "./types";
+
+export function getDatabaseInfoFromConfig(config: Config, name: string) {
+	for (const d1Database of config.d1_databases) {
+		if (
+			d1Database.database_id &&
+			(name === d1Database.database_name || name === d1Database.binding)
+		) {
+			return {
+				uuid: d1Database.database_id,
+				binding: d1Database.binding,
+				name: d1Database.database_name,
+			};
+		}
+	}
+	return null;
+}
+
+export const getDatabaseByNameOrBinding = async (
+	config: Config,
+	accountId: string,
+	name: string
+): Promise<Database> => {
+	const dbFromConfig = getDatabaseInfoFromConfig(config, name);
+	if (dbFromConfig) return dbFromConfig;
+
+	const allDBs = await listDatabases(accountId);
+	const matchingDB = allDBs.find((db) => db.name === name);
+	if (!matchingDB) {
+		throw new Error(`Couldn't find DB with name '${name}'`);
+	}
+	return matchingDB;
+};
+
+export const d1BetaWarning = process.env.NO_D1_WARNING
+	? ""
+	: "🚧 'wrangler d1 <command>' is a beta command. Please report any issues to https://github.com/cloudflare/wrangler2/issues/new/choose";

package/src/dev/dev.tsx

@@ -5,7 +5,7 @@ import { watch } from "chokidar";
 import clipboardy from "clipboardy";
 import commandExists from "command-exists";
 import { Box, Text, useApp, useInput, useStdin } from "ink";
-import React, { useState, useEffect, useRef } from "react";
+import React, { useState, useEffect, useRef, useMemo } from "react";
 import { withErrorBoundary, useErrorHandler } from "react-error-boundary";
 import onExit from "signal-exit";
 import tmp from "tmp-promise";
@@ -145,7 +145,6 @@ export type DevProps = {
 	host: string | undefined;
 	routes: Route[] | undefined;
 	inspect: boolean;
-	logLevel: "none" | "error" | "log" | "warn" | "debug" | undefined;
 	logPrefix?: string;
 	onReady: ((ip: string, port: number) => void) | undefined;
 	showInteractiveDevSession: boolean | undefined;
@@ -154,6 +153,7 @@ export type DevProps = {
 	firstPartyWorker: boolean | undefined;
 	sendMetrics: boolean | undefined;
 	testScheduled: boolean | undefined;
+	experimentalLocal: boolean | undefined;
 };
 
 export function DevImplementation(props: DevProps): JSX.Element {
@@ -214,12 +214,36 @@ function InteractiveDevSession(props: DevProps) {
 
 type DevSessionProps = DevProps & {
 	local: boolean;
+	experimentalLocal?: boolean;
 };
 
 function DevSession(props: DevSessionProps) {
 	useCustomBuild(props.entry, props.build);
 
 	const directory = useTmpDir();
+	const handleError = useErrorHandler();
+
+	// Note: when D1 is out of beta, this (and all instances of `betaD1Shims`) can be removed.
+	// Additionally, useMemo is used so that new arrays aren't created on every render
+	// cause re-rendering further down.
+	const betaD1Shims = useMemo(
+		() => props.bindings.d1_databases?.map((db) => db.binding),
+		[props.bindings.d1_databases]
+	);
+	const everyD1BindingHasPreview = props.bindings.d1_databases?.every(
+		(binding) => binding.preview_database_id
+	);
+	if (
+		betaD1Shims &&
+		betaD1Shims.length > 0 &&
+		!(props.local || everyD1BindingHasPreview)
+	) {
+		handleError(
+			new Error(
+				"D1 bindings require dev --local or preview_database_id for now"
+			)
+		);
+	}
 
 	const workerDefinitions = useDevRegistry(
 		props.name,
@@ -240,6 +264,7 @@ function DevSession(props: DevSessionProps) {
 		tsconfig: props.tsconfig,
 		minify: props.minify,
 		nodeCompat: props.nodeCompat,
+		betaD1Shims,
 		define: props.define,
 		noBundle: props.noBundle,
 		assets: props.assetsConfig,
@@ -247,9 +272,11 @@ function DevSession(props: DevSessionProps) {
 		services: props.bindings.services,
 		durableObjects: props.bindings.durable_objects || { bindings: [] },
 		firstPartyWorkerDevFacade: props.firstPartyWorker,
+		local: props.local,
 		// Enable the bundling to know whether we are using dev or publish
 		targetConsumer: "dev",
 		testScheduled: props.testScheduled ?? false,
+		experimentalLocalStubCache: props.local && props.experimentalLocal,
 	});
 
 	return props.local ? (
@@ -272,11 +299,11 @@ function DevSession(props: DevSessionProps) {
 			crons={props.crons}
 			localProtocol={props.localProtocol}
 			localUpstream={props.localUpstream}
-			logLevel={props.logLevel}
 			logPrefix={props.logPrefix}
 			inspect={props.inspect}
 			onReady={props.onReady}
 			enablePagesAssetsServiceBinding={props.enablePagesAssetsServiceBinding}
+			experimentalLocal={props.experimentalLocal}
 		/>
 	) : (
 		<Remote

package/src/dev/get-local-persistence-path.tsx

@@ -0,0 +1,31 @@
+import path from "node:path";
+
+export function getLocalPersistencePath(
+	persistTo: string | undefined,
+	doPersist: true,
+	configPath: string | undefined
+): string;
+
+export function getLocalPersistencePath(
+	persistTo: string | undefined,
+	doPersist: boolean,
+	configPath: string | undefined
+): string | null;
+
+export function getLocalPersistencePath(
+	persistTo: string | undefined,
+	doPersist: boolean,
+	configPath: string | undefined
+) {
+	return persistTo
+		? // If path specified, always treat it as relative to cwd()
+		  path.resolve(process.cwd(), persistTo)
+		: doPersist
+		? // If just flagged on, treat it as relative to wrangler.toml,
+		  // if one can be found, otherwise cwd()
+		  path.resolve(
+				configPath ? path.dirname(configPath) : process.cwd(),
+				".wrangler/state"
+		  )
+		: null;
+}