ultimabase 0.0.0-stage → 0.1.0

package/src/index.ts

@@ -0,0 +1,125 @@
+// The kernel's public surface (D-01).
+export type {
+	Adapter,
+	Aggregate,
+	AggregateRead,
+	Arithmetic,
+	Change,
+	Changed,
+	Check,
+	Condition,
+	CountRead,
+	Feed,
+	Filter,
+	GroupKey,
+	ListUpdate,
+	Migration,
+	Migrations,
+	Nested,
+	Order,
+	Read,
+	Row,
+	Run,
+	ScalarType,
+	Schema,
+	SchemaField,
+	SchemaModel,
+	Sort,
+	Statement,
+	Stored,
+	Transaction,
+	Write,
+} from './adapter.ts'
+export {
+	CheckFailed,
+	feedFields,
+	feedIds,
+	MigrationFailed,
+	OutOfRange,
+	RelationViolation,
+	UniqueViolation,
+} from './adapter.ts'
+export type { Logger, MasterKey } from './app-options.ts'
+export { defineApp } from './define-app.ts'
+export { defineEnum } from './define-enum.ts'
+export type {
+	FieldShapes,
+	ModelFields,
+	ModelIndexes,
+	ModelOf,
+} from './define-model.ts'
+export { defineModel } from './define-model.ts'
+export type { ErrorCode, Issue } from './errors.ts'
+export { InvalidQuery, PublicError, Refused } from './errors.ts'
+export type {
+	DecimalOptions,
+	Enum,
+	Field,
+	FieldBuilders,
+	JsonOptions,
+	JsonValue,
+	ListOptions,
+	Many,
+	ManyOptions,
+	One,
+	ScalarOptions,
+	SingleOptions,
+	StringOptions,
+} from './fields.ts'
+export { defineFunction, defineType } from './define-function.ts'
+export type {
+	FieldType,
+	FieldTypeDefinition,
+	FieldTypeOptions,
+} from './field-type.ts'
+export { defineFieldType } from './field-type.ts'
+export { isLocalHost } from './hosts.ts'
+export type {
+	Client,
+	CookieOptions,
+	FunctionPlugin,
+	FunctionShape,
+	Nullable,
+	ObjectType,
+	OutputSpec,
+	ServerFunction,
+} from './function.ts'
+export { memory } from './memory.ts'
+export type { App, FindOperation, IndexShape, Model } from './model.ts'
+export type {
+	ExposedFunction,
+	ExposedModel,
+	ExposedOutput,
+	ExposedVirtual,
+	ExposedType,
+	ModelQueries,
+	ModelUse,
+	OutputForm,
+	PluginOf,
+	PluginTypes,
+	UseOf,
+} from './plugin.ts'
+export type { Payload, Queries } from './query.ts'
+export { definePlugin } from './plugin.ts'
+export type {
+	Caller,
+	CallerRule,
+	Identity,
+	Register,
+	UserCallerKeys,
+	UserOf,
+} from './rules.ts'
+export { allOf, allow, anyOf, isMaster, master } from './rules.ts'
+export { transaction } from './transaction.ts'
+export type {
+	ArgsOf,
+	CreateDataOf,
+	IncludeOf,
+	InputOf,
+	OrderByOf,
+	OutputOf,
+	RecordOf,
+	SelectOf,
+	UpdateDataOf,
+	WhereOf,
+} from './types.ts'

package/src/internal.ts

@@ -0,0 +1,68 @@
+// What official packages read from an application (D-130), kept off the
+// documented surface: new runs of it on another database, quieter, the
+// running plugins' APIs, what raw SQL and tests on the declared database
+// need (D-137), and what live reads need (D-154).
+import type { Adapter } from './adapter.ts'
+import { internalsOf } from './app-internals.ts'
+import type { Logger } from './app-options.ts'
+import { consoleLogger } from './console-logger.ts'
+import type { Lifecycle } from './lifecycle.ts'
+import { lifecycle } from './lifecycle.ts'
+import { current } from './running.ts'
+
+export type { Lifecycle }
+export type { Scratch } from './scratch.ts'
+export { scratchable, scratchOf } from './scratch.ts'
+export { isRow } from './field-codec.ts'
+export { joining } from './transaction.ts'
+export { issueOf } from './write-data.ts'
+export { decimalPattern } from './decimal.ts'
+export type { Recorded } from './recorder.ts'
+export { recording } from './recorder.ts'
+export type { LivePlan } from './live.ts'
+export {
+	inMemory,
+	liveSchema,
+	livePlan,
+	matches,
+	servedRows,
+	sortKeys,
+	storedRows,
+} from './live.ts'
+export { compareKeys } from './memory-values.ts'
+
+/** `logger` without its info lines. */
+function quieted(logger: Logger): Logger {
+	return {
+		info: () => undefined,
+		warn: logger.warn.bind(logger),
+		error: logger.error.bind(logger),
+		child: (bindings) => quieted(logger.child(bindings)),
+	}
+}
+
+/** The application's database, and new runs of it on another (D-130). */
+export type QuietRuns = {
+	readonly declared: Adapter
+	/** A run on `database`, without info lines. */
+	readonly on: (database: Adapter) => Lifecycle
+}
+
+/**
+ * Runs of the application `defineApp` returned as `app`, or `undefined` for
+ * another value.
+ */
+export function quietRuns(app: unknown): QuietRuns | undefined {
+	const options = internalsOf(app)?.options
+	if (options === undefined) return undefined
+	const logger = quieted(options.logger ?? consoleLogger())
+	return {
+		declared: options.database,
+		on: (database) => lifecycle({ ...options, database, logger }),
+	}
+}
+
+/** The running application's plugin API named `name`, if any. */
+export function runningApi(name: string): unknown {
+	return current()?.apis.get(name)
+}

package/src/lifecycle.ts

@@ -0,0 +1,214 @@
+// `start` and `stop` (D-59, D-91): the database, the plugins' APIs, their
+// `start` and `stop`, the operations still running, and `SIGTERM`.
+import type { Adapter, Schema } from './adapter.ts'
+import type { Logger, MasterKey } from './app-options.ts'
+import type { PluginCaller } from './complete.ts'
+import { completer } from './complete.ts'
+import { consoleLogger } from './console-logger.ts'
+import { feedOf } from './feed.ts'
+import type { PluginIdentify } from './identify.ts'
+import { identifier } from './identify.ts'
+import { checkVersions } from './manifest.ts'
+import { checked, synced } from './migrate.ts'
+import type { Installed } from './plugin-app.ts'
+import {
+	buildApis,
+	installed,
+	pluginCallers,
+	pluginIdentifiers,
+	withContexts,
+} from './plugin-app.ts'
+import type { Running } from './running.ts'
+import { register, unregister, Work } from './running.ts'
+import { resolve } from './schema.ts'
+
+export type Lifecycle = {
+	readonly start: () => Promise<void>
+	readonly stop: () => Promise<void>
+}
+
+type Started = {
+	readonly app: Running
+	readonly plugins: readonly Installed[]
+	readonly onSigterm: () => void
+}
+
+/** Stops `plugins` in reverse order, each even when another fails. */
+async function stopped(plugins: readonly Installed[]): Promise<unknown[]> {
+	const errors: unknown[] = []
+	for (const { code, context } of plugins.toReversed()) {
+		if (code.stop === undefined) continue
+		try {
+			await Reflect.apply(code.stop, undefined, [context])
+		} catch (error) {
+			context.logger.error({ err: error }, 'The plugin failed to stop.')
+			errors.push(error)
+		}
+	}
+	return errors
+}
+
+export type LifecycleOptions = {
+	readonly database: Adapter
+	readonly plugins?: readonly unknown[]
+	/** The application's `defineFunction` values (D-27). */
+	readonly functions?: readonly unknown[]
+	readonly models: readonly unknown[]
+	readonly logger?: Logger
+	readonly masterKey?: MasterKey
+}
+
+/** Once stdout and stderr have written what they were given. */
+const written = () =>
+	Promise.all(
+		[process.stdout, process.stderr].map(
+			(stream) => new Promise((done) => stream.write('', done)),
+		),
+	)
+
+/** The application's schema, or a message on what is wrong with it. */
+export function schemaOf(options: LifecycleOptions): Schema {
+	const { plugins } = installed(options.plugins ?? [])
+	return resolve(
+		options.models,
+		plugins.map((plugin) => plugin.code),
+	).schema
+}
+
+/** The kernel's logger: the application's, or JSON lines. */
+export const kernelLogger = (options: LifecycleOptions): Logger =>
+	(options.logger ?? consoleLogger()).child({ name: 'ultimabase' })
+
+/** Syncs a development database, or checks a production one (D-105). */
+async function follow(
+	database: Adapter,
+	schema: Schema,
+	logger: Logger,
+): Promise<void> {
+	try {
+		await (process.env['NODE_ENV'] === 'production'
+			? checked(database, schema, logger)
+			: synced(database, schema, logger, false))
+	} catch (error) {
+		await database.stop()
+		throw error
+	}
+}
+
+export function lifecycle(options: LifecycleOptions): Lifecycle {
+	const { database, models } = options
+	let started: Started | undefined
+	let stopping: Promise<void> | undefined
+	let starting = false
+
+	const shutDown = async ({ app, plugins, onSigterm }: Started) => {
+		process.off('SIGTERM', onSigterm)
+		const errors = await stopped(plugins)
+		// A plugin's stop may still query; the database closes last.
+		await app.work.settled()
+		unregister(app)
+		started = undefined
+		await database.stop()
+		app.logger.info('Application stopped.')
+		if (errors.length > 0) throw errors[0]
+	}
+
+	const stop = async (): Promise<void> => {
+		if (stopping !== undefined) return stopping
+		if (started === undefined) return
+		stopping = shutDown(started).finally(() => {
+			stopping = undefined
+		})
+		return stopping
+	}
+
+	const start = async (): Promise<void> => {
+		if (started !== undefined || starting) {
+			throw new Error('The application is running.')
+		}
+		// Set before the first await: a second start meanwhile is refused.
+		starting = true
+		try {
+			await launch()
+		} finally {
+			starting = false
+		}
+	}
+
+	const launch = async (): Promise<void> => {
+		const { plugins: records, apiOrder } = installed(options.plugins ?? [])
+		const resolved = resolve(
+			models,
+			records.map((plugin) => plugin.code),
+		)
+		const base = options.logger ?? consoleLogger()
+		const logger = base.child({ name: 'ultimabase' })
+		checkVersions(records, logger)
+		// Plugins identify with their contexts, which hold identify.
+		const identifiers: PluginIdentify[] = []
+		const callers: PluginCaller[] = []
+		const complete = completer(callers)
+		const identify = identifier(
+			options.masterKey,
+			logger,
+			identifiers,
+			complete,
+		)
+		const { plugins, functions } = withContexts(
+			apiOrder,
+			base,
+			identify,
+			resolved,
+			options.functions ?? [],
+		)
+		identifiers.push(...pluginIdentifiers(records, plugins))
+		callers.push(...pluginCallers(records, plugins))
+		await database.start(resolved.schema, logger, feedOf(records, plugins))
+		const apis = new Map<string, unknown>()
+		const app: Running = {
+			resolved,
+			adapter: database,
+			logger,
+			apis,
+			functions: functions.definitions,
+			work: new Work(),
+			complete,
+		}
+		const ran: Installed[] = []
+		try {
+			await follow(database, resolved.schema, logger)
+			register(app)
+			// An API may read the APIs it needs as it is built (D-89).
+			buildApis(plugins, apis)
+			for (const plugin of plugins) {
+				const { code, context } = plugin
+				if (code.start !== undefined) {
+					await Reflect.apply(code.start, undefined, [context])
+				}
+				ran.push(plugin)
+			}
+		} catch (error) {
+			await stopped(ran)
+			unregister(app)
+			await database.stop()
+			throw error
+		}
+		const onSigterm = () => {
+			void stop()
+				.catch((error: unknown) => {
+					logger.error({ err: error }, 'The application stopped with an error.')
+				})
+				// The last lines first: a pipe may write them later.
+				.then(written)
+				.finally(() => {
+					// Its listener is gone: the signal now ends the process.
+					process.kill(process.pid, 'SIGTERM')
+				})
+		}
+		process.once('SIGTERM', onSigterm)
+		started = { app, plugins, onSigterm }
+		logger.info('Application started.')
+	}
+
+	return { start, stop }
+}

package/src/list-update.ts

@@ -0,0 +1,71 @@
+// List updates (D-149): `{ set }`, `{ push }`, `{ pushUnique }` or
+// `{ remove }` on a list field, with an element or a list of them, computed
+// on the row as the write finds it, as a number update is (D-132).
+import type { ListUpdate } from './adapter.ts'
+import { InvalidQuery } from './errors.ts'
+import { isObject } from './query-where.ts'
+import type { ModelInfo } from './schema.ts'
+import type { Draft } from './write-data.ts'
+import { invalid, issueOf } from './write-data.ts'
+
+const operations = ['set', 'push', 'pushUnique', 'remove'] as const
+type Operation = (typeof operations)[number]
+
+const isOperation = (key: unknown): key is Operation =>
+	operations.some((operation) => operation === key)
+
+/**
+ * Takes the operations out of an update's draft: `{ set }` becomes its
+ * list, an operation by `undefined` leaves its field as it is (D-84), and
+ * each other one comes back to be computed on the row.
+ */
+export function listUpdates(
+	model: ModelInfo,
+	draft: Draft,
+	at: readonly (string | number)[] = [],
+): Map<string, ListUpdate> {
+	const lists = new Map<string, ListUpdate>()
+	for (const [name, value] of Object.entries(draft)) {
+		const field = model.fields.get(name)
+		// A field type's value is its own, as for numbers (D-132).
+		const plain = field?.kind === 'scalar' && field.list && !field.codec
+		if (!plain || !isObject(value)) continue
+		const keys = Object.keys(value)
+		const used = operations.filter((key) => value[key] !== undefined)
+		if (keys.length === 0 || used.length > 1 || !keys.every(isOperation)) {
+			const path = ['data', ...at, name].join('.')
+			throw new InvalidQuery(
+				path,
+				`${path} takes a list or one of ${operations.join(', ')}.`,
+			)
+		}
+		delete draft[name]
+		const [operation] = used
+		if (operation === undefined) continue
+		const given = value[operation]
+		const elements: readonly unknown[] = Array.isArray(given) ? given : [given]
+		const issue = issueOf(elements, field)
+		if (issue !== undefined) {
+			throw invalid([{ path: [...at, name], message: issue }])
+		}
+		if (operation === 'set') draft[name] = elements
+		else lists.set(name, { operation, elements })
+	}
+	return lists
+}
+
+/** A list update's result on `current`, the elements in their order. */
+export function listed(
+	current: unknown,
+	{ operation, elements }: ListUpdate,
+): unknown[] {
+	const from: readonly unknown[] = Array.isArray(current) ? current : []
+	if (operation === 'push') return [...from, ...elements]
+	if (operation === 'remove') {
+		return from.filter((item) => !elements.includes(item))
+	}
+	const added = elements.filter(
+		(item, index) => !from.includes(item) && elements.indexOf(item) === index,
+	)
+	return [...from, ...added]
+}

package/src/live.ts

@@ -0,0 +1,199 @@
+// What the realtime plugin asks the kernel (D-154): a live read's effective
+// filter and order, planned once for its caller, and, on a stored row, that
+// filter and the row's sort keys, tested as `memory()` reads.
+import { accessFor, and } from './access.ts'
+import type { Filter, Nested, Order, Read, Row, Schema } from './adapter.ts'
+import { rootFind } from './engine.ts'
+import { decoded, isRow } from './field-codec.ts'
+import { Reader } from './memory-read.ts'
+import { compareKeys, sortable } from './memory-values.ts'
+import { isFilters } from './memory-conditions.ts'
+import { plan, uniqueWhere } from './query-args.ts'
+import { modelOf } from './query-where.ts'
+import { isObject } from './query-values.ts'
+import type { Caller } from './rules.ts'
+import type { Running } from './running.ts'
+import { current } from './running.ts'
+import { QueryScope, planned } from './scope.ts'
+
+/**
+ * The rows a live read follows, and their order, ending with `id`; `filter`
+ * is the read's own part of `where`, `rights` the part the caller's read
+ * rules and `beforeFind` add, and
+ * `include`, `count` and `fields` what its selection reads, as the adapter
+ * receives them.
+ */
+export type LivePlan = Pick<Nested, 'include' | 'count' | 'fields'> & {
+	readonly where: Filter | undefined
+	readonly filter: Filter | undefined
+	readonly rights: Filter | undefined
+	readonly orderBy: readonly Order[]
+}
+
+function started(): Running {
+	const app = current()
+	if (app === undefined) throw new Error('No application is running.')
+	return app
+}
+
+const shapeKeys = ['select', 'include', 'omit']
+const liveKeys = new Set(['where', 'orderBy', ...shapeKeys])
+const rowKeys = new Set(shapeKeys)
+
+/**
+ * `args` as `caller` reads `model`: its read rules and every `beforeFind`
+ * included. A live row's `where` names one row, as `findUnique`'s.
+ */
+export async function livePlan(
+	model: string,
+	args: unknown,
+	caller: Caller,
+	row = false,
+): Promise<LivePlan> {
+	const app = started()
+	const { models } = app.resolved
+	const info = modelOf(models, model)
+	const access = accessFor(models, caller, app.logger)
+	const scope = new QueryScope(models, caller, access)
+	const { where: unique, ...shape } = isObject(args) ? args : {}
+	return app.work.run(async () =>
+		planned(scope, () => {
+			const { read } = row
+				? plan(models, info, shape, '', 'many', rowKeys, scope)
+				: plan(models, info, args, '', 'many', liveKeys, scope)
+			const rights = scope.readable(model)
+			const where = row
+				? uniqueWhere(models, info, unique, 'where', scope)
+				: read.where
+			const { include, count, fields } = read
+			return {
+				...(include && { include }),
+				...(count && { count }),
+				...(fields && { fields }),
+				where: and(where, rights),
+				filter: where,
+				rights,
+				orderBy: read.orderBy ?? [{ field: 'id', direction: 'asc' }],
+			}
+		}),
+	)
+}
+
+/** The stored rows `read` finds, as Master, plugin fields undecoded. */
+export async function storedRows(
+	model: string,
+	read: Nested,
+): Promise<readonly Row[]> {
+	const app = started()
+	return app.work.run(async () => app.adapter.find({ ...read, model }))
+}
+
+/** The running application's schema, its plugins' models included. */
+export function liveSchema(): Schema {
+	return started().resolved.schema
+}
+
+/** Whether `filter` reads `model`'s row alone, so `matches` can test it. */
+export function inMemory(model: string, filter: Filter | undefined): boolean {
+	const fields = modelOf(started().resolved.models, model).fields
+	return Object.entries(filter ?? {}).every(([key, value]) => {
+		if (value === undefined) return true
+		if (isFilters(value)) return value.every((part) => inMemory(model, part))
+		// A search ranks as each database does (D-148).
+		if (value.search !== undefined || value.removedBy !== undefined) {
+			return false
+		}
+		const field = fields.get(key)
+		return field === undefined || field.kind === 'scalar'
+	})
+}
+
+const readers = new WeakMap<Running, Reader>()
+
+function readerOf(app: Running): Reader {
+	const known = readers.get(app)
+	if (known !== undefined) return known
+	const fields = new Map(
+		app.resolved.schema.models.map((model) => [
+			model.name,
+			new Map(model.fields.map((field) => [field.name, field])),
+		]),
+	)
+	const reader = new Reader({ tables: new Map(), joins: new Map() }, fields)
+	readers.set(app, reader)
+	return reader
+}
+
+/** Whether the stored `row` of `model` passes `filter`, which `inMemory` takes. */
+export function matches(
+	model: string,
+	row: Row,
+	filter: Filter | undefined,
+): boolean {
+	return readerOf(started()).matches(model, row, filter) === true
+}
+
+/** Whether rows alone answer `read`: one level, unpaged, read in memory. */
+const servable = (read: Read): boolean =>
+	read.include === undefined &&
+	read.count === undefined &&
+	read.distinct === undefined &&
+	read.cursor === undefined &&
+	read.skip === undefined &&
+	read.take === undefined &&
+	(read.orderBy ?? []).every(
+		(order) => (order.through ?? order.count ?? order.relevance) === undefined,
+	) &&
+	inMemory(read.model, read.where)
+
+/**
+ * The stored `rows` that pass `read`, in its order; the find that asked for
+ * them keeps the fields its shape selects.
+ */
+function served(read: Read, rows: Iterable<Row>): Row[] {
+	const { model, orderBy = [] } = read
+	const kept = [...rows].filter((row) => matches(model, row, read.where))
+	const keys = new Map(kept.map((row) => [row, sortKeys(model, row, orderBy)]))
+	const key = (row: Row): unknown[] => keys.get(row) ?? []
+	kept.sort((a, b) => compareKeys(key(a), key(b), orderBy))
+	return kept
+}
+
+/**
+ * `findMany(args)` on `model` as `caller`, its rows taken from the stored
+ * `rows` when they alone answer it, else from the database. `rows` holds
+ * every row its `where` finds.
+ */
+export async function servedRows(
+	model: string,
+	args: unknown,
+	caller: Caller,
+	rows: ReadonlyMap<string, Row>,
+): Promise<readonly Row[]> {
+	const { models } = started().resolved
+	const found = await rootFind(
+		model,
+		args,
+		{ as: caller },
+		async (store, read) =>
+			servable(read)
+				? decoded(models, read, served(read, rows.values()))
+				: store.find(read),
+	)
+	return Array.isArray(found) ? found.filter(isRow) : []
+}
+
+/** The sort keys of the stored `row`, for an order on its own fields. */
+export function sortKeys(
+	model: string,
+	row: Row,
+	orderBy: readonly Order[],
+): unknown[] {
+	const reader = readerOf(started())
+	return orderBy.map((order) => {
+		if (order.through ?? order.count ?? order.relevance) {
+			throw new Error(`The order by ${order.field} reads another row.`)
+		}
+		return sortable(row[order.field] ?? null, reader.field(model, order.field))
+	})
+}