pipework 0.290.0 → 0.291.0

package/src/data/db/function-source.ts

@@ -9,6 +9,7 @@ import type { InferFieldOutput } from '../domain/brand.js'
 import type { FieldDescriptor } from '../domain/types.js'
 import { projectTable } from '../domain/build-table.js'
 import { camelToSnake } from '../domain/naming.js'
+import { assertReadFunction, FUNCTION_DEFINITION, type FunctionDefinition, type FunctionVolatility } from './function-definition.js'
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- variance-safe bound for descriptors with any output/flags
 type AnyField = FieldDescriptor<any, any>
@@ -21,6 +22,38 @@ export interface FunctionSpec<TArgs extends readonly AnyField[], TColumns extend
   readonly args: readonly [...TArgs]
   /** One `pipe.field.*` descriptor per result column; keys are camelCase, columns are their snake_case. */
   readonly columns: TColumns
+  /** The body of a function pipework creates: one `LANGUAGE sql` SELECT that refers to the arguments as `$1`, `$2`, ... Without it the function is only referenced (it exists already). */
+  readonly body?: string
+  /** `'stable'` (the default) or `'immutable'`; a read is never volatile. Only with `body`. */
+  readonly volatility?: FunctionVolatility
+  /** The typical number of rows one call returns, a planner estimate (ROWS). Required with `body`. */
+  readonly rows?: number
+  /** Only `'sql'`: Postgres inlines no other language. */
+  readonly language?: 'sql'
+}
+
+/** Declares a SQL function that returns one value. See {@link fn}. */
+export interface ScalarFunctionSpec<TArgs extends readonly AnyField[], TReturn extends AnyField> {
+  readonly args: readonly [...TArgs]
+  readonly returns: TReturn
+  readonly body: string
+  readonly volatility?: FunctionVolatility
+  readonly language?: 'sql'
+}
+
+/** A function callable with typed arguments; `definition` is set when pipework creates it (the spec had a `body`). */
+export type DefinedCall<TCall> = TCall & {
+  readonly definition: FunctionDefinition | undefined
+  /** Where the function lives and its argument types; what a plan or catalog check looks it up by. */
+  readonly ref: FunctionRef
+}
+
+/** A function's schema, name and SQL argument types. */
+export interface FunctionRef {
+  readonly schema: string
+  readonly name: string
+  readonly args: readonly string[]
+  readonly setReturning: boolean
 }
 
 /** A column whose data type is `T` (and that cannot be null unless `T` can). */
@@ -56,7 +89,7 @@ function quote(name: string): string {
 }
 
 /**
- * Declares a set-returning SQL function and returns a callable that yields a source for `.from()` and joins.
+ * Declares a SQL function and returns a callable. A set-returning one (`columns`) yields a source for `.from()` and joins.
  * Each argument is a value, a column or a SQL expression of the declared type. A value is bound as a parameter,
  * converted the way a column of that field is on insert (jsonb as JSON text, dates, bigints, arrays), and cast to its
  * declared SQL type: `ext.f($1::uuid, $2::text)`. A null binds SQL NULL. An array argument (`field.uuid().array()`) is
@@ -64,10 +97,15 @@ function quote(name: string): string {
  * `unnest(t.arr::uuid[])`, so an array column can be unnested in a lateral join (`.crossJoinLateral(unnestUuids(t.arr))`).
  * Result columns are typed from `columns`; the default alias is the function's unqualified name.
  */
+export function fn<const TArgs extends readonly AnyField[], TReturn extends AnyField>(
+  name: string,
+  spec: ScalarFunctionSpec<TArgs, TReturn>,
+): DefinedCall<(...values: ArgValues<TArgs>) => SQL<InferFieldOutput<TReturn>>>
 export function fn<const TArgs extends readonly AnyField[], TColumns extends Record<string, AnyField>>(
   name: string,
   spec: FunctionSpec<TArgs, TColumns>,
-): (...values: ArgValues<TArgs>) => FunctionSource<string, TColumns> {
+): DefinedCall<(...values: ArgValues<TArgs>) => FunctionSource<string, TColumns>>
+export function fn(name: string, spec: FunctionSpec<readonly AnyField[], Record<string, AnyField>> | ScalarFunctionSpec<readonly AnyField[], AnyField>): unknown {
   const parts = name.split('.')
   if (parts.length > 2 || parts.some(p => !NAME_PART.test(p))) {
     fail(
@@ -75,29 +113,58 @@ export function fn<const TArgs extends readonly AnyField[], TColumns extends Rec
       'use "fn" or "schema.fn", each part lowercase letters, digits and underscores, not starting with a digit.',
     )
   }
-  const keys = Object.keys(spec.columns)
-  if (keys.length === 0) fail(`pipe.fn("${name}"): columns is empty.`, 'declare at least one result column.')
+  const scalar = 'returns' in spec ? spec.returns : undefined
+  const columns: Record<string, AnyField> = 'columns' in spec ? spec.columns : {}
+  const keys = Object.keys(columns)
+  if (scalar === undefined && keys.length === 0) fail(`pipe.fn("${name}"): columns is empty.`, 'declare at least one result column.')
+  if (scalar !== undefined && 'columns' in spec) fail(`pipe.fn("${name}"): both returns and columns are declared.`, 'declare `returns` for a scalar function or `columns` for a set-returning one.')
   for (const key of keys) {
     if (key === 'as' || key === '_') {
       fail(`pipe.fn("${name}"): a result column cannot be named "${key}".`, 'rename the key; the column name comes from camelToSnake(key).')
     }
   }
 
+  const sqlTypeOf = (fields: Record<string, AnyField>): Record<string, string> => {
+    const cols = getTableColumns(projectTable('pw_fn_args', fields, {})) as Record<string, { getSQLType(): string }>
+    return Object.fromEntries(Object.entries(cols).map(([k, c]) => {
+      const t = c.getSQLType()
+      return [k, t === 'serial' ? 'integer' : t === 'bigserial' ? 'bigint' : t]
+    }))
+  }
+
   const argFields: Record<string, AnyField> = {}
   spec.args.forEach((a, i) => { argFields[`a${i}`] = a })
   const argCols = getTableColumns(projectTable('pw_fn_args', argFields, {})) as Record<string, { getSQLType(): string; mapToDriverValue(value: unknown): unknown }>
-  const argTypes = spec.args.map((_, i) => {
-    const t = argCols[`a${i}`]!.getSQLType()
-    return t === 'serial' ? 'integer' : t === 'bigserial' ? 'bigint' : t
-  })
+  const argTypes = spec.args.map((_, i) => sqlTypeOf({ a: spec.args[i]! })['a']!)
+
+  let definition: FunctionDefinition | undefined
+  if (spec.body !== undefined) {
+    if (typeof spec.body !== 'string') fail(`pipe.fn("${name}"): body must be a string.`, 'pass the SELECT as a string, with arguments as $1, $2, ...')
+    assertReadFunction({ name, argCount: spec.args.length, body: spec.body, spec: spec as unknown as Record<string, unknown>, setReturning: scalar === undefined })
+    const returnTypes = scalar === undefined ? sqlTypeOf(columns) : sqlTypeOf({ r: scalar })
+    definition = {
+      schema: parts.length === 2 ? parts[0]! : 'public',
+      name: parts[parts.length - 1]!,
+      args: argTypes,
+      returns: scalar === undefined
+        ? { kind: 'table', columns: keys.map(k => ({ name: camelToSnake(k), type: returnTypes[k]! })), rows: (spec as FunctionSpec<readonly AnyField[], Record<string, AnyField>>).rows! }
+        : { kind: 'scalar', type: returnTypes['r']! },
+      body: spec.body,
+      volatility: spec.volatility ?? 'stable',
+    }
+  } else {
+    for (const key of ['volatility', 'rows', 'language'] as const) {
+      if (key in spec) fail(`pipe.fn("${name}"): "${key}" is given without a body.`, 'add `body` to have pipework create the function, or remove the option (a referenced function keeps what it was created with).')
+    }
+  }
 
   const fnSql = sql.raw(parts.map(quote).join('.'))
   const defaultAlias = parts[parts.length - 1]!
+  const argList = (values: readonly unknown[]): SQL => sql.join(values.map((v, i) => sql`${bindIfParam(v, argCols[`a${i}`] as unknown as Column)}::${sql.raw(argTypes[i]!)}`), sql`, `)
 
-  const build = (values: readonly unknown[], alias: string): FunctionSource<string, TColumns> => {
+  const build = (values: readonly unknown[], alias: string): FunctionSource<string, Record<string, AnyField>> => {
     const columnList = sql.join(keys.map(k => sql.raw(quote(camelToSnake(k)))), sql`, `)
-    const argList = sql.join(values.map((v, i) => sql`${bindIfParam(v, argCols[`a${i}`] as unknown as Column)}::${sql.raw(argTypes[i]!)}`), sql`, `)
-    const inner = sql`select ${columnList} from ${fnSql}(${argList})`
+    const inner = sql`select ${columnList} from ${fnSql}(${argList(values)})`
     const fields: Record<string, SQL> = {}
     for (const k of keys) {
       fields[k] = sql.raw(`${quote(alias)}.${quote(camelToSnake(k))}`)
@@ -109,16 +176,21 @@ export function fn<const TArgs extends readonly AnyField[], TColumns extends Rec
         if (prop === 'as') return (next: string) => build(values, next)
         return selection.get(target, prop)
       },
-    }) as unknown as FunctionSource<string, TColumns>
+    }) as unknown as FunctionSource<string, Record<string, AnyField>>
   }
 
-  return (...values: ArgValues<TArgs>) => {
+  const call = (...values: unknown[]): unknown => {
     if (values.length !== spec.args.length) {
       fail(
         `pipe.fn("${name}"): called with ${values.length} argument(s), declared ${spec.args.length}.`,
         'pass one value per declared argument.',
       )
     }
-    return build(values, defaultAlias)
+    return scalar !== undefined ? sql`${fnSql}(${argList(values)})` : build(values, defaultAlias)
   }
+  return Object.defineProperties(call, {
+    definition: { value: definition, enumerable: true },
+    ref: { value: { schema: parts.length === 2 ? parts[0]! : 'public', name: defaultAlias, args: argTypes, setReturning: scalar === undefined }, enumerable: true },
+    [FUNCTION_DEFINITION]: { value: definition },
+  })
 }

package/src/data/db/namespace.ts

@@ -88,7 +88,7 @@ export const pipe = Object.assign(getDb, {
   /** A typed rowset from parallel array parameters, one parameter per column whatever the row count, as a source for `.from()` and joins — `pipe.rows({ g: 'text', id: 'int' }, { g: [...], id: [...] })` renders `unnest($1::text[], $2::int[]) AS rows(g, id)`. For composite-key matches that would otherwise be a union or a JS-side tuple match. */
   rows,
 
-  /** A set-returning SQL function as a typed source for `.from()` and joins — `pipe.fn('schema.f', { args: [field.uuid()], columns: { path: field.text() } })`. Arguments are bound and cast to their declared types. */
+  /** A set-returning SQL function as a typed source for `.from()` and joins — `pipe.fn('schema.f', { args: [field.uuid()], columns: { path: field.text() } })`. Arguments are bound and cast to their declared types. With a `body` (one LANGUAGE sql SELECT, `volatility`, `rows`) pipework creates the function and migrates it, under rules that keep it inlinable; check it with `expectFunctionRead` from `pipework/test`. */
   fn,
 
   /** Field builders for domain definitions — uuid(), text(), integer(), etc. */

package/src/data/migrate/diff.ts

@@ -1,3 +1,4 @@
+import type { FunctionDefinition } from '../db/function-definition.js'
 import type {
   PwSnapshot,
   PwTableSnapshot,
@@ -10,6 +11,8 @@ import type {
 } from './snapshot.js'
 
 export type DiffOp =
+  | { type: 'create_function'; fn: FunctionDefinition }
+  | { type: 'drop_function'; fn: FunctionDefinition }
   | { type: 'create_table'; table: PwTableSnapshot }
   | { type: 'drop_table'; table: PwTableSnapshot }
   | { type: 'add_column'; tableName: string; tableSchema: string; column: PwColumnSnapshot }
@@ -39,11 +42,16 @@ export type DiffOp =
 
 /** The wire name of the table an op acts on, across all `DiffOp` shapes. */
 export function diffOpTableName(op: DiffOp): string {
+  if (op.type === 'create_function' || op.type === 'drop_function') return ''
   return op.type === 'create_table' || op.type === 'drop_table' ? op.table.name : op.tableName
 }
 
 export function diffSnapshots(prev: PwSnapshot, cur: PwSnapshot): DiffOp[] {
   const ops: DiffOp[] = []
+  const functionOps = diffFunctions(prev.functions ?? {}, cur.functions ?? {})
+
+  // Function drops come first: a function that reads a table must be gone before the table or a column it reads is.
+  ops.push(...functionOps.filter(op => op.type === 'drop_function'))
 
   const prevKeys = new Set(Object.keys(prev.tables))
   const curKeys = new Set(Object.keys(cur.tables))
@@ -93,6 +101,43 @@ export function diffSnapshots(prev: PwSnapshot, cur: PwSnapshot): DiffOp[] {
     diffAddConstraints(prevTable, curTable, ops)
   }
 
+  // Function creates come last: a LANGUAGE sql body is checked when the function is created, so every table,
+  // column and index it names must exist by then.
+  ops.push(...functionOps.filter(op => op.type === 'create_function'))
+
+  return ops
+}
+
+/** What a function returns, without its ROWS estimate (which CREATE OR REPLACE can change). */
+function shape(fn: FunctionDefinition): unknown {
+  return fn.returns.kind === 'table' ? { columns: fn.returns.columns } : fn.returns
+}
+
+/**
+ * A function is identified by `schema.name(argtypes)`. A new key is created; a key with a changed body, volatility
+ * or ROWS is replaced in place; a changed result shape cannot be replaced (Postgres refuses to change a return
+ * type), so it is dropped and created; a key that is gone is dropped. A changed argument list is a different key,
+ * so the old signature is dropped and the new one created.
+ */
+function diffFunctions(
+  prev: Record<string, FunctionDefinition>,
+  cur: Record<string, FunctionDefinition>,
+): DiffOp[] {
+  const ops: DiffOp[] = []
+  for (const [key, old] of Object.entries(prev)) {
+    const now = cur[key]
+    if (now === undefined || JSON.stringify(shape(now)) !== JSON.stringify(shape(old))) {
+      ops.push({ type: 'drop_function', fn: old })
+    }
+  }
+  for (const [key, now] of Object.entries(cur)) {
+    const old = prev[key]
+    if (old === undefined) {
+      ops.push({ type: 'create_function', fn: now })
+    } else if (JSON.stringify(old) !== JSON.stringify(now)) {
+      ops.push({ type: 'create_function', fn: now })
+    }
+  }
   return ops
 }
 

package/src/data/migrate/history.ts

@@ -2,6 +2,7 @@ import { existsSync, readFileSync, readdirSync } from 'node:fs'
 import { join } from 'node:path'
 import { EMPTY_SNAPSHOT, type PwSnapshot, type PwTableSnapshot, type PwPartitionSnapshot } from './snapshot.js'
 import type { DoorSnapshot } from './door-generate.js'
+import type { FunctionDefinition } from '../db/function-definition.js'
 
 export interface MigrationEntry {
   tag: string
@@ -29,6 +30,7 @@ export interface SnapshotDelta {
   version: 'pw1-delta'
   tables: Record<string, TableDelta | null>
   doors?: Record<string, DoorSnapshot | null>
+  functions?: Record<string, FunctionDefinition | null>
 }
 
 const MAP_KEYS = ['columns', 'indexes', 'foreignKeys', 'checks', 'uniqueConstraints', 'compositePrimaryKeys'] as const
@@ -148,6 +150,10 @@ export function snapshotDelta(prev: PwSnapshot, next: PwSnapshot): SnapshotDelta
     const doors = diffMap<DoorSnapshot>(prev.doors ?? {}, next.doors ?? {})
     if (Object.keys(doors).length > 0) delta.doors = doors
   }
+  if (next.functions !== undefined || prev.functions !== undefined) {
+    const functions = diffMap<FunctionDefinition>(prev.functions ?? {}, next.functions ?? {})
+    if (Object.keys(functions).length > 0) delta.functions = functions
+  }
   return delta
 }
 
@@ -185,6 +191,9 @@ export function applyDelta(snap: PwSnapshot, delta: SnapshotDelta): PwSnapshot {
   if (delta.doors !== undefined) {
     result.doors = applyMap<DoorSnapshot>(snap.doors ?? {}, delta.doors) as NonNullable<PwSnapshot['doors']>
   }
+  if (delta.functions !== undefined) {
+    result.functions = applyMap<FunctionDefinition>(snap.functions ?? {}, delta.functions)
+  }
   return result
 }
 

package/src/data/migrate/snapshot.ts

@@ -10,6 +10,7 @@ import { Column } from '../query/column.js'
 import type { IndexedColumn } from '../query/pg-core/columns/common.js'
 import { camelToSnake } from '../domain/naming.js'
 import type { DefineOptions } from '../domain/types.js'
+import { FUNCTION_DEFINITION, functionKey, type FunctionDefinition } from '../db/function-definition.js'
 
 export interface PwSnapshot {
   version: 'pw1'
@@ -18,6 +19,8 @@ export interface PwSnapshot {
   tables: Record<string, PwTableSnapshot>
   /** Cross-tenant read doors with a generated policy; absent in snapshots written before doors existed. */
   doors?: Record<string, { table: string; role: string; ops?: readonly ('select' | 'update' | 'delete')[] }>
+  /** Functions pipework defines (`pipe.fn` with a `body`), by `schema.name(argtypes)`; absent in snapshots written before functions existed. */
+  functions?: Record<string, FunctionDefinition>
 }
 
 export interface PwTableSnapshot {
@@ -166,8 +169,21 @@ export function buildSnapshot(
   prevId: string,
 ): PwSnapshot {
   const tables: Record<string, PwTableSnapshot> = {}
+  const functions: Record<string, FunctionDefinition> = {}
 
   for (const value of Object.values(imports)) {
+    const definition = typeof value === 'function' ? (value as { [FUNCTION_DEFINITION]?: FunctionDefinition })[FUNCTION_DEFINITION] : undefined
+    if (definition !== undefined) {
+      const key = functionKey(definition)
+      const seen = functions[key]
+      if (seen !== undefined && JSON.stringify(seen) !== JSON.stringify(definition)) {
+        throw new Error(
+          `[pipework] Two different definitions of the function ${key} are exported from the schema files.\n\n  Fix: define it once and import it where it is called.\n`,
+        )
+      }
+      functions[key] = definition
+      continue
+    }
     if (!is(value, PgTable)) continue
     if (isExternal(value)) continue
 
@@ -281,6 +297,7 @@ export function buildSnapshot(
     id: randomUUID(),
     prevId,
     tables,
+    ...(Object.keys(functions).length > 0 ? { functions } : {}),
   }
 }
 

package/src/data/migrate/sql-emit.ts

@@ -1,6 +1,8 @@
 import type {
   DiffOp,
 } from './diff.js'
+import { bodyQuote } from '../db/function-definition.js'
+import type { FunctionDefinition } from '../db/function-definition.js'
 import type {
   PwTableSnapshot,
   PwColumnSnapshot,
@@ -24,7 +26,7 @@ function qualifiedTable(schema: string, name: string): string {
  * Every op type except the two `emitSQL` routes itself: `create_table` (whose
  * foreign keys are hoisted out) and `add_foreign_key`. Issue #312.
  */
-type InlineOp = Exclude<DiffOp, { type: 'create_table' } | { type: 'add_foreign_key' }>
+type InlineOp = Exclude<DiffOp, { type: 'create_table' } | { type: 'add_foreign_key' } | { type: 'create_function' } | { type: 'drop_function' }>
 
 export function emitSQL(ops: DiffOp[]): string[] {
   // Foreign keys are hoisted to run after every CREATE TABLE. A from-scratch
@@ -34,8 +36,15 @@ export function emitSQL(ops: DiffOp[]): string[] {
   // tables in place before any constraint is added. Issue #312.
   const tables: string[] = []
   const foreignKeys: string[] = []
+  // Function drops lead and function creates trail, whatever order the ops come in; see diffSnapshots.
+  const functionDrops: string[] = []
+  const functionCreates: string[] = []
   for (const op of ops) {
-    if (op.type === 'create_table') {
+    if (op.type === 'drop_function') {
+      functionDrops.push(emitDropFunction(op.fn))
+    } else if (op.type === 'create_function') {
+      functionCreates.push(emitCreateFunction(op.fn))
+    } else if (op.type === 'create_table') {
       tables.push(...emitCreateTable(op.table))
       const partitioned = op.table.partitionBy !== undefined
       for (const fk of Object.values(op.table.foreignKeys)) {
@@ -47,7 +56,7 @@ export function emitSQL(ops: DiffOp[]): string[] {
       tables.push(...emitOp(op))
     }
   }
-  return [...tables, ...foreignKeys]
+  return [...functionDrops, ...tables, ...foreignKeys, ...functionCreates]
 }
 
 function emitOp(op: InlineOp): string[] {
@@ -390,3 +399,24 @@ function emitAddCompositePK(tableName: string, tableSchema: string, pk: PwCompos
 function emitReplaceCompositePK(tableName: string, tableSchema: string, from: PwCompositePKSnapshot, to: PwCompositePKSnapshot, partitioned: boolean): string[] {
   return emitKeyConstraint(tableName, tableSchema, 'PRIMARY KEY', to, partitioned, from.name)
 }
+
+function qualifiedFunction(fn: FunctionDefinition): string {
+  return `${q(fn.schema)}.${q(fn.name)}`
+}
+
+function emitDropFunction(fn: FunctionDefinition): string {
+  return `DROP FUNCTION ${qualifiedFunction(fn)}(${fn.args.join(', ')})`
+}
+
+/**
+ * `CREATE OR REPLACE FUNCTION`, always `LANGUAGE sql`, STABLE or IMMUTABLE, no SET clause, not SECURITY DEFINER, not STRICT:
+ * the form Postgres inlines into the calling query. The body is dollar-quoted so the statement scanner keeps it whole.
+ */
+function emitCreateFunction(fn: FunctionDefinition): string {
+  const returns = fn.returns.kind === 'scalar'
+    ? fn.returns.type
+    : `TABLE(${fn.returns.columns.map(c => `${q(c.name)} ${c.type}`).join(', ')})`
+  const rows = fn.returns.kind === 'table' ? ` ROWS ${fn.returns.rows}` : ''
+  const tag = bodyQuote(fn.body)
+  return `CREATE OR REPLACE FUNCTION ${qualifiedFunction(fn)}(${fn.args.join(', ')}) RETURNS ${returns} LANGUAGE sql ${fn.volatility.toUpperCase()}${rows} AS ${tag}${fn.body}${tag}`
+}

package/src/index.ts

@@ -11,7 +11,8 @@ export type { Aggregate, EmptyableAggregate, AggregateOptions, JsonOf } from './
 export type { ValueColumnDef } from './data/db/values.js'
 export type { RowsColumns, RowsData, RowsSource, RowsColumnValue } from './data/db/rows.js'
 export type { ArraySource, ElementOf, JsonbElements } from './data/db/array-elements.js'
-export type { FunctionSpec, FunctionSource, FunctionSourceFields } from './data/db/function-source.js'
+export type { FunctionSpec, ScalarFunctionSpec, FunctionSource, FunctionSourceFields, DefinedCall, FunctionRef } from './data/db/function-source.js'
+export type { FunctionDefinition, FunctionVolatility } from './data/db/function-definition.js'
 export type { SerializableOptions } from './data/db/serializable.js'
 export { StatementTimeoutError, LockTimeoutError } from './data/db/timeouts.js'
 export type { TimeoutOptions } from './data/db/timeouts.js'