sovrium 0.0.2

package/src/infrastructure/database/table-operations/column-generators.ts

@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) 2025 ESSENTIAL SERVICES
+ *
+ * This source code is licensed under the Business Source License 1.1
+ * found in the LICENSE.md file in the root directory of this source tree.
+ */
+
+import type { Table } from '@/domain/models/app/table'
+
+/**
+ * Generate automatic id column definition based on primary key type
+ * @param primaryKeyType - Type of primary key (uuid, bigserial, or default serial)
+ * @param isPrimaryKey - Whether this id column is the primary key (adds PRIMARY KEY constraint inline)
+ */
+export const generateIdColumn = (
+  primaryKeyType: string | undefined,
+  isPrimaryKey: boolean
+): string => {
+  const pkConstraint = isPrimaryKey ? ' PRIMARY KEY' : ''
+  if (primaryKeyType === 'uuid') {
+    return `id UUID NOT NULL DEFAULT gen_random_uuid()${pkConstraint}`
+  }
+  if (primaryKeyType === 'bigserial') {
+    return `id BIGSERIAL NOT NULL${pkConstraint}`
+  }
+  return `id SERIAL NOT NULL${pkConstraint}`
+}
+
+/**
+ * Determine if table needs an automatic id column
+ * Creates automatic id column if:
+ * - No explicit id field is defined in the fields array
+ * - Either: no primary key is defined OR primary key references the special 'id' field
+ */
+export const needsAutomaticIdColumn = (
+  table: Table,
+  primaryKeyFields: readonly string[]
+): boolean => {
+  const hasIdField = table.fields.some((field) => field.name === 'id')
+  const primaryKeyReferencesId = primaryKeyFields.includes('id')
+  const hasNonIdPrimaryKey = primaryKeyFields.length > 0 && !primaryKeyReferencesId
+  return !hasIdField && !hasNonIdPrimaryKey
+}
+
+/**
+ * Generate created_at column definition if not explicitly defined
+ * Note: Includes DEFAULT CURRENT_TIMESTAMP to support INSERT ... DEFAULT VALUES
+ * Triggers also set the value to ensure consistency
+ */
+export const generateCreatedAtColumn = (table: Table): readonly string[] => {
+  const hasCreatedAtField = table.fields.some((field) => field.name === 'created_at')
+  return !hasCreatedAtField ? ['created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP'] : []
+}
+
+/**
+ * Generate updated_at column definition if not explicitly defined
+ * Note: Includes DEFAULT CURRENT_TIMESTAMP to support INSERT ... DEFAULT VALUES
+ * Triggers update the value on INSERT and UPDATE to ensure currency
+ */
+export const generateUpdatedAtColumn = (table: Table): readonly string[] => {
+  const hasUpdatedAtField = table.fields.some((field) => field.name === 'updated_at')
+  return !hasUpdatedAtField ? ['updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP'] : []
+}
+
+/**
+ * Generate deleted_at column definition if not explicitly defined
+ */
+export const generateDeletedAtColumn = (table: Table): readonly string[] => {
+  const hasDeletedAtField = table.fields.some((field) => field.name === 'deleted_at')
+  return !hasDeletedAtField ? ['deleted_at TIMESTAMPTZ'] : []
+}
+
+/**
+ * Generate primary key constraint if needed
+ */
+export const generatePrimaryKeyConstraintIfNeeded = (
+  table: Table,
+  primaryKeyFields: readonly string[]
+): readonly string[] =>
+  needsAutomaticIdColumn(table, primaryKeyFields) && primaryKeyFields.length === 0
+    ? ['PRIMARY KEY (id)']
+    : []

package/src/infrastructure/database/table-operations/create-table-sql.ts

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) 2025 ESSENTIAL SERVICES
+ *
+ * This source code is licensed under the Business Source License 1.1
+ * found in the LICENSE.md file in the root directory of this source tree.
+ */
+
+import { shouldCreateDatabaseColumn, sanitizeTableName } from '../field-utils'
+import { shouldUseView, getBaseTableName } from '../lookup/lookup-view-generators'
+import { generateColumnDefinition, generateTableConstraints } from '../sql/sql-generators'
+import {
+  generateIdColumn,
+  needsAutomaticIdColumn,
+  generateCreatedAtColumn,
+  generateUpdatedAtColumn,
+  generateDeletedAtColumn,
+  generatePrimaryKeyConstraintIfNeeded,
+} from './column-generators'
+import type { Table } from '@/domain/models/app/table'
+
+/**
+ * Generate CREATE TABLE statement
+ * When table has lookup fields, creates a base table (_base suffix) and will later create a VIEW
+ *
+ * @param table - Table definition
+ * @param tableUsesView - Map of table names to whether they use a VIEW
+ * @param skipForeignKeys - Skip foreign key constraints (for circular dependencies)
+ */
+export const generateCreateTableSQL = (
+  table: Table,
+  tableUsesView?: ReadonlyMap<string, boolean>,
+  skipForeignKeys?: boolean,
+  hasAuthConfig: boolean = true
+): string => {
+  // Sanitize table name for PostgreSQL (lowercase, underscores)
+  const sanitized = sanitizeTableName(table.name)
+  // Determine table name (add _base suffix if using VIEW for lookup fields)
+  const tableName = shouldUseView(table) ? getBaseTableName(sanitized) : sanitized
+
+  // Identify primary key fields
+  const primaryKeyFields =
+    table.primaryKey?.type === 'composite' ? (table.primaryKey.fields ?? []) : []
+
+  // Generate automatic id column based on primary key type
+  // Add PRIMARY KEY inline if the primary key is on the 'id' field
+  const primaryKeyOnId = primaryKeyFields.length === 1 && primaryKeyFields[0] === 'id'
+  const idColumnDefinition = needsAutomaticIdColumn(table, primaryKeyFields)
+    ? [generateIdColumn(table.primaryKey?.type, primaryKeyOnId)]
+    : []
+
+  // Filter out UI-only fields (like button), lookup fields, and rollup fields (handled by VIEW)
+  // Lookup and rollup fields don't exist as columns in the base table
+  const columnDefinitions = table.fields
+    .filter(
+      (field) =>
+        shouldCreateDatabaseColumn(field) && field.type !== 'lookup' && field.type !== 'rollup'
+    )
+    .map((field) => {
+      // Only add inline PRIMARY KEY for single-field composite keys (handled by generateSerialColumn)
+      // Multi-field composite keys must have PRIMARY KEY at table level to avoid "multiple primary keys" error
+      const isPrimaryKey = primaryKeyFields.includes(field.name) && primaryKeyFields.length === 1
+      return generateColumnDefinition(field, isPrimaryKey, table.fields, hasAuthConfig)
+    })
+
+  // Add PRIMARY KEY constraint on id if no custom primary key is defined
+  const tableConstraints = generateTableConstraints(table, tableUsesView, skipForeignKeys)
+
+  const allDefinitions = [
+    ...idColumnDefinition,
+    ...generateCreatedAtColumn(table),
+    ...generateUpdatedAtColumn(table),
+    ...generateDeletedAtColumn(table),
+    ...columnDefinitions,
+    ...tableConstraints,
+    ...generatePrimaryKeyConstraintIfNeeded(table, primaryKeyFields),
+  ]
+
+  return `CREATE TABLE IF NOT EXISTS ${tableName} (
+  ${allDefinitions.join(',\n  ')}
+)`
+}

package/src/infrastructure/database/table-operations/index.ts

@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) 2025 ESSENTIAL SERVICES
+ *
+ * This source code is licensed under the Business Source License 1.1
+ * found in the LICENSE.md file in the root directory of this source tree.
+ */
+
+/**
+ * Table Operations Module
+ *
+ * This module provides utilities for database table operations:
+ * - CREATE TABLE statement generation
+ * - Table migration (ALTER TABLE, data preservation)
+ * - Table feature application (indexes, triggers, RLS policies)
+ * - Lookup VIEW creation
+ * - User-defined VIEW creation
+ *
+ * All functions use Effect for type-safe error handling.
+ */
+
+// Type compatibility utilities
+export { normalizeType, areTypesCompatible } from './type-compatibility'
+
+// Column generators
+export {
+  generateIdColumn,
+  needsAutomaticIdColumn,
+  generateCreatedAtColumn,
+  generateUpdatedAtColumn,
+  generateDeletedAtColumn,
+  generatePrimaryKeyConstraintIfNeeded,
+} from './column-generators'
+
+// CREATE TABLE SQL generation
+export { generateCreateTableSQL } from './create-table-sql'
+
+// Table features (indexes, triggers, RLS policies)
+export { applyTableFeatures, applyTableFeaturesWithoutIndexes } from './table-features'
+
+// Migration utilities
+export {
+  getCompatibleColumns,
+  copyDataAndResetSequences,
+  recreateTableWithDataEffect,
+} from './migration-utils'
+
+// Table effect operations
+export type { MigrationConfig } from './table-effects'
+export {
+  migrateExistingTableEffect,
+  createNewTableEffect,
+  createLookupViewsEffect,
+  createTableViewsEffect,
+  createOrMigrateTableEffect,
+} from './table-effects'

package/src/infrastructure/database/table-operations/migration-utils.ts

@@ -0,0 +1,157 @@
+/**
+ * Copyright (c) 2025 ESSENTIAL SERVICES
+ *
+ * This source code is licensed under the Business Source License 1.1
+ * found in the LICENSE.md file in the root directory of this source tree.
+ */
+
+import { Effect } from 'effect'
+import { sanitizeTableName } from '../field-utils'
+import { shouldUseView, getBaseTableName } from '../lookup/lookup-view-generators'
+import { executeSQL, SQLExecutionError, type TransactionLike } from '../sql/sql-execution'
+import { generateCreateTableSQL } from './create-table-sql'
+import { areTypesCompatible } from './type-compatibility'
+import type { Table } from '@/domain/models/app/table'
+
+/**
+ * Get compatible columns between existing and new table for data migration
+ */
+export const getCompatibleColumns = (
+  existingColumns: ReadonlyMap<
+    string,
+    { dataType: string; isNullable: string; columnDefault: string | null }
+  >,
+  newColumnInfo: ReadonlyMap<string, { columnDefault: string | null; dataType: string }>
+): readonly string[] =>
+  Array.from(existingColumns.keys()).filter((col) => {
+    const newColInfo = newColumnInfo.get(col)
+    if (!newColInfo) return false
+    const oldType = existingColumns.get(col)?.dataType.toLowerCase() ?? ''
+    return areTypesCompatible(oldType, newColInfo.dataType.toLowerCase())
+  })
+
+interface CopyDataParams {
+  readonly tx: TransactionLike
+  readonly tempTableName: string
+  readonly physicalTableName: string
+  readonly commonColumns: readonly string[]
+  readonly newColumnInfo: ReadonlyMap<string, { columnDefault: string | null; dataType: string }>
+}
+
+/**
+ * Copy data and reset SERIAL sequences for migration
+ */
+export const copyDataAndResetSequences = (
+  params: CopyDataParams
+): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    const { tx, tempTableName, physicalTableName, commonColumns, newColumnInfo } = params
+    if (commonColumns.length === 0) return
+
+    const columnList = commonColumns.join(', ')
+    yield* executeSQL(
+      tx,
+      `INSERT INTO ${tempTableName} (${columnList}) SELECT ${columnList} FROM ${physicalTableName}`
+    )
+
+    // Reset SERIAL sequences to max value + 1 to avoid conflicts
+    const serialColumns = commonColumns.filter((col) =>
+      newColumnInfo.get(col)?.columnDefault?.includes('nextval')
+    )
+    yield* Effect.forEach(serialColumns, (col) =>
+      executeSQL(
+        tx,
+        `SELECT setval(pg_get_serial_sequence('${tempTableName}', '${col}'), COALESCE((SELECT MAX(${col}) FROM ${tempTableName}), 1), true)`
+      )
+    )
+  })
+
+/** Get column metadata from a table */
+const fetchColumnInfo = (
+  tx: TransactionLike,
+  tableName: string
+): Effect.Effect<
+  Map<string, { columnDefault: string | null; dataType: string }>,
+  SQLExecutionError
+> =>
+  Effect.gen(function* () {
+    const columns = yield* executeSQL(
+      tx,
+      `SELECT column_name, column_default, data_type FROM information_schema.columns WHERE table_name = '${tableName}'`
+    )
+    return new Map(
+      (
+        columns as readonly {
+          column_name: string
+          column_default: string | null
+          data_type: string
+        }[]
+      ).map((row) => [
+        row.column_name,
+        { columnDefault: row.column_default, dataType: row.data_type },
+      ])
+    )
+  })
+
+/** Finalize table recreation by dropping old table and renaming temp table */
+const finalizeTableRecreation = (
+  tx: TransactionLike,
+  physicalTableName: string,
+  tempTableName: string
+): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    yield* executeSQL(tx, `DROP TABLE ${physicalTableName} CASCADE`)
+    yield* executeSQL(tx, `ALTER TABLE ${tempTableName} RENAME TO ${physicalTableName}`)
+    yield* executeSQL(
+      tx,
+      `DO $$ BEGIN ALTER TABLE ${physicalTableName} RENAME CONSTRAINT ${tempTableName}_pkey TO ${physicalTableName}_pkey; EXCEPTION WHEN undefined_object THEN NULL; END $$`
+    )
+  })
+
+/**
+ * Recreate table with data preservation when schema changes are incompatible with ALTER TABLE
+ * Used when primary key type changes or other incompatible schema modifications occur
+ */
+export const recreateTableWithDataEffect = (
+  tx: TransactionLike,
+  table: Table,
+  existingColumns: ReadonlyMap<
+    string,
+    { dataType: string; isNullable: string; columnDefault: string | null }
+  >,
+  tableUsesView?: ReadonlyMap<string, boolean>
+): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    const sanitized = sanitizeTableName(table.name)
+    const physicalTableName = shouldUseView(table) ? getBaseTableName(sanitized) : sanitized
+    const tempTableName = `${physicalTableName}_migration_temp`
+
+    // Create temporary table with new schema
+    const createTableSQL = yield* Effect.try({
+      try: () =>
+        generateCreateTableSQL(table, tableUsesView).replace(
+          `CREATE TABLE IF NOT EXISTS ${physicalTableName}`,
+          `CREATE TABLE ${tempTableName}`
+        ),
+      catch: (error) =>
+        new SQLExecutionError({
+          message: `Failed to generate CREATE TABLE DDL for migration: ${String(error)}`,
+          cause: error,
+        }),
+    })
+    yield* executeSQL(tx, createTableSQL)
+
+    // Get new table column info and copy compatible data
+    const newColumnInfo = yield* fetchColumnInfo(tx, tempTableName)
+    const commonColumns = getCompatibleColumns(existingColumns, newColumnInfo)
+    yield* copyDataAndResetSequences({
+      tx,
+      tempTableName,
+      physicalTableName,
+      commonColumns,
+      newColumnInfo,
+    })
+
+    // Finalize by dropping old table and renaming temp table
+    yield* finalizeTableRecreation(tx, physicalTableName, tempTableName)
+  })

package/src/infrastructure/database/table-operations/table-effects.ts

@@ -0,0 +1,234 @@
+/**
+ * Copyright (c) 2025 ESSENTIAL SERVICES
+ *
+ * This source code is licensed under the Business Source License 1.1
+ * found in the LICENSE.md file in the root directory of this source tree.
+ */
+
+import { Effect } from 'effect'
+import {
+  shouldUseView,
+  generateLookupViewSQL,
+  generateLookupViewTriggers,
+} from '../lookup/lookup-view-generators'
+import {
+  generateAlterTableStatements,
+  syncUniqueConstraints,
+  syncForeignKeyConstraints,
+  syncCheckConstraints,
+  syncIndexes,
+  type BunSQLTransaction,
+} from '../schema-migration-helpers'
+import {
+  executeSQL,
+  executeSQLStatements,
+  getExistingColumns,
+  SQLExecutionError,
+  type TransactionLike,
+} from '../sql/sql-execution'
+import { generateTableViewStatements, generateReadOnlyViewTrigger } from '../views/view-generators'
+import { generateCreateTableSQL } from './create-table-sql'
+import { recreateTableWithDataEffect } from './migration-utils'
+import { applyTableFeatures, applyTableFeaturesWithoutIndexes } from './table-features'
+import type { Table } from '@/domain/models/app/table'
+
+/**
+ * Configuration for table migration operations
+ */
+export type MigrationConfig = {
+  readonly tableUsesView?: ReadonlyMap<string, boolean>
+  readonly previousSchema?: { readonly tables: readonly object[] }
+}
+
+/**
+ * Migrate existing table (ALTER statements + constraints + indexes)
+ */
+export const migrateExistingTableEffect = (params: {
+  readonly tx: TransactionLike
+  readonly table: Table
+  readonly existingColumns: ReadonlyMap<
+    string,
+    { dataType: string; isNullable: string; columnDefault: string | null }
+  >
+  readonly tableUsesView?: ReadonlyMap<string, boolean>
+  readonly previousSchema?: { readonly tables: readonly object[] }
+}): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    const { tx, table, existingColumns, tableUsesView, previousSchema } = params
+    const alterStatements = generateAlterTableStatements(table, existingColumns, previousSchema)
+
+    // If alterStatements is empty, table has incompatible schema changes
+    // (e.g., primary key type change) - need to recreate with data preservation
+    if (alterStatements.length === 0) {
+      yield* recreateTableWithDataEffect(tx, table, existingColumns, tableUsesView)
+    } else {
+      // Apply incremental migrations
+      yield* executeSQLStatements(tx, alterStatements)
+    }
+
+    // Always add/update unique constraints for existing tables
+    yield* syncUniqueConstraints(tx, table, previousSchema)
+
+    // Always sync foreign key constraints to ensure referential actions are up-to-date
+    yield* syncForeignKeyConstraints(tx, table, tableUsesView)
+
+    // Always sync CHECK constraints for fields with validation requirements
+    yield* syncCheckConstraints(tx, table)
+
+    // Always sync indexes when field indexed property changes or custom indexes are modified
+    yield* syncIndexes(tx, table, previousSchema)
+
+    // Apply table features (triggers, RLS) - indexes handled by syncIndexes above
+    yield* applyTableFeaturesWithoutIndexes(tx, table)
+  })
+
+/**
+ * Create new table (CREATE statement + indexes + triggers)
+ * Note: VIEWs are created in a separate phase after all base tables exist
+ *
+ * @param tx - Transaction object
+ * @param table - Table definition
+ * @param tableUsesView - Map of table names to whether they use a VIEW
+ * @param skipForeignKeys - Skip foreign key constraints (for circular dependencies)
+ */
+export const createNewTableEffect = (params: {
+  readonly tx: TransactionLike
+  readonly table: Table
+  readonly tableUsesView?: ReadonlyMap<string, boolean>
+  readonly skipForeignKeys?: boolean
+  readonly hasAuthConfig?: boolean
+}): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    const { tx, table, tableUsesView, skipForeignKeys, hasAuthConfig = true } = params
+    // Wrap DDL generation in Effect.try to catch synchronous errors (e.g., unknown field types)
+    const createTableSQL = yield* Effect.try({
+      try: () => generateCreateTableSQL(table, tableUsesView, skipForeignKeys, hasAuthConfig),
+      catch: (error) =>
+        new SQLExecutionError({
+          message: `Failed to generate CREATE TABLE DDL: ${String(error)}`,
+          cause: error,
+        }),
+    })
+    yield* executeSQL(tx, createTableSQL)
+
+    // Apply all table features (indexes, triggers, RLS)
+    yield* applyTableFeatures(tx, table)
+  })
+
+/**
+ * Create lookup VIEWs for tables with lookup fields
+ * Called after all base tables have been created to avoid dependency issues
+ */
+export const createLookupViewsEffect = (
+  tx: TransactionLike,
+  table: Table
+): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    if (shouldUseView(table)) {
+      const createViewSQL = generateLookupViewSQL(table)
+      if (createViewSQL) {
+        // Drop existing table if it exists (to allow VIEW creation)
+        // This handles the transition from TABLE to VIEW when rollup/lookup fields are added
+        yield* executeSQL(tx, `DROP TABLE IF EXISTS ${table.name} CASCADE`)
+
+        yield* executeSQL(tx, createViewSQL)
+
+        // Create INSTEAD OF triggers to make the VIEW writable
+        const triggerStatements = generateLookupViewTriggers(table)
+        yield* executeSQLStatements(tx, triggerStatements)
+      }
+    }
+  })
+
+/**
+ * Create table views (user-defined VIEWs from table.views configuration)
+ * Called after all tables and lookup views have been created
+ */
+export const createTableViewsEffect = (
+  tx: TransactionLike,
+  table: Table
+): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    // Views are dropped globally in schema-initializer.ts before this function is called
+    // This ensures all obsolete views are removed before creating new ones
+
+    // Only create views if table has views defined
+    if (!table.views || table.views.length === 0) {
+      return
+    }
+
+    // Drop and recreate each view (PostgreSQL doesn't support IF NOT EXISTS for views)
+    const viewSQL = generateTableViewStatements(table)
+
+    // Process each view sequentially (views may depend on each other)
+    /* eslint-disable functional/no-loop-statements */
+    for (const view of table.views) {
+      // Convert view.id to string (ViewId can be number or string)
+      const viewIdStr = String(view.id)
+
+      // Drop existing view or materialized view (if any)
+      // Try both types since we don't know what exists in the database
+      if (view.materialized) {
+        yield* executeSQL(tx, `DROP MATERIALIZED VIEW IF EXISTS ${viewIdStr} CASCADE`)
+      } else {
+        yield* executeSQL(tx, `DROP VIEW IF EXISTS ${viewIdStr} CASCADE`)
+      }
+
+      // Create view (regular or materialized)
+      const createSQL = viewSQL.find((sql) => sql.includes(viewIdStr))
+      if (createSQL) {
+        yield* executeSQL(tx, createSQL)
+
+        // For regular (non-materialized) views, add read-only triggers
+        // PostgreSQL views can be automatically updatable, so we need triggers to prevent modifications
+        if (!view.materialized) {
+          const readOnlyTriggerSQL = generateReadOnlyViewTrigger(view.id)
+          /* eslint-disable functional/no-loop-statements */
+          for (const triggerSQL of readOnlyTriggerSQL) {
+            yield* executeSQL(tx, triggerSQL)
+          }
+          /* eslint-enable functional/no-loop-statements */
+        }
+
+        // Refresh materialized view if requested
+        if (view.materialized && view.refreshOnMigration) {
+          yield* executeSQL(tx, `REFRESH MATERIALIZED VIEW ${viewIdStr}`)
+        }
+      }
+    }
+  })
+
+/**
+ * Create or migrate table based on existence
+ */
+export const createOrMigrateTableEffect = (params: {
+  readonly tx: BunSQLTransaction
+  readonly table: Table
+  readonly exists: boolean
+  readonly tableUsesView?: ReadonlyMap<string, boolean>
+  readonly previousSchema?: { readonly tables: readonly object[] }
+  readonly skipForeignKeys?: boolean
+  readonly hasAuthConfig?: boolean
+}): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    const { tx, table, exists, tableUsesView, previousSchema, skipForeignKeys, hasAuthConfig } =
+      params
+    if (exists) {
+      const existingColumns = yield* getExistingColumns(tx, table.name)
+      yield* migrateExistingTableEffect({
+        tx,
+        table,
+        existingColumns,
+        tableUsesView,
+        previousSchema,
+      })
+    } else {
+      yield* createNewTableEffect({
+        tx,
+        table,
+        tableUsesView,
+        skipForeignKeys,
+        hasAuthConfig: hasAuthConfig ?? true,
+      })
+    }
+  })