sovrium 0.0.2

package/src/infrastructure/database/schema-migration/column-detection.ts

@@ -0,0 +1,286 @@
+/**
+ * Copyright (c) 2025 ESSENTIAL SERVICES
+ *
+ * This source code is licensed under the Business Source License 1.1
+ * found in the LICENSE.md file in the root directory of this source tree.
+ */
+
+import { shouldCreateDatabaseColumn } from '../field-utils'
+import { generateColumnDefinition, isFieldNotNull } from '../sql/sql-generators'
+import {
+  normalizeDataType,
+  doesColumnTypeMatch,
+  generateAlterColumnTypeStatement,
+} from './type-utils'
+import type { Table } from '@/domain/models/app/table'
+import type { Fields } from '@/domain/models/app/table/fields'
+
+/** Column info from database */
+export type ExistingColumnInfo = {
+  dataType: string
+  isNullable: string
+  columnDefault: string | null
+}
+
+/**
+ * Check if id column needs to be recreated due to type mismatch
+ */
+export const needsIdColumnRecreation = (
+  existingColumns: ReadonlyMap<string, ExistingColumnInfo>,
+  shouldProtectIdColumn: boolean
+): boolean => {
+  if (!shouldProtectIdColumn) return false
+  if (!existingColumns.has('id')) return false
+
+  const idType = normalizeDataType(existingColumns.get('id')!.dataType)
+  return idType !== 'integer' && idType !== 'serial'
+}
+
+/**
+ * Find columns that should be added to the table
+ * Excludes columns that exist but have type mismatches (those will be altered, not dropped/added)
+ */
+export const findColumnsToAdd = (
+  table: Table,
+  existingColumns: ReadonlyMap<string, ExistingColumnInfo>,
+  renamedNewNames: ReadonlySet<string>
+): readonly Fields[number][] =>
+  table.fields.filter((field) => {
+    if (!shouldCreateDatabaseColumn(field)) return false // Skip UI-only fields
+    if (renamedNewNames.has(field.name)) return false // Skip renamed fields
+    if (!existingColumns.has(field.name)) return true // New column (needs ADD COLUMN)
+    // Existing columns with type mismatches will be handled by ALTER COLUMN TYPE
+    return false
+  })
+
+/**
+ * Find columns that should be dropped from the table
+ * Excludes columns with type mismatches (those will be altered, not dropped)
+ */
+export const findColumnsToDrop = (
+  existingColumns: ReadonlyMap<string, ExistingColumnInfo>,
+  schemaFieldsByName: ReadonlyMap<string, Fields[number]>,
+  shouldProtectIdColumn: boolean,
+  renamedOldNames: ReadonlySet<string>
+): readonly string[] =>
+  Array.from(existingColumns.keys()).filter((columnName) => {
+    // Never drop protected id column (it's already the correct type at this point)
+    if (shouldProtectIdColumn && columnName === 'id') return false
+
+    // Never drop intrinsic special fields (APP-TABLES-SPECIAL-FIELDS-007)
+    // These are automatic timestamp columns managed by triggers
+    if (columnName === 'created_at') return false
+    if (columnName === 'updated_at') return false
+    if (columnName === 'deleted_at') return false
+
+    // Don't drop if it's being renamed
+    if (renamedOldNames.has(columnName)) return false
+
+    // Drop if not in schema (but only if it's not a UI-only field)
+    if (!schemaFieldsByName.has(columnName)) return true
+
+    const field = schemaFieldsByName.get(columnName)!
+
+    // If this is a UI-only field that shouldn't have a column, drop it
+    if (!shouldCreateDatabaseColumn(field)) return true
+
+    // Existing columns with type mismatches will be handled by ALTER COLUMN TYPE (not dropped)
+    return false
+  })
+
+/**
+ * Filter fields that should be checked for schema modifications
+ * Excludes UI-only fields, renamed fields, and fields not in the database
+ */
+export const filterModifiableFields = (
+  fields: readonly Fields[number][],
+  existingColumns: ReadonlyMap<string, ExistingColumnInfo>,
+  renamedNewNames: ReadonlySet<string>
+): readonly Fields[number][] =>
+  fields.filter((field) => {
+    // Skip UI-only fields, renamed fields, and fields not in database
+    if (!shouldCreateDatabaseColumn(field)) return false
+    if (renamedNewNames.has(field.name)) return false
+    if (!existingColumns.has(field.name)) return false
+    return true
+  })
+
+/**
+ * Find columns that need type changes
+ * Returns ALTER COLUMN TYPE statements for type mismatches
+ */
+export const findTypeChanges = (
+  table: Table,
+  existingColumns: ReadonlyMap<string, ExistingColumnInfo>,
+  renamedNewNames: ReadonlySet<string>
+): readonly string[] =>
+  filterModifiableFields(table.fields, existingColumns, renamedNewNames).flatMap((field) => {
+    const existing = existingColumns.get(field.name)!
+    if (doesColumnTypeMatch(field, existing.dataType)) return []
+
+    // Type mismatch detected - generate ALTER COLUMN TYPE statement
+    return [generateAlterColumnTypeStatement(table.name, field, existing.dataType)]
+  })
+
+/**
+ * Generate validation query to check if existing data contains NULL values
+ * Returns validation query that will throw error if NULL values exist
+ */
+export const generateNotNullValidationQuery = (tableName: string, fieldName: string): string => `
+  DO $$
+  DECLARE
+    null_count INTEGER;
+  BEGIN
+    SELECT COUNT(*) INTO null_count
+    FROM ${tableName}
+    WHERE ${fieldName} IS NULL;
+
+    IF null_count > 0 THEN
+      RAISE EXCEPTION 'Migration failed: cannot add NOT NULL constraint to column "${fieldName}" in table "${tableName}" because % existing row(s) contain null values. Either provide a default value or update existing rows first.', null_count;
+    END IF;
+  END$$;
+`
+
+/**
+ * Generate backfill query to update NULL values with the default
+ * Returns UPDATE statement that sets NULL values to the default value
+ */
+export const generateBackfillQuery = (table: Table, field: Fields[number]): string => {
+  const columnDef = generateColumnDefinition(field, false, table.fields)
+  // Extract DEFAULT clause from column definition
+  // Handle quoted strings (DEFAULT 'value'), unquoted values (DEFAULT 42), and functions (DEFAULT NOW())
+  const defaultMatch = columnDef.match(/DEFAULT\s+('(?:[^']|'')*'|[^\s]+)/)
+  if (!defaultMatch) {
+    // This shouldn't happen if hasDefault is true, but handle gracefully
+    return ''
+  }
+
+  const defaultClause = defaultMatch[1]
+  return `UPDATE ${table.name} SET ${field.name} = ${defaultClause} WHERE ${field.name} IS NULL`
+}
+
+/**
+ * Find columns that need nullability changes
+ * Returns ALTER COLUMN statements for SET NOT NULL / DROP NOT NULL
+ * CRITICAL: Validates existing data before applying NOT NULL constraint
+ * Migration will FAIL if any existing data contains NULL values without a default
+ *
+ * When a default value is provided:
+ * 1. Set the default value on the column (handled by findDefaultValueChanges)
+ * 2. Backfill existing NULL values with the default
+ * 3. Then set NOT NULL
+ */
+export const findNullabilityChanges = (
+  table: Table,
+  existingColumns: ReadonlyMap<string, ExistingColumnInfo>,
+  renamedNewNames: ReadonlySet<string>,
+  primaryKeyFields: readonly string[]
+): readonly string[] =>
+  filterModifiableFields(table.fields, existingColumns, renamedNewNames).flatMap((field) => {
+    const existing = existingColumns.get(field.name)!
+    const isPrimaryKey = primaryKeyFields.includes(field.name)
+    const shouldBeNotNull = isFieldNotNull(field, isPrimaryKey)
+    const currentlyNotNull = existing.isNullable === 'NO'
+
+    // If nullability differs, generate ALTER COLUMN statement
+    if (shouldBeNotNull && !currentlyNotNull) {
+      // Check if field has a default value
+      const hasDefault = 'default' in field && field.default !== undefined
+
+      if (hasDefault) {
+        // If has default: backfill NULL values, then set NOT NULL
+        const backfillQuery = generateBackfillQuery(table, field)
+        return [
+          ...(backfillQuery ? [backfillQuery] : []),
+          `ALTER TABLE ${table.name} ALTER COLUMN ${field.name} SET NOT NULL`,
+        ]
+      }
+
+      // If no default value, validate that existing data doesn't contain NULL
+      // This prevents migration from failing with PostgreSQL's "column contains null values" error
+      return [
+        generateNotNullValidationQuery(table.name, field.name),
+        `ALTER TABLE ${table.name} ALTER COLUMN ${field.name} SET NOT NULL`,
+      ]
+    }
+    if (!shouldBeNotNull && currentlyNotNull && !isPrimaryKey) {
+      // Only DROP NOT NULL if it's not a primary key or auto-managed field
+      return [`ALTER TABLE ${table.name} ALTER COLUMN ${field.name} DROP NOT NULL`]
+    }
+    return []
+  })
+
+/**
+ * Find columns that need default value changes
+ * Returns ALTER COLUMN statements for SET DEFAULT or DROP DEFAULT
+ */
+export const findDefaultValueChanges = (
+  table: Table,
+  existingColumns: ReadonlyMap<string, ExistingColumnInfo>,
+  renamedNewNames: ReadonlySet<string>,
+  previousSchema?: { readonly tables: readonly object[] }
+): readonly string[] => {
+  // Find previous table definition to compare default values
+  const previousTable = previousSchema?.tables.find(
+    (t: object) => 'name' in t && t.name === table.name
+  ) as
+    | {
+        name: string
+        fields?: readonly { id?: number; name?: string; default?: unknown }[]
+      }
+    | undefined
+
+  const previousFieldsByName = new Map(
+    previousTable?.fields?.filter((f) => f.name !== undefined).map((f) => [f.name!, f.default]) ??
+      []
+  )
+
+  return filterModifiableFields(table.fields, existingColumns, renamedNewNames).flatMap((field) => {
+    const currentDefault = 'default' in field ? field.default : undefined
+    const previousDefault = previousFieldsByName.get(field.name)
+
+    // Only generate ALTER statements if default value changed
+    if (currentDefault === previousDefault) return []
+
+    // Case 1: Default removed (was set, now undefined)
+    if (previousDefault !== undefined && currentDefault === undefined) {
+      return [`ALTER TABLE ${table.name} ALTER COLUMN ${field.name} DROP DEFAULT`]
+    }
+
+    // Case 2: Default added or modified (generate SET DEFAULT statement)
+    if (currentDefault !== undefined) {
+      const columnDef = generateColumnDefinition(field, false, table.fields)
+      // Extract DEFAULT clause from column definition
+      // Handle quoted strings (DEFAULT 'value'), unquoted values (DEFAULT 42), and functions (DEFAULT NOW())
+      const defaultMatch = columnDef.match(/DEFAULT\s+('(?:[^']|'')*'|[^\s]+)/)
+      if (defaultMatch) {
+        const defaultClause = defaultMatch[1]
+        return [`ALTER TABLE ${table.name} ALTER COLUMN ${field.name} SET DEFAULT ${defaultClause}`]
+      }
+    }
+
+    return []
+  })
+}
+
+/**
+ * Build drop/add column statements from the computed columns to modify
+ */
+export const buildColumnStatements = (options: {
+  readonly tableName: string
+  readonly columnsToDrop: readonly string[]
+  readonly columnsToAdd: readonly Fields[number][]
+  readonly primaryKeyFields: readonly string[]
+  readonly allFields: readonly Fields[number][]
+}): { readonly dropStatements: readonly string[]; readonly addStatements: readonly string[] } => {
+  const { tableName, columnsToDrop, columnsToAdd, primaryKeyFields, allFields } = options
+  const dropStatements = columnsToDrop.map(
+    (columnName) => `ALTER TABLE ${tableName} DROP COLUMN ${columnName} CASCADE`
+  )
+  const addStatements = columnsToAdd.map((field) => {
+    const isPrimaryKey = primaryKeyFields.includes(field.name)
+    const columnDef = generateColumnDefinition(field, isPrimaryKey, allFields)
+    return `ALTER TABLE ${tableName} ADD COLUMN ${columnDef}`
+  })
+  return { dropStatements, addStatements }
+}

package/src/infrastructure/database/schema-migration/constants.ts

@@ -0,0 +1,31 @@
+/**
+ * Copyright (c) 2025 ESSENTIAL SERVICES
+ *
+ * This source code is licensed under the Business Source License 1.1
+ * found in the LICENSE.md file in the root directory of this source tree.
+ */
+
+/**
+ * System tables that should never be dropped
+ * These tables are managed by Better Auth/Drizzle migrations or migration system, not by runtime schema
+ * Note: Better Auth tables are in the auth schema with native Better Auth table names
+ * System tables are in the system schema (system.*)
+ */
+export const PROTECTED_SYSTEM_TABLES = new Set([
+  // Better Auth tables (in auth schema)
+  'auth.user',
+  'auth.session',
+  'auth.account',
+  'auth.verification',
+  'auth.two_factor',
+  'auth.team',
+  'auth.team_member',
+  'auth.role',
+  // Migration system tables (in system schema)
+  'system.migration_history',
+  'system.migration_log',
+  'system.schema_checksum',
+  // Activity and comment tables (in system schema)
+  'system.activity_logs',
+  'system.record_comments',
+])

package/src/infrastructure/database/schema-migration/constraint-sync.ts

@@ -0,0 +1,288 @@
+/**
+ * Copyright (c) 2025 ESSENTIAL SERVICES
+ *
+ * This source code is licensed under the Business Source License 1.1
+ * found in the LICENSE.md file in the root directory of this source tree.
+ */
+
+import { Effect } from 'effect'
+import {
+  executeSQLStatements,
+  type TransactionLike,
+  type SQLExecutionError,
+} from '../sql/sql-execution'
+import { generateForeignKeyConstraints, generateTableConstraints } from '../sql/sql-generators'
+import type { Table } from '@/domain/models/app/table'
+
+/**
+ * Get drop statements for removed unique constraints
+ */
+const getUniqueConstraintDropStatements = (
+  table: Table,
+  previousSchema: { readonly tables: readonly object[] } | undefined,
+  currentUniqueFields: readonly string[]
+): readonly string[] => {
+  if (!previousSchema) return []
+
+  const previousTable = previousSchema.tables.find(
+    (t: object) => 'name' in t && t.name === table.name
+  ) as
+    | {
+        name: string
+        fields?: readonly { name?: string; unique?: boolean }[]
+        uniqueConstraints?: readonly { name: string }[]
+      }
+    | undefined
+
+  if (!previousTable) return []
+
+  // Single-field constraints that were removed
+  const previousUniqueFields =
+    previousTable.fields?.filter((f) => f.name && 'unique' in f && f.unique).map((f) => f.name!) ??
+    []
+
+  const removedFields = previousUniqueFields.filter(
+    (fieldName) => !currentUniqueFields.includes(fieldName)
+  )
+
+  const singleFieldDrops = removedFields.map((fieldName) => {
+    const constraintName = `${table.name}_${fieldName}_key`
+    return `ALTER TABLE ${table.name} DROP CONSTRAINT IF EXISTS ${constraintName}`
+  })
+
+  // Composite constraints that were removed
+  const previousCompositeNames = previousTable.uniqueConstraints?.map((c) => c.name) ?? []
+  const currentCompositeNames = table.uniqueConstraints?.map((c) => c.name) ?? []
+
+  const removedComposites = previousCompositeNames.filter(
+    (name) => !currentCompositeNames.includes(name)
+  )
+
+  const compositeDrops = removedComposites.map(
+    (constraintName) => `ALTER TABLE ${table.name} DROP CONSTRAINT IF EXISTS ${constraintName}`
+  )
+
+  return [...singleFieldDrops, ...compositeDrops]
+}
+
+/**
+ * Build SQL statements to add unique constraints (single-field and composite)
+ */
+const buildUniqueConstraintAddStatements = (
+  table: Table,
+  uniqueFields: readonly string[]
+): readonly string[] => {
+  // Single-field constraints
+  const singleFieldStatements = uniqueFields.map((fieldName) => {
+    const constraintName = `${table.name}_${fieldName}_key`
+    return `
+      DO $$
+      BEGIN
+        IF NOT EXISTS (
+          SELECT 1 FROM information_schema.table_constraints
+          WHERE table_name = '${table.name}'
+            AND constraint_type = 'UNIQUE'
+            AND constraint_name = '${constraintName}'
+        ) THEN
+          ALTER TABLE ${table.name} ADD CONSTRAINT ${constraintName} UNIQUE (${fieldName});
+        END IF;
+      END$$;
+    `
+  })
+
+  // Composite constraints
+  const compositeStatements =
+    table.uniqueConstraints?.map((constraint) => {
+      const fields = constraint.fields.join(', ')
+      return `
+        DO $$
+        BEGIN
+          IF NOT EXISTS (
+            SELECT 1 FROM information_schema.table_constraints
+            WHERE table_name = '${table.name}'
+              AND constraint_type = 'UNIQUE'
+              AND constraint_name = '${constraint.name}'
+          ) THEN
+            ALTER TABLE ${table.name} ADD CONSTRAINT ${constraint.name} UNIQUE (${fields});
+          END IF;
+        END$$;
+      `
+    }) ?? []
+
+  return [...singleFieldStatements, ...compositeStatements]
+}
+
+/**
+ * Sync unique constraints for existing table
+ * Adds named UNIQUE constraints for:
+ * 1. Single-field constraints (fields with unique property)
+ * 2. Composite unique constraints (table.uniqueConstraints)
+ * Removes constraints that are no longer in the schema
+ * Uses PostgreSQL default naming convention: {table}_{column}_key for single fields
+ */
+export const syncUniqueConstraints = (
+  tx: TransactionLike,
+  table: Table,
+  previousSchema?: { readonly tables: readonly object[] }
+): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    const uniqueFields = table.fields.filter((f) => 'unique' in f && f.unique).map((f) => f.name)
+    const dropStatements = getUniqueConstraintDropStatements(table, previousSchema, uniqueFields)
+    const addStatements = buildUniqueConstraintAddStatements(table, uniqueFields)
+
+    yield* executeSQLStatements(tx, [...dropStatements, ...addStatements])
+  })
+
+/**
+ * Sync foreign key constraints for existing table
+ * Drops and recreates FK constraints to ensure referential actions (ON DELETE, ON UPDATE) are up-to-date
+ * This is needed when table schema is updated with new referential actions
+ *
+ * NOTE: After RENAME COLUMN, PostgreSQL preserves FK constraints but doesn't rename them.
+ * We need to drop old constraints by column name, not just by expected constraint name.
+ */
+export const syncForeignKeyConstraints = (
+  tx: TransactionLike,
+  table: Table,
+  tableUsesView?: ReadonlyMap<string, boolean>
+): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    const fkConstraints = generateForeignKeyConstraints(table.name, table.fields, tableUsesView)
+
+    // Build drop and add statements for each FK constraint
+    const statements = fkConstraints.flatMap((constraint) => {
+      // Extract column name from the constraint SQL
+      // Format: "CONSTRAINT {constraintName} FOREIGN KEY ({columnName}) REFERENCES ..."
+      const match = constraint.match(/CONSTRAINT\s+\w+\s+FOREIGN KEY\s+\((\w+)\)/)
+      if (!match) return []
+
+      const columnName = match[1]
+
+      // Drop ALL existing FK constraints on this column (handles renamed columns)
+      // PostgreSQL doesn't rename constraints when column is renamed, so we need to drop by column
+      const dropStatement = `
+        DO $$
+        DECLARE
+          constraint_rec RECORD;
+        BEGIN
+          FOR constraint_rec IN
+            SELECT tc.constraint_name
+            FROM information_schema.table_constraints tc
+            JOIN information_schema.key_column_usage kcu
+              ON tc.constraint_name = kcu.constraint_name
+              AND tc.table_schema = kcu.table_schema
+            WHERE tc.table_name = '${table.name}'
+              AND tc.constraint_type = 'FOREIGN KEY'
+              AND kcu.column_name = '${columnName}'
+          LOOP
+            EXECUTE 'ALTER TABLE ${table.name} DROP CONSTRAINT ' || constraint_rec.constraint_name;
+          END LOOP;
+        END$$;
+      `
+
+      // Add constraint with updated referential actions
+      const addStatement = `ALTER TABLE ${table.name} ADD ${constraint}`
+
+      return [dropStatement, addStatement]
+    })
+
+    // Execute all FK constraint statements sequentially
+    yield* executeSQLStatements(tx, statements)
+  })
+
+/**
+ * Validate existing data against new CHECK constraint before applying
+ * Returns validation query that will throw error if data violates constraint
+ */
+const generateConstraintValidationQuery = (
+  tableName: string,
+  constraint: string
+): string | undefined => {
+  // Extract constraint condition from the constraint SQL
+  // Format: "CONSTRAINT {constraintName} CHECK ({condition})"
+  const match = constraint.match(/CHECK\s*\((.+)\)$/)
+  if (!match) return undefined
+
+  const condition = match[1]
+
+  // Generate query that will fail if any row violates the new constraint
+  // Using NOT ({condition}) to find rows that violate the constraint
+  return `
+    DO $$
+    DECLARE
+      violation_count INTEGER;
+    BEGIN
+      SELECT COUNT(*) INTO violation_count
+      FROM ${tableName}
+      WHERE NOT (${condition});
+
+      IF violation_count > 0 THEN
+        RAISE EXCEPTION 'Migration failed: existing data violates check constraint. % row(s) in table "${tableName}" violate the new constraint condition.', violation_count;
+      END IF;
+    END$$;
+  `
+}
+
+/**
+ * Sync CHECK constraints for existing table
+ * Adds CHECK constraints for fields with validation requirements (enum values, ranges, formats, etc.)
+ * Drops and recreates constraints when they are modified (e.g., min/max value changes)
+ * This is needed when fields are added via ALTER TABLE and need their CHECK constraints
+ *
+ * CRITICAL: Validates existing data before applying new constraints
+ * Migration will FAIL if any existing data violates the new constraint
+ */
+export const syncCheckConstraints = (
+  tx: TransactionLike,
+  table: Table
+): Effect.Effect<void, SQLExecutionError> =>
+  Effect.gen(function* () {
+    const allConstraints = generateTableConstraints(table, undefined)
+
+    // Filter only CHECK constraints (not UNIQUE, FK, or PRIMARY KEY)
+    const checkConstraints = allConstraints.filter(
+      (constraint) =>
+        constraint.startsWith('CONSTRAINT') &&
+        constraint.includes('CHECK') &&
+        !constraint.includes('UNIQUE') &&
+        !constraint.includes('FOREIGN KEY') &&
+        !constraint.includes('PRIMARY KEY')
+    )
+
+    // Build statements to drop existing constraints and add new ones
+    // This ensures constraints are updated when validation rules change (e.g., max value increases)
+    const statements = checkConstraints.flatMap((constraint) => {
+      // Extract constraint name from the constraint SQL
+      // Format: "CONSTRAINT {constraintName} CHECK ..."
+      const match = constraint.match(/CONSTRAINT\s+(\w+)\s+CHECK/)
+      if (!match) return []
+
+      const constraintName = match[1]
+
+      // Generate validation query to check if existing data violates new constraint
+      const validationQuery = generateConstraintValidationQuery(table.name, constraint)
+
+      // Drop existing constraint if it exists, validate data, then add the new constraint
+      // This handles both new constraints and modified constraints
+      return [
+        `
+          DO $$
+          BEGIN
+            IF EXISTS (
+              SELECT 1 FROM information_schema.table_constraints
+              WHERE table_name = '${table.name}'
+                AND constraint_type = 'CHECK'
+                AND constraint_name = '${constraintName}'
+            ) THEN
+              ALTER TABLE ${table.name} DROP CONSTRAINT ${constraintName};
+            END IF;
+          END$$;
+        `,
+        ...(validationQuery ? [validationQuery] : []),
+        `ALTER TABLE ${table.name} ADD ${constraint}`,
+      ]
+    })
+
+    // Execute all statements sequentially (drop, validate, then add for each constraint)
+    yield* executeSQLStatements(tx, statements)
+  })