proagents 1.0.0

package/proagents/database/examples/006-data-migration.sql

@@ -0,0 +1,196 @@
+-- Migration: Migrate user roles from string to normalized table
+-- Author: ProAgents
+-- Date: 2024-01-20
+-- Risk Level: High
+-- Requires Approval: Manager + DBA
+-- Estimated Duration: 5-30 minutes depending on data volume
+
+-- Description:
+-- Transforms the denormalized 'role' enum column into a normalized
+-- many-to-many relationship with a roles table. This allows users
+-- to have multiple roles.
+--
+-- CRITICAL: This is a data migration. Test thoroughly on staging first.
+-- Create backup before running in production.
+
+-- ============================================
+-- PRE-FLIGHT CHECKS
+-- ============================================
+
+-- Create backup table
+CREATE TABLE users_backup_20240120 AS SELECT * FROM users;
+
+-- Verify backup
+SELECT COUNT(*) FROM users_backup_20240120;
+
+-- Check current role distribution
+SELECT role, COUNT(*) as count
+FROM users
+GROUP BY role
+ORDER BY count DESC;
+
+-- ============================================
+-- UP MIGRATION
+-- ============================================
+
+-- Step 1: Create roles table
+CREATE TABLE roles (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name VARCHAR(50) NOT NULL UNIQUE,
+    description TEXT,
+    permissions JSONB NOT NULL DEFAULT '[]'::jsonb,
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+);
+
+-- Step 2: Create junction table for many-to-many
+CREATE TABLE user_roles (
+    user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
+    granted_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    granted_by UUID REFERENCES users(id),
+    PRIMARY KEY (user_id, role_id)
+);
+
+-- Create indexes
+CREATE INDEX idx_user_roles_user_id ON user_roles(user_id);
+CREATE INDEX idx_user_roles_role_id ON user_roles(role_id);
+
+-- Step 3: Populate roles table from existing enum values
+INSERT INTO roles (name, description, permissions) VALUES
+    ('admin', 'Full system access', '["read", "write", "delete", "admin"]'::jsonb),
+    ('developer', 'Development access', '["read", "write", "deploy"]'::jsonb),
+    ('viewer', 'Read-only access', '["read"]'::jsonb),
+    ('guest', 'Limited guest access', '["read_public"]'::jsonb);
+
+-- Step 4: Migrate existing user roles to junction table
+INSERT INTO user_roles (user_id, role_id, granted_at)
+SELECT
+    u.id as user_id,
+    r.id as role_id,
+    u.created_at as granted_at
+FROM users u
+JOIN roles r ON r.name = u.role::text
+WHERE u.deleted_at IS NULL;
+
+-- Step 5: Verify migration counts match
+DO $$
+DECLARE
+    original_count INTEGER;
+    migrated_count INTEGER;
+BEGIN
+    SELECT COUNT(*) INTO original_count
+    FROM users WHERE deleted_at IS NULL;
+
+    SELECT COUNT(DISTINCT user_id) INTO migrated_count
+    FROM user_roles;
+
+    IF original_count != migrated_count THEN
+        RAISE EXCEPTION 'Migration count mismatch: % vs %',
+            original_count, migrated_count;
+    END IF;
+
+    RAISE NOTICE 'Migration verified: % users migrated', migrated_count;
+END $$;
+
+-- Step 6: Create view for backward compatibility
+CREATE VIEW user_primary_role AS
+SELECT
+    u.id as user_id,
+    r.name as role,
+    r.id as role_id
+FROM users u
+JOIN user_roles ur ON ur.user_id = u.id
+JOIN roles r ON r.id = ur.role_id
+WHERE ur.granted_at = (
+    SELECT MIN(granted_at)
+    FROM user_roles
+    WHERE user_id = u.id
+);
+
+-- Step 7: Drop old role column (after application updated)
+-- IMPORTANT: Only run this after verifying application works with new schema
+-- ALTER TABLE users DROP COLUMN role;
+-- DROP TYPE IF EXISTS user_role;
+
+-- ============================================
+-- DOWN MIGRATION (Rollback)
+-- ============================================
+
+-- Step 1: Restore role column if dropped
+-- ALTER TABLE users ADD COLUMN role user_role;
+
+-- Step 2: Restore data from junction table
+UPDATE users u
+SET role = (
+    SELECT r.name::user_role
+    FROM user_roles ur
+    JOIN roles r ON r.id = ur.role_id
+    WHERE ur.user_id = u.id
+    ORDER BY ur.granted_at
+    LIMIT 1
+);
+
+-- Step 3: Drop new tables
+DROP VIEW IF EXISTS user_primary_role;
+DROP TABLE IF EXISTS user_roles;
+DROP TABLE IF EXISTS roles;
+
+-- Step 4: Restore from backup if needed
+-- DROP TABLE users;
+-- ALTER TABLE users_backup_20240120 RENAME TO users;
+
+-- ============================================
+-- VERIFICATION
+-- ============================================
+
+-- Compare counts
+SELECT
+    (SELECT COUNT(*) FROM users WHERE deleted_at IS NULL) as total_users,
+    (SELECT COUNT(DISTINCT user_id) FROM user_roles) as users_with_roles,
+    (SELECT COUNT(*) FROM user_roles) as total_assignments;
+
+-- Check role distribution matches original
+SELECT r.name, COUNT(ur.user_id) as user_count
+FROM roles r
+LEFT JOIN user_roles ur ON ur.role_id = r.id
+GROUP BY r.name
+ORDER BY user_count DESC;
+
+-- Verify no orphaned records
+SELECT COUNT(*) as orphaned
+FROM user_roles ur
+WHERE NOT EXISTS (SELECT 1 FROM users u WHERE u.id = ur.user_id);
+
+-- Test backward compatibility view
+SELECT * FROM user_primary_role LIMIT 5;
+
+-- ============================================
+-- CLEANUP (Run after verification period)
+-- ============================================
+
+-- Drop backup table
+-- DROP TABLE IF EXISTS users_backup_20240120;
+
+-- Drop old column and enum
+-- ALTER TABLE users DROP COLUMN IF EXISTS role;
+-- DROP TYPE IF EXISTS user_role;
+
+-- ============================================
+-- APPLICATION CODE UPDATES
+-- ============================================
+
+-- Old query:
+-- SELECT * FROM users WHERE role = 'admin';
+
+-- New query:
+-- SELECT u.*
+-- FROM users u
+-- JOIN user_roles ur ON ur.user_id = u.id
+-- JOIN roles r ON r.id = ur.role_id
+-- WHERE r.name = 'admin';
+
+-- Or using the view for backward compatibility:
+-- SELECT u.*
+-- FROM users u
+-- JOIN user_primary_role upr ON upr.user_id = u.id
+-- WHERE upr.role = 'admin';

package/proagents/database/examples/007-drop-column.sql

@@ -0,0 +1,163 @@
+-- Migration: Drop deprecated 'legacy_id' column
+-- Author: ProAgents
+-- Date: 2024-01-21
+-- Risk Level: High
+-- Requires Approval: Manager + DBA
+-- Estimated Duration: < 1 second (metadata only, but irreversible)
+
+-- Description:
+-- Removes the deprecated 'legacy_id' column that was used during
+-- migration from the old system. All references have been updated
+-- to use the new UUID primary key.
+--
+-- CRITICAL: This is a DESTRUCTIVE operation. Data cannot be recovered
+-- without a backup. Ensure all dependent code and queries have been updated.
+
+-- ============================================
+-- PRE-FLIGHT CHECKS (MANDATORY)
+-- ============================================
+
+-- 1. Check if column is still referenced in application code
+--    Run: grep -r "legacy_id" ./src/
+--    Expected: No results
+
+-- 2. Check if column is used in any views
+SELECT viewname, definition
+FROM pg_views
+WHERE definition LIKE '%legacy_id%';
+
+-- 3. Check if column is used in any functions/procedures
+SELECT proname, prosrc
+FROM pg_proc
+WHERE prosrc LIKE '%legacy_id%';
+
+-- 4. Check if column is used in any triggers
+SELECT tgname
+FROM pg_trigger t
+JOIN pg_proc p ON p.oid = t.tgfoid
+WHERE p.prosrc LIKE '%legacy_id%';
+
+-- 5. Check if any foreign keys reference this column
+SELECT
+    tc.constraint_name,
+    tc.table_name,
+    kcu.column_name,
+    ccu.table_name AS foreign_table_name,
+    ccu.column_name AS foreign_column_name
+FROM information_schema.table_constraints AS tc
+JOIN information_schema.key_column_usage AS kcu
+    ON tc.constraint_name = kcu.constraint_name
+JOIN information_schema.constraint_column_usage AS ccu
+    ON ccu.constraint_name = tc.constraint_name
+WHERE ccu.column_name = 'legacy_id'
+   OR kcu.column_name = 'legacy_id';
+
+-- 6. Verify backup exists or create one
+SELECT COUNT(*) FROM users; -- Note the count
+-- CREATE TABLE users_backup_legacy_id AS SELECT id, legacy_id FROM users;
+
+-- ============================================
+-- UP MIGRATION
+-- ============================================
+
+-- OPTION 1: Immediate drop (brief exclusive lock)
+ALTER TABLE users
+DROP COLUMN legacy_id;
+
+-- OPTION 2: Two-phase drop (for zero-downtime on very large tables)
+-- Phase 1: Mark column as dropped (immediate, no lock)
+-- UPDATE pg_attribute
+-- SET attisdropped = true
+-- WHERE attrelid = 'users'::regclass
+--   AND attname = 'legacy_id';
+--
+-- Phase 2: Actually reclaim space (during maintenance window)
+-- VACUUM FULL users;
+
+-- Drop related indexes if any
+DROP INDEX IF EXISTS idx_users_legacy_id;
+
+-- Drop related constraints if any
+ALTER TABLE users
+DROP CONSTRAINT IF EXISTS chk_users_legacy_id;
+
+-- ============================================
+-- DOWN MIGRATION (Rollback)
+-- ============================================
+
+-- WARNING: This only recreates the column structure.
+-- DATA CANNOT BE RECOVERED without a backup!
+
+-- Recreate column
+ALTER TABLE users
+ADD COLUMN legacy_id INTEGER;
+
+-- Recreate index if it existed
+CREATE INDEX idx_users_legacy_id ON users(legacy_id);
+
+-- Restore data from backup (if backup exists)
+UPDATE users u
+SET legacy_id = b.legacy_id
+FROM users_backup_legacy_id b
+WHERE u.id = b.id;
+
+-- If no backup exists, column will be NULL for all rows
+-- Consider whether this is acceptable before dropping
+
+-- ============================================
+-- VERIFICATION
+-- ============================================
+
+-- Confirm column no longer exists
+SELECT column_name
+FROM information_schema.columns
+WHERE table_name = 'users'
+  AND column_name = 'legacy_id';
+-- Expected: 0 rows
+
+-- Check table structure
+\d users
+
+-- Verify no errors in dependent objects
+SELECT *
+FROM pg_stat_user_functions
+WHERE funcname LIKE '%legacy%';
+
+-- Test that application queries still work
+EXPLAIN ANALYZE
+SELECT id, email, created_at
+FROM users
+WHERE email = 'test@example.com';
+
+-- ============================================
+-- CLEANUP
+-- ============================================
+
+-- After verification period, drop backup table
+-- DROP TABLE IF EXISTS users_backup_legacy_id;
+
+-- Update documentation
+-- - Remove legacy_id from API docs
+-- - Remove legacy_id from database schema docs
+-- - Update migration guides
+
+-- ============================================
+-- COMMON ISSUES AND SOLUTIONS
+-- ============================================
+
+-- Issue: "column does not exist" errors after drop
+-- Solution: Application code still references the column
+--           Search and update all references
+
+-- Issue: View/function errors after drop
+-- Solution: Recreate views/functions without the column
+--           DROP VIEW affected_view;
+--           CREATE VIEW affected_view AS ...;
+
+-- Issue: Need to restore data
+-- Solution: Restore from backup table or full database backup
+--           If no backup, data is permanently lost
+
+-- Issue: Replication lag causes errors
+-- Solution: Ensure all replicas have applied the change
+--           before updating application code

package/proagents/database/examples/README.md

@@ -0,0 +1,89 @@
+# Database Migration Examples
+
+Concrete migration examples for common database operations.
+
+---
+
+## Example Migrations
+
+| Migration | Description | Risk Level |
+|-----------|-------------|------------|
+| [001-create-users](./001-create-users.sql) | Create users table | Low |
+| [002-add-preferences](./002-add-preferences.sql) | Add column with default | Low |
+| [003-add-index](./003-add-index.sql) | Add performance index | Medium |
+| [004-rename-column](./004-rename-column.sql) | Rename existing column | Medium |
+| [005-add-foreign-key](./005-add-foreign-key.sql) | Add relationship constraint | Medium |
+| [006-data-migration](./006-data-migration.sql) | Transform existing data | High |
+| [007-drop-column](./007-drop-column.sql) | Remove column safely | High |
+
+---
+
+## Using These Examples
+
+### 1. Copy and Customize
+
+```bash
+# Copy template to your migrations folder
+cp proagents/database/examples/001-create-users.sql \
+   migrations/20240115_001_create_users.sql
+```
+
+### 2. Naming Convention
+
+```
+YYYYMMDD_NNN_description.sql
+
+Example:
+20240115_001_create_users.sql
+20240115_002_add_user_email_index.sql
+```
+
+### 3. Required Structure
+
+Every migration file must include:
+- Header comment with metadata
+- UP migration (forward)
+- DOWN migration (rollback)
+- Verification query
+
+---
+
+## Migration Template
+
+```sql
+-- Migration: [description]
+-- Author: [name]
+-- Date: [YYYY-MM-DD]
+-- Risk Level: [Low|Medium|High|Critical]
+-- Requires Approval: [Yes|No]
+-- Estimated Duration: [time]
+
+-- ============================================
+-- UP MIGRATION
+-- ============================================
+
+-- [Your forward migration SQL here]
+
+-- ============================================
+-- DOWN MIGRATION (Rollback)
+-- ============================================
+
+-- [Your rollback SQL here]
+
+-- ============================================
+-- VERIFICATION
+-- ============================================
+
+-- [Query to verify migration succeeded]
+```
+
+---
+
+## Risk Levels
+
+| Level | Examples | Approval Required |
+|-------|----------|------------------|
+| **Low** | Add nullable column, create table | No |
+| **Medium** | Add index, add constraint | Team lead |
+| **High** | Data migration, drop column | Manager + DBA |
+| **Critical** | Drop table, schema restructure | CTO + DBA |