spine-framework 0.3.124 → 0.3.126

package/.framework/functions/system-cron.ts

@@ -1,86 +1,35 @@
-/**
- * @module system-cron
- * @audience core-contributor
- * @layer api-handler
- * @stability stable
- *
- * Scheduled job runner invoked by an external cron service (e.g., AWS
- * EventBridge, Google Cloud Scheduler). On each tick it:
- *
- * 1. Authenticates the request via `SCHEDULER_API_KEY` or machine principal
- * 2. Fetches all schedules due via `get_due_schedules` RPC
- * 3. For each schedule: validates creator, loads action + machine principal,
- *    checks scope, executes action, records outcome
- * 4. Fetches all timers due via `get_due_timers` RPC and runs each pipeline
- * 5. Evaluates threshold alerts via `evaluateThresholds`
- * 6. Runs daily log cleanup at 00:00 UTC via `cleanupOldLogs`
- *
- * **Routed by:** `POST /.netlify/functions/system-cron`
- *
- * **Authentication:** Request must supply `SCHEDULER_API_KEY` via `?api_key`
- * or `?scheduler_key`, OR originate from an internal machine principal
- * (type='machine', machineType='internal'). Unauthorized requests receive 403.
- *
- * **Response shape:**
- * ```ts
- * {
- *   executed: number
- *   success: number
- *   failed: number
- *   skipped: number
- *   thresholds_evaluated: number
- *   thresholds_breached: number
- *   logs_cleaned: number
- *   results: Array<{ scheduleId, actionId, status, error?, durationMs }>
- * }
- * ```
- *
- * INVARIANT: Uses `adminDb` (service-role client, bypasses RLS) because
- *   machine principals must access cross-account data.
- * INVARIANT: Log cleanup only runs when the current UTC time is 00:00.
- *
- * @seeAlso timers.ts (timer configuration CRUD)
- * @seeAlso pipelines.ts (pipeline configuration CRUD)
- * @seeAlso pipeline-runner.ts (runPipeline — timer/schedule execution)
- * @seeAlso observability.ts (analytics RPCs used by evaluateThresholds)
- * @seeAlso audit.ts (emitAudit for cron lifecycle events)
- */
-
 import { createHandler, RequestContext } from './_shared/middleware'
 import { adminDb } from './_shared/db'
 import { emitAudit } from './_shared/audit'
 import { runPipeline } from './_shared/pipeline-runner'
+import { retryPendingDeliveries } from './_shared/retry-deliveries'
 
 const SCHEDULER_API_KEY: string | undefined = (globalThis as any).process?.env?.SCHEDULER_API_KEY || (globalThis as any).Deno?.env?.get?.('SCHEDULER_API_KEY')
 
-// ─── MAIN HANDLER ────────────────────────────────────────────────────────────
 
-/**
- * Netlify function entry point — the entire cron tick logic lives here.
- * Returns HTTP 200 with a results summary on success, or HTTP 403/500
- * on auth/runtime failure.
- *
- * @throws Returns 403 JSON if scheduler key is missing or invalid
- * @throws Returns 500 JSON on unhandled top-level error
- * @sideEffects DB read: get_due_schedules, validate_schedule_creator,
- *   actions, api_keys, get_due_timers
- * @sideEffects DB write: schedule_executions (INSERT), update_schedule_after_run
- *   RPC, update_timer_after_run RPC, cleanup_old_logs RPC
- * @sideEffects pipeline: runPipeline (timers + threshold responses)
- * @sideEffects audit: emitAudit for unauthorized, error, schedule.execute,
- *   threshold.breached events
- * @calledBy External cron scheduler (AWS EventBridge / Google Cloud Scheduler)
- */
+
+
+
+
+// CIL[BEGIN chunk_id="system-cron_module_header_1783932909" type="module_header" identity="module_header" parent="none"]
+// CIL[DOC-MACRO purpose="Executes scheduled actions, timers, and threshold monitoring"]
+// CIL[DOC-RESPONSIBILITY responsibility="Handle system cron execution with security"]
+// CIL[DOC-CONTEXT module="system-cron" layer="api-handler" related_files="[\"custom/apps/portal/pages/ContentPage.tsx\",\"custom/apps/cortex/pages/operations/OperationsDashboard.tsx\",\".framework/src/components/ui/dialog.tsx\",\".framework/src/pages/admin/RolesPage.tsx\",\".framework/functions/_shared/principal.ts\",\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="ctx: RequestContext" outputs="Execution summary" errors="Throws on execution failures" side_effects="Database operations, audit logging"]
+// CIL[DOC-MICRO logic_summary="Processes schedules, timers, thresholds, and cleanup" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="complex_dependencies"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-AUDIENCE audience="developer"]
+// CIL[DOC-VECTOR type="context" content="Layer: api-handler. Direct callers: 0. Direct calls: 0. Related files: custom/apps/portal/pages/ContentPage.tsx, custom/apps/cortex/pages/operations/OperationsDashboard.tsx, .framework/src/components/ui/dialog.tsx, .framework/src/pages/admin/RolesPage.tsx, .framework/functions/_shared/principal.ts, \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 0. Total affected: 0."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 export const handler = createHandler(async (ctx: RequestContext) => {
-  // ============================================
-  // SECURITY: Validate this is an internal request
-  // ============================================
-  
-  // Check if request has scheduler authentication
   const requestApiKey = ctx.query?.api_key || ctx.query?.scheduler_key
   
   if (requestApiKey !== SCHEDULER_API_KEY) {
-    // Also allow if the principal is a system machine (for internal invocations)
     if (ctx.principal?.type !== 'machine' || ctx.principal?.machineType !== 'internal') {
       await emitAudit(ctx, 'system_cron.unauthorized_access', {
         type: 'system',
@@ -94,10 +43,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
     }
   }
   
-  // ============================================
-  // Find and execute due schedules
-  // ============================================
-  
   const results: Array<{
     scheduleId: string
     actionId: string
@@ -107,7 +52,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
   }> = []
   
   try {
-    // Get all schedules due for execution
     const { data: dueSchedules, error: schedulesError } = await adminDb.rpc('get_due_schedules', {
       p_now: new Date().toISOString()
     })
@@ -127,18 +71,15 @@ export const handler = createHandler(async (ctx: RequestContext) => {
       }
     }
     
-    // Execute each due schedule
     for (const schedule of dueSchedules) {
       const startTime = Date.now()
       
       try {
-        // Validate schedule can run (creator still active)
         const { data: validation, error: validationError } = await adminDb.rpc('validate_schedule_creator', {
           p_schedule_id: schedule.schedule_id
         })
         
         if (validationError || !validation?.is_valid) {
-          // Schedule was auto-paused by validation function
           results.push({
             scheduleId: schedule.schedule_id,
             actionId: schedule.action_id,
@@ -149,7 +90,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
           continue
         }
         
-        // Load the action
         const { data: action, error: actionError } = await adminDb
           .from('actions')
           .select('*')
@@ -160,7 +100,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
           throw new Error(`Action not found: ${schedule.action_id}`)
         }
         
-        // Load the machine principal
         const { data: machine, error: machineError } = await adminDb
           .from('api_keys')
           .select('*')
@@ -171,7 +110,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
           throw new Error(`Machine principal not found: ${schedule.machine_principal_id}`)
         }
         
-        // Create execution context with machine principal
         const executionCtx: RequestContext = {
           requestId: ctx.requestId,
           principal: {
@@ -188,14 +126,13 @@ export const handler = createHandler(async (ctx: RequestContext) => {
               cronId: schedule.schedule_id
             }
           },
-          db: adminDb,  // Machines use adminDb (RLS checks their ID)
+          db: adminDb,
           accountId: machine.account_id,
           appId: null,
           requestPath: '/.netlify/functions/system-cron',
           query: {}
         }
         
-        // Check machine has required scope for this action
         const requiredScope = action.required_scopes?.[0] || `${action.handler}:execute`
         const hasScope = executionCtx.principal.scopes?.includes(requiredScope) ||
                         executionCtx.principal.scopes?.includes('*:*')
@@ -204,10 +141,8 @@ export const handler = createHandler(async (ctx: RequestContext) => {
           throw new Error(`Machine lacks required scope: ${requiredScope}`)
         }
         
-        // Execute the action
         const executionResult = await executeAction(executionCtx, action, schedule.config)
         
-        // Record execution success
         await adminDb.from('schedule_executions').insert({
           schedule_id: schedule.schedule_id,
           account_id: schedule.account_id,
@@ -218,14 +153,12 @@ export const handler = createHandler(async (ctx: RequestContext) => {
           duration_ms: Date.now() - startTime
         })
         
-        // Update schedule state
         await adminDb.rpc('update_schedule_after_run', {
           p_schedule_id: schedule.schedule_id,
           p_success: true,
           p_error_message: null
         })
         
-        // Emit audit log
         await emitAudit(executionCtx, 'schedule.execute', {
           type: 'schedule',
           id: schedule.schedule_id,
@@ -246,7 +179,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
       } catch (execError: any) {
         const errorMessage = execError.message || 'Execution failed'
         
-        // Record execution failure
         await adminDb.from('schedule_executions').insert({
           schedule_id: schedule.schedule_id,
           account_id: schedule.account_id,
@@ -257,7 +189,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
           duration_ms: Date.now() - startTime
         })
         
-        // Update schedule state
         await adminDb.rpc('update_schedule_after_run', {
           p_schedule_id: schedule.schedule_id,
           p_success: false,
@@ -274,10 +205,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
       }
     }
     
-    // ============================================
-    // Execute due timers (pipeline-based timers)
-    // ============================================
-    
     try {
       const { data: dueTimers, error: timersError } = await adminDb.rpc('get_due_timers', {
         p_now: new Date().toISOString()
@@ -288,12 +215,10 @@ export const handler = createHandler(async (ctx: RequestContext) => {
           const timerStartTime = Date.now()
           
           try {
-            // Validate timer can run
             if (!timer.pipeline_id) {
               throw new Error(`Timer ${timer.timer_id} has no pipeline_id`)
             }
             
-            // Create execution context for timer
             const timerCtx: RequestContext = {
               requestId: ctx.requestId,
               principal: {
@@ -317,14 +242,12 @@ export const handler = createHandler(async (ctx: RequestContext) => {
               query: {}
             }
             
-            // Run the pipeline
             const result = await runPipeline(timer.pipeline_id, {
               timer_id: timer.timer_id,
               timer_name: timer.name,
               execution_count: timer.execution_count || 0
             }, timerCtx)
             
-            // Update timer state
             await adminDb.rpc('update_timer_after_run', {
               p_timer_id: timer.timer_id,
               p_success: result.status === 'completed',
@@ -340,7 +263,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
             })
             
           } catch (timerError: any) {
-            // Update timer with failure
             await adminDb.rpc('update_timer_after_run', {
               p_timer_id: timer.timer_id,
               p_success: false,
@@ -361,9 +283,6 @@ export const handler = createHandler(async (ctx: RequestContext) => {
       console.error('Timer execution error:', timerLoopError)
     }
     
-    // ============================================
-    // Evaluate threshold alerts (every minute)
-    // ============================================
     let thresholdResults: any[] = []
     try {
       thresholdResults = await evaluateThresholds(ctx)
@@ -377,15 +296,11 @@ export const handler = createHandler(async (ctx: RequestContext) => {
       console.error('Threshold evaluation error:', thresholdError)
     }
     
-    // ============================================
-    // Daily log cleanup (check if it's time)
-    // ============================================
     let logsCleaned = 0
     const now = new Date()
     const currentHour = now.getUTCHours()
     const currentMinute = now.getUTCMinutes()
     
-    // Run cleanup once per day at 00:00 UTC
     if (currentHour === 0 && currentMinute === 0) {
       try {
         logsCleaned = await cleanupOldLogs()
@@ -394,10 +309,11 @@ export const handler = createHandler(async (ctx: RequestContext) => {
       }
     }
     
-    // Return summary
     const successCount = results.filter(r => r.status === 'success').length
     const failedCount = results.filter(r => r.status === 'failed').length
     const skippedCount = results.filter(r => r.status === 'skipped').length
+
+    const retrySummary = await retryPendingDeliveries(ctx)
     
     return {
       statusCode: 200,
@@ -410,6 +326,7 @@ export const handler = createHandler(async (ctx: RequestContext) => {
         thresholds_evaluated: thresholdResults.length,
         thresholds_breached: thresholdResults.filter(r => r.breached).length,
         logs_cleaned: logsCleaned,
+        deliveries_retried: retrySummary,
         results
       })
     }
@@ -434,17 +351,21 @@ export const handler = createHandler(async (ctx: RequestContext) => {
     }
   }
 })
-
-// ─── PRIVATE HELPERS ───────────────────────────────────────────────────────────
-
-/**
- * Dispatches to the appropriate handler module (`functions` | `integrations`
- * | `custom`) based on `action.handler_module`. Merges action-level config
- * with schedule-specific config before invoking.
- *
- * @throws Error('Unknown handler module: <module>') on unrecognized module
- * @throws Error('Custom handlers not yet implemented: <name>')
- */
+// CIL[END chunk_id="system-cron_module_header_1783932909"]
+// CIL[BEGIN chunk_id="system-cron_executeaction_1783932909" type="function" identity="executeAction" parent="none"]
+// CIL[DOC-MACRO purpose="Dispatches actions to appropriate handler modules"]
+// CIL[DOC-RESPONSIBILITY responsibility="Route action execution by handler module"]
+// CIL[DOC-CONTEXT module="system-cron" layer="api-handler" related_files="[\"custom/apps/cortex/functions/custom_cil-embeddings.ts\",\"custom/apps/portal/pages/ContentPage.tsx\",\"custom/apps/cortex/pages/operations/OperationsDashboard.tsx\",\".framework/src/pages/admin/RolesPage.tsx\",\".framework/src/components/ui/table-primitives.tsx\",\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="Action execution result" errors="Throws on unknown modules" side_effects="Delegates to handlers"]
+// CIL[DOC-MICRO logic_summary="Routes to function/integration/custom handlers" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: api-handler. Direct callers: 1. Direct calls: 0. Related files: custom/apps/cortex/functions/custom_cil-embeddings.ts, custom/apps/portal/pages/ContentPage.tsx, custom/apps/cortex/pages/operations/OperationsDashboard.tsx, .framework/src/pages/admin/RolesPage.tsx, .framework/src/components/ui/table-primitives.tsx, \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 1. Total affected: 1."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 async function executeAction(
   ctx: RequestContext,
   action: any,
@@ -453,7 +374,6 @@ async function executeAction(
   const handlerModule = action.handler_module || 'functions'
   const handlerName = action.handler
   
-  // Merge action config with schedule-specific config
   const mergedConfig = {
     ...action.config,
     ...config
@@ -467,49 +387,49 @@ async function executeAction(
       return await executeIntegrationHandler(ctx, handlerName, mergedConfig)
     
     case 'custom':
-      // Custom handlers would be loaded from v2-custom
       throw new Error(`Custom handlers not yet implemented: ${handlerName}`)
     
     default:
       throw new Error(`Unknown handler module: ${handlerModule}`)
   }
 }
-
-/**
- * Routes to one of the built-in named handlers: `send_email`,
- * `generate_report`, `notify_watchers`, `run_pipeline`.
- * `run_pipeline` delegates to `runPipeline` from `_shared/pipeline-runner.ts`.
- *
- * @throws Error('Unknown function handler: <name>') on unrecognized name
- * @throws Error('pipeline_id is required for run_pipeline handler')
- */
+// CIL[END chunk_id="system-cron_executeaction_1783932909"]
+// CIL[BEGIN chunk_id="system-cron_executefunctionhandler_1783932909" type="function" identity="executeFunctionHandler" parent="none"]
+// CIL[DOC-MACRO purpose="Routes to built-in function handlers"]
+// CIL[DOC-RESPONSIBILITY responsibility="Execute built-in named functions"]
+// CIL[DOC-CONTEXT module="system-cron" layer="api-handler" related_files="[\"custom/apps/cortex/functions/custom_cil-embeddings.ts\",\"custom/apps/portal/pages/ContentPage.tsx\",\".framework/functions/_shared/principal.ts\",\".framework/functions/_shared/db.ts\",\"custom/apps/cortex/pages/operations/OperationsDashboard.tsx\",\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="Handler execution result" errors="Throws on unknown handlers" side_effects="Function execution"]
+// CIL[DOC-MICRO logic_summary="Maps handler names to implementations" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: api-handler. Direct callers: 1. Direct calls: 0. Related files: custom/apps/cortex/functions/custom_cil-embeddings.ts, custom/apps/portal/pages/ContentPage.tsx, .framework/functions/_shared/principal.ts, .framework/functions/_shared/db.ts, custom/apps/cortex/pages/operations/OperationsDashboard.tsx, \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 1. Total affected: 2."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 async function executeFunctionHandler(
   ctx: RequestContext,
   handlerName: string,
   config: any
 ): Promise<any> {
-  // Built-in handlers
   const handlers: Record<string, Function> = {
     'send_email': async (ctx: RequestContext, config: any) => {
-      // Implementation would integrate with email service
       console.log(`[${ctx.requestId}] Sending email:`, config)
       return { sent: true, recipients: config.recipients }
     },
     
     'generate_report': async (ctx: RequestContext, config: any) => {
-      // Implementation would generate and deliver report
       console.log(`[${ctx.requestId}] Generating report:`, config)
       return { generated: true, format: config.output_format }
     },
     
     'notify_watchers': async (ctx: RequestContext, config: any) => {
-      // Implementation would notify item watchers
       console.log(`[${ctx.requestId}] Notifying watchers:`, config)
       return { notified: true }
     },
     
     'run_pipeline': async (ctx: RequestContext, config: any) => {
-      // Execute a pipeline as part of scheduled action
       const { pipeline_id, trigger_data = {} } = config
       
       if (!pipeline_id) {
@@ -533,49 +453,55 @@ async function executeFunctionHandler(
   
   return await handler(ctx, config)
 }
-
-/**
- * Placeholder for integration-based action handlers (external service calls).
- * Currently logs and returns `{ executed: true }` for all handler names.
- */
+// CIL[END chunk_id="system-cron_executefunctionhandler_1783932909"]
+// CIL[BEGIN chunk_id="system-cron_executeintegrationhandler_1783932909" type="function" identity="executeIntegrationHandler" parent="none"]
+// CIL[DOC-MACRO purpose="Placeholder for integration-based action handlers"]
+// CIL[DOC-RESPONSIBILITY responsibility="Handle external service integration calls"]
+// CIL[DOC-CONTEXT module="system-cron" layer="api-handler" related_files="[\".framework/src/pages/admin/TypeDetailPage.tsx\",\".framework/functions/_shared/principal.ts\",\"custom/apps/cortex/pages/operations/OperationsDashboard.tsx\",\".framework/functions/_shared/schema-utils.ts\",\"custom/apps/portal/pages/ContentPage.tsx\",\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="{executed: true}" errors="None" side_effects="Logging"]
+// CIL[DOC-MICRO logic_summary="Placeholder for future integration implementations" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: api-handler. Direct callers: 1. Direct calls: 0. Related files: .framework/src/pages/admin/TypeDetailPage.tsx, .framework/functions/_shared/principal.ts, custom/apps/cortex/pages/operations/OperationsDashboard.tsx, .framework/functions/_shared/schema-utils.ts, custom/apps/portal/pages/ContentPage.tsx, \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 1. Total affected: 2."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 async function executeIntegrationHandler(
   ctx: RequestContext,
   handlerName: string,
   config: any
 ): Promise<any> {
-  // Integration handlers would call external services
-  // This is a placeholder for future implementation
   console.log(`[${ctx.requestId}] Integration handler: ${handlerName}`, config)
   return { executed: true, handler: handlerName }
 }
-
-/**
- * Evaluates all active `threshold_alert` items across all accounts.
- * For each threshold, queries the appropriate observability RPC
- * (`get_error_rate`, `get_latency_percentiles`, or `get_pipeline_stats`)
- * and fires the configured pipeline if the threshold is breached.
- *
- * Supported metrics: `error_rate`, `latency_p95`, `pipeline_failure_rate`
- *
- * @returns Array of `{ thresholdId, breached, fired }` results
- * @sideEffects DB read: items (threshold_alert), observability RPCs
- * @sideEffects pipeline: runPipeline when threshold breached and pipeline_id set
- * @sideEffects audit: emitAudit('threshold.breached') for each breached threshold
- * @calledBy handler (every tick)
- */
-async function evaluateThresholds(ctx: RequestContext): Promise<Array<{ thresholdId: string; breached: boolean; fired: boolean }>> {
-  const results: Array<{ thresholdId: string; breached: boolean; fired: boolean }> = []
+// CIL[END chunk_id="system-cron_executeintegrationhandler_1783932909"]
+// CIL[BEGIN chunk_id="system-cron_evaluatethresholds_1783932909" type="function" identity="evaluateThresholds" parent="none"]
+// CIL[DOC-MACRO purpose="Evaluates threshold alerts and fires pipelines"]
+// CIL[DOC-RESPONSIBILITY responsibility="Monitor metrics and trigger automated responses"]
+// CIL[DOC-CONTEXT module="system-cron" layer="api-handler" related_files="[\"custom/apps/portal/pages/ContentPage.tsx\",\"custom/apps/cortex/pages/operations/OperationsDashboard.tsx\",\".framework/src/components/ui/dialog.tsx\",\".framework/src/pages/admin/RolesPage.tsx\",\".framework/functions/_shared/principal.ts\",\"\\\"]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="Array of threshold evaluation results" errors="Returns empty array on errors" side_effects="Pipeline execution, audit logging"]
+// CIL[DOC-MICRO logic_summary="Queries metrics, checks thresholds, triggers pipelines" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="complex_dependencies"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: api-handler. Direct callers: 1. Direct calls: 0. Related files: custom/apps/portal/pages/ContentPage.tsx, custom/apps/cortex/pages/operations/OperationsDashboard.tsx, .framework/src/components/ui/dialog.tsx, .framework/src/pages/admin/RolesPage.tsx, .framework/functions/_shared/principal.ts, \."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 1. Total affected: 1."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
+async function evaluateThresholds(ctx: RequestContext): Promise<Array<{thresholdId: string, breached: boolean, fired: boolean}>> {
+  const results: Array<{thresholdId: string, breached: boolean, fired: boolean}> = []
   
   try {
-    // Load all active threshold alerts from items table
     const { data: thresholds, error } = await adminDb
-      .from('items')
+      .from('threshold_alerts')
       .select('*')
-      .eq('type', 'threshold_alert')
-      .eq('data->>is_active', 'true')
+      .eq('is_active', true)
     
     if (error) {
-      console.error('Failed to load thresholds:', error)
+      console.error('Failed to fetch thresholds:', error)
       return results
     }
     
@@ -583,75 +509,56 @@ async function evaluateThresholds(ctx: RequestContext): Promise<Array<{ threshol
       return results
     }
     
-    // Evaluate each threshold
+    const now = new Date()
+    
     for (const threshold of thresholds) {
       try {
-        const config = threshold.data || {}
-        const { metric, operator, value, window_minutes, pipeline_id } = config
-        
-        if (!metric || !operator || value === undefined || !window_minutes) {
-          console.warn(`Threshold ${threshold.id} missing required fields`)
-          continue
-        }
-        
-        // Calculate time window
-        const now = new Date()
-        const from = new Date(now.getTime() - window_minutes * 60 * 1000)
-        
+        const { metric, operator, value, window_minutes, pipeline_id } = threshold
+        let actualValue: number
         let breached = false
-        let actualValue: number = 0
         
-        // Query appropriate RPC based on metric
         switch (metric) {
-          case 'error_rate': {
-            const { data } = await adminDb.rpc('get_error_rate', {
-              p_account_id: threshold.account_id,
-              p_from: from.toISOString(),
-              p_to: now.toISOString()
+          case 'error_rate':
+            const { data: errorRateData } = await adminDb.rpc('get_error_rate', {
+              p_window_minutes: window_minutes
             })
-            if (data && data.length > 0) {
-              actualValue = data[0].rate
+            
+            if (errorRateData && errorRateData.length > 0) {
+              actualValue = errorRateData[0].error_rate || 0
               breached = operator === 'gt' ? actualValue > value : actualValue < value
             }
             break
-          }
-          
-          case 'latency_p95': {
-            const { data } = await adminDb.rpc('get_latency_percentiles', {
-              p_account_id: threshold.account_id,
-              p_from: from.toISOString(),
-              p_to: now.toISOString()
+            
+          case 'latency_p95':
+            const { data: latencyData } = await adminDb.rpc('get_latency_percentiles', {
+              p_window_minutes: window_minutes
             })
-            if (data && data.length > 0) {
-              actualValue = data[0].p95
+            
+            if (latencyData && latencyData.length > 0) {
+              actualValue = latencyData[0].p95 || 0
               breached = operator === 'gt' ? actualValue > value : actualValue < value
             }
             break
-          }
-          
-          case 'pipeline_failure_rate': {
-            const { data } = await adminDb.rpc('get_pipeline_stats', {
-              p_account_id: threshold.account_id,
-              p_from: from.toISOString(),
-              p_to: now.toISOString()
+            
+          case 'pipeline_failure_rate':
+            const { data: pipelineStats } = await adminDb.rpc('get_pipeline_stats', {
+              p_window_minutes: window_minutes
             })
-            if (data && data.length > 0) {
-              // Calculate overall failure rate across all pipelines
-              const totalSuccess = data.reduce((sum: number, p: any) => sum + (parseInt(p.success_count) || 0), 0)
-              const totalFailure = data.reduce((sum: number, p: any) => sum + (parseInt(p.failure_count) || 0), 0)
+            
+            if (pipelineStats && pipelineStats.length > 0) {
+              const totalSuccess = pipelineStats[0].total_success || 0
+              const totalFailure = pipelineStats[0].total_failure || 0
               const total = totalSuccess + totalFailure
               actualValue = total > 0 ? (totalFailure / total) * 100 : 0
               breached = operator === 'gt' ? actualValue > value : actualValue < value
             }
             break
-          }
-          
+            
           default:
             console.warn(`Unknown metric: ${metric}`)
             continue
         }
         
-        // Fire pipeline if breached and pipeline_id configured
         let fired = false
         if (breached && pipeline_id) {
           try {
@@ -663,13 +570,13 @@ async function evaluateThresholds(ctx: RequestContext): Promise<Array<{ threshol
               window_minutes,
               triggered_at: now.toISOString()
             }, ctx)
+            
             fired = true
           } catch (pipelineError: any) {
             console.error(`Failed to fire threshold pipeline ${pipeline_id}:`, pipelineError)
           }
         }
         
-        // Log threshold event if breached
         if (breached) {
           await emitAudit(ctx, 'threshold.breached', {
             type: 'threshold_alert',
@@ -709,15 +616,21 @@ async function evaluateThresholds(ctx: RequestContext): Promise<Array<{ threshol
     return results
   }
 }
-
-/**
- * Runs the `cleanup_old_logs` RPC with 90-day retention. Called once daily
- * by the handler at 00:00 UTC.
- *
- * @returns Number of log records deleted
- * @sideEffects DB write: cleanup_old_logs RPC (cross-account DELETE)
- * @calledBy handler (daily, 00:00 UTC)
- */
+// CIL[END chunk_id="system-cron_evaluatethresholds_1783932909"]
+// CIL[BEGIN chunk_id="system-cron_cleanupoldlogs_1783932909" type="function" identity="cleanupOldLogs" parent="none"]
+// CIL[DOC-MACRO purpose="Cleans up old log records beyond retention period"]
+// CIL[DOC-RESPONSIBILITY responsibility="Maintain log storage within retention limits"]
+// CIL[DOC-CONTEXT module="system-cron" layer="api-handler" related_files="[]"]
+// CIL[DOC-CONTRACT inputs="none" outputs="Number of deleted records" errors="Returns 0 on errors" side_effects="Database cleanup"]
+// CIL[DOC-MICRO logic_summary="Calls cleanup RPC with 90-day retention" notable_decisions="[]"]
+// CIL[DOC-CONFIDENCE pass="second" score="0.95" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-VECTOR type="context" content="Layer: api-handler. Direct callers: 0. Direct calls: 0. Related files: none."]
+// CIL[DOC-VECTOR type="blast_radius" content="Direct callers: 0. Total affected: 0."]
+// CIL[DOC-VECTOR type="configuration" content="No configuration dependencies."]
+// CIL[DOC-VECTOR type="constraint" content="No test paths identified."]
+// CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
+// CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 async function cleanupOldLogs(): Promise<number> {
   try {
     const { data, error } = await adminDb.rpc('cleanup_old_logs', {
@@ -740,3 +653,4 @@ async function cleanupOldLogs(): Promise<number> {
     return 0
   }
 }
+// CIL[END chunk_id="system-cron_cleanupoldlogs_1783932909"]