spine-framework 0.3.124 → 0.3.126

package/.framework/functions/cil-pipeline.ts

@@ -0,0 +1,318 @@
+import { createHandler } from './_shared/middleware'
+import { adminDb } from './_shared/db'
+
+const PIPELINE_STEPS = {
+  4: { name: 'chunk', prerequisites: [], target_status: '4-chunked' },
+  5: { name: 'document', prerequisites: [4], target_status: '5-documented' },
+  6: { name: 'embed-1st-pass', prerequisites: [5], target_status: '6-embedded' },
+  7: { name: 'map-1st-pass', prerequisites: [6], target_status: '7-mapped' },
+  8: { name: 'enrich', prerequisites: [7], target_status: '8-enriched' },
+  9: { name: 'embed-2nd-pass', prerequisites: [8], target_status: '9-embedded' },
+  10: { name: 'map-2nd-pass', prerequisites: [9], target_status: '10-mapped' },
+} as const
+
+type PipelineStep = keyof typeof PIPELINE_STEPS
+
+async function getProject(project_id: string) {
+  const { data: project } = await adminDb.from('items').select('data')
+    .eq('id', project_id).maybeSingle()
+  if (!project) throw new Error(`Project not found: ${project_id}`)
+  return project
+}
+
+function getCompletedSteps(status: string): PipelineStep[] {
+  const stepMap: Record<string, PipelineStep> = {
+    '4-chunked': 4,
+    '5-documented': 5,
+    '6-embedded': 6,
+    '7-mapped': 7,
+    '7.5-selected': 7,
+    '8-enriched': 8,
+    '9-embedded': 9,
+    '10-mapped': 10,
+  }
+  const completed: PipelineStep[] = []
+  for (const [statusName, stepNum] of Object.entries(stepMap)) {
+    if (status === statusName) {
+      completed.push(stepNum)
+    }
+  }
+  return completed
+}
+
+function getReadySteps(status: string): PipelineStep[] {
+  const completed = getCompletedSteps(status)
+  const ready: PipelineStep[] = []
+  for (const [stepNum, stepDef] of Object.entries(PIPELINE_STEPS)) {
+    const num = parseInt(stepNum) as PipelineStep
+    if (stepDef.prerequisites.every(prereq => completed.includes(prereq))) {
+      ready.push(num)
+    }
+  }
+  return ready
+}
+
+function getBlockedSteps(status: string): PipelineStep[] {
+  const completed = getCompletedSteps(status)
+  const blocked: PipelineStep[] = []
+  for (const [stepNum, stepDef] of Object.entries(PIPELINE_STEPS)) {
+    const num = parseInt(stepNum) as PipelineStep
+    if (!stepDef.prerequisites.every(prereq => completed.includes(prereq))) {
+      blocked.push(num)
+    }
+  }
+  return blocked
+}
+
+function calculateProgress(status: string): number {
+  const completed = getCompletedSteps(status)
+  const totalSteps = Object.keys(PIPELINE_STEPS).length
+  return (completed.length / totalSteps) * 100
+}
+
+function getBlockingReasons(status: string, step: PipelineStep): string[] {
+  const completed = getCompletedSteps(status)
+  const reasons: string[] = []
+  for (const prereq of PIPELINE_STEPS[step].prerequisites) {
+    if (!completed.includes(prereq)) {
+      reasons.push(`Missing prerequisite step ${prereq} (${PIPELINE_STEPS[prereq].name})`)
+    }
+  }
+  return reasons
+}
+
+function getStepBlockingReasons(files: Array<[string, string]>, step: PipelineStep): string[] {
+  const blockingReasons: string[] = []
+  for (const [file_path, status] of files) {
+    const completed = getCompletedSteps(status)
+    if (!PIPELINE_STEPS[step].prerequisites.every(prereq => completed.includes(prereq))) {
+      for (const prereq of PIPELINE_STEPS[step].prerequisites) {
+        if (!completed.includes(prereq)) {
+          blockingReasons.push(`${file_path}: missing step ${prereq}`)
+        }
+      }
+    }
+  }
+  return [...new Set(blockingReasons)]
+}
+
+function getOptimalProcessingOrder(files: Array<[string, string]>): Array<{file_path: string, suggested_next_step: PipelineStep | null}> {
+  const fileStates = files.map(([file_path, status]) => ({
+    file_path,
+    status,
+    completed_steps: getCompletedSteps(status),
+    ready_for: getReadySteps(status)
+  }))
+  return fileStates
+    .sort((a, b) => {
+      if (a.ready_for.length !== b.ready_for.length) {
+        return b.ready_for.length - a.ready_for.length
+      }
+      return a.completed_steps.length - b.completed_steps.length
+    })
+    .map(({ file_path, ready_for }) => ({
+      file_path,
+      suggested_next_step: ready_for[0] || null
+    }))
+}
+
+async function getPipelineStatus(project_id: string) {
+  const project = await getProject(project_id)
+  const fileStatuses: Record<string, string> = project.data?.file_statuses || {}
+  const files = Object.entries(fileStatuses).map(([file_path, status]) => ({
+    file_path,
+    current_status: status,
+    completed_steps: getCompletedSteps(status),
+    ready_for: getReadySteps(status),
+    blocked_by: getBlockedSteps(status),
+    progress_percentage: calculateProgress(status)
+  }))
+  const stepSummary: Record<PipelineStep, { completed: number; pending: number; in_progress: number }> = {} as any
+  for (const step of Object.keys(PIPELINE_STEPS) as PipelineStep[]) {
+    const stepNum = step as unknown as number
+    stepSummary[step] = {
+      completed: files.filter(f => f.completed_steps.includes(stepNum)).length,
+      pending: files.filter(f => f.ready_for.includes(stepNum)).length,
+      in_progress: files.filter(f => f.current_status === PIPELINE_STEPS[step].target_status).length
+    }
+  }
+  const overallProgress = files.reduce((sum, file) => sum + file.progress_percentage, 0) / files.length
+  return {
+    project_id,
+    files,
+    step_summary: stepSummary,
+    overall_progress: Math.round(overallProgress)
+  }
+}
+
+async function getFilesReadyForStep(project_id: string, step: number, include_blocking_info: boolean) {
+  const stepKey = step as PipelineStep
+  if (!PIPELINE_STEPS[stepKey]) {
+    throw new Error(`Invalid step: ${step}`)
+  }
+  const project = await getProject(project_id)
+  const fileStatuses: Record<string, string> = project.data?.file_statuses || {}
+  const allFiles = Object.entries(fileStatuses)
+  const readyFiles = allFiles.filter(([file_path, status]) => {
+    const completedSteps = getCompletedSteps(status)
+    return PIPELINE_STEPS[stepKey].prerequisites.every(prereq => completedSteps.includes(prereq))
+  })
+  const filePaths = readyFiles.map(([file_path]) => file_path)
+  const { data: chunks } = await adminDb.from('items').select('data')
+    .filter('data->>kb_type', 'eq', 'cil_chunk')
+    .filter('data->>project_id', 'eq', project_id)
+    .eq('is_active', true)
+    .in('data->>file_path' as any, filePaths)
+  const chunkCountByFile: Record<string, number> = {}
+  for (const c of chunks || []) {
+    const fp = c.data?.file_path
+    if (fp) chunkCountByFile[fp] = (chunkCountByFile[fp] || 0) + 1
+  }
+  const files = readyFiles.map(([file_path, status]) => {
+    const fileInfo: any = {
+      file_path,
+      status,
+      chunk_count: chunkCountByFile[file_path] || 0,
+      completed_steps: getCompletedSteps(status),
+      ready_for: getReadySteps(status),
+      blocked_by: getBlockedSteps(status)
+    }
+    if (include_blocking_info) {
+      fileInfo.blocking_reasons = getBlockingReasons(status, stepKey)
+      fileInfo.can_proceed = fileInfo.ready_for.includes(stepKey)
+    }
+    return fileInfo
+  })
+  return {
+    ok: true,
+    project_id,
+    step: stepKey,
+    step_name: PIPELINE_STEPS[stepKey].name,
+    prerequisites: PIPELINE_STEPS[stepKey].prerequisites,
+    target_status: PIPELINE_STEPS[stepKey].target_status,
+    file_count: files.length,
+    files
+  }
+}
+
+async function updateFilePipelineState(
+  project_id: string,
+  file_path: string,
+  step_completed: number,
+  new_status: string
+) {
+  const project = await getProject(project_id)
+  const fileStatuses = project.data?.file_statuses || {}
+  const previous_status = fileStatuses[file_path]
+  fileStatuses[file_path] = new_status
+  const { error } = await adminDb.from('items')
+    .update({
+      data: {
+        ...project.data,
+        file_statuses: fileStatuses,
+        last_ingested_at: new Date().toISOString()
+      }
+    })
+    .eq('id', project_id)
+  if (error) throw new Error(error.message)
+  return {
+    ok: true,
+    file_path,
+    previous_status,
+    new_status,
+    step_completed,
+    completed_steps: getCompletedSteps(new_status),
+    ready_for: getReadySteps(new_status),
+    progress_percentage: calculateProgress(new_status)
+  }
+}
+
+async function getPipelineBottlenecks(project_id: string) {
+  const project = await getProject(project_id)
+  const fileStatuses: Record<string, string> = project.data?.file_statuses || {}
+  const files = Object.entries(fileStatuses)
+  const bottlenecks = []
+  const recommendations = []
+  for (const step of Object.keys(PIPELINE_STEPS) as PipelineStep[]) {
+    const stepNum = step as unknown as number
+    const readyCount = files.filter(([_, status]) => {
+      const completedSteps = getCompletedSteps(status)
+      return PIPELINE_STEPS[step].prerequisites.every(prereq => completedSteps.includes(prereq))
+    }).length
+    const totalCount = files.length
+    const blockedCount = totalCount - readyCount
+    if (blockedCount > 0) {
+      bottlenecks.push({
+        step: stepNum,
+        step_name: PIPELINE_STEPS[step].name,
+        ready: readyCount,
+        blocked: blockedCount,
+        blocking_reasons: getStepBlockingReasons(files, step)
+      })
+      recommendations.push({
+        step: stepNum,
+        action: `Process ${blockedCount} files through prerequisite steps first`,
+        priority: step === 8 ? 'high' : 'medium'
+      })
+    }
+  }
+  const optimalOrder = getOptimalProcessingOrder(files)
+  return {
+    project_id,
+    bottlenecks,
+    recommendations: recommendations.sort((a, b) => {
+      const priorityOrder = { high: 3, medium: 2, low: 1 }
+      return priorityOrder[b.priority as keyof typeof priorityOrder] - priorityOrder[a.priority as keyof typeof priorityOrder]
+    }),
+    optimal_processing_order: optimalOrder,
+    summary: {
+      total_files: files.length,
+      bottleneck_count: bottlenecks.length,
+      ready_for_processing: files.filter(([_, status]) =>
+        getCompletedSteps(status).length >= 3
+      ).length
+    }
+  }
+}
+
+export const handler = createHandler(async (ctx, body) => {
+  const action = ctx.query?.action
+  if (!action) throw new Error('action is required')
+
+  if (action === 'get_pipeline_status') {
+    const { project_id } = body as { project_id: string }
+    if (!project_id) throw new Error('project_id is required')
+    return await getPipelineStatus(project_id)
+  }
+
+  if (action === 'get_files_ready_for_step') {
+    const { project_id, step, include_blocking_info } = body as {
+      project_id: string
+      step: number
+      include_blocking_info?: boolean
+    }
+    if (!project_id || step === undefined) throw new Error('project_id and step are required')
+    return await getFilesReadyForStep(project_id, step, include_blocking_info ?? false)
+  }
+
+  if (action === 'update_file_pipeline_state') {
+    const { project_id, file_path, step_completed, new_status } = body as {
+      project_id: string
+      file_path: string
+      step_completed: number
+      new_status: string
+    }
+    if (!project_id || !file_path || step_completed === undefined || !new_status) {
+      throw new Error('project_id, file_path, step_completed, and new_status are required')
+    }
+    return await updateFilePipelineState(project_id, file_path, step_completed, new_status)
+  }
+
+  if (action === 'get_pipeline_bottlenecks') {
+    const { project_id } = body as { project_id: string }
+    if (!project_id) throw new Error('project_id is required')
+    return await getPipelineBottlenecks(project_id)
+  }
+
+  throw new Error(`Unknown action: ${action}`)
+})