task-summary-extractor 9.2.2 → 9.4.0

package/src/services/gemini.js

@@ -90,7 +90,7 @@ async function prepareDocsForGemini(ai, docFileList) {
         const pollStart = Date.now();
         while (file.state === 'PROCESSING') {
           if (Date.now() - pollStart > GEMINI_POLL_TIMEOUT_MS) {
-            console.warn(`    ${c.warn(`${name} — polling timed out after ${(GEMINI_POLL_TIMEOUT_MS / 1000).toFixed(0)}s, skipping`)}`);
+            console.warn(`    ${c.warn(`${name} — file is still processing after ${(GEMINI_POLL_TIMEOUT_MS / 1000).toFixed(0)}s, skipping (you can increase the wait time with GEMINI_POLL_TIMEOUT_MS in .env)`)}`);
             file = null;
             break;
           }
@@ -287,7 +287,7 @@ async function processWithGemini(ai, filePath, displayName, contextDocs = [], pr
     const pollStart = Date.now();
     while (uploaded.state === 'PROCESSING') {
       if (Date.now() - pollStart > GEMINI_POLL_TIMEOUT_MS) {
-        throw new Error(`Gemini file processing timed out after ${(GEMINI_POLL_TIMEOUT_MS / 1000).toFixed(0)}s for ${displayName}. Try again or increase GEMINI_POLL_TIMEOUT_MS.`);
+        throw new Error(`File "${displayName}" is still processing after ${(GEMINI_POLL_TIMEOUT_MS / 1000).toFixed(0)}s. Try again or increase the wait time by setting GEMINI_POLL_TIMEOUT_MS in your .env file.`);
       }
       process.stdout.write(`    Processing${'.'.repeat((waited % 3) + 1)}   \r`);
       await new Promise(r => setTimeout(r, 5000));
@@ -343,7 +343,7 @@ async function processWithGemini(ai, filePath, displayName, contextDocs = [], pr
     buildProgressiveContext(previousAnalyses, userName) || ''
   );
   const docBudget = Math.max(100000, config.GEMINI_CONTEXT_WINDOW - 350000 - prevContextEstimate);
-  console.log(`    Context budget: ${(docBudget / 1000).toFixed(0)}K tokens for docs (${contextDocs.length} available)`);
+  console.log(`    Reference docs budget: ${(docBudget / 1000).toFixed(0)}K (${contextDocs.length} doc${contextDocs.length !== 1 ? 's' : ''} available)`);
 
   const { selected: selectedDocs, excluded, stats } = selectDocsByBudget(
     contextDocs, docBudget, { segmentIndex }
@@ -459,16 +459,53 @@ async function processWithGemini(ai, filePath, displayName, contextDocs = [], pr
         throw reuploadErr;
       }
     } else {
-      // Log request diagnostics for other errors to aid debugging
-      const partSummary = contentParts.map((p, i) => {
-        if (p.fileData) return `  [${i}] fileData: ${p.fileData.mimeType} → ${(p.fileData.fileUri || '').substring(0, 120)}`;
-        if (p.text) return `  [${i}] text: ${p.text.length} chars → ${p.text.substring(0, 80).replace(/\n/g, ' ')}...`;
-        return `  [${i}] unknown part`;
-      });
-      console.error(`    ${c.error('Request diagnostics:')}`);
-      console.error(`    Model: ${config.GEMINI_MODEL} | Parts: ${contentParts.length} | maxOutput: 65536`);
-      partSummary.forEach(s => console.error(`    ${s}`));
-      throw apiErr;
+      // Handle RESOURCE_EXHAUSTED specifically — shed lower-priority docs and retry
+      if (errMsg.includes('RESOURCE_EXHAUSTED') || errMsg.includes('429') || errMsg.includes('quota')) {
+        console.warn(`    ${c.warn('Context window or quota exceeded — shedding docs and retrying after 30s...')}`);
+        await new Promise(r => setTimeout(r, 30000));
+        // Rebuild with half the doc budget
+        const reducedBudget = Math.floor(docBudget * 0.5);
+        const { selected: reducedDocs } = selectDocsByBudget(contextDocs, reducedBudget, { segmentIndex });
+        const reducedParts = [contentParts[0]]; // keep video
+        for (const doc of reducedDocs) {
+          if (doc.type === 'inlineText') {
+            let content = doc.content;
+            const isVtt = doc.fileName.toLowerCase().endsWith('.vtt') || doc.fileName.toLowerCase().endsWith('.srt');
+            if (isVtt && segmentStartSec != null && segmentEndSec != null) {
+              content = sliceVttForSegment(content, segmentStartSec, segmentEndSec);
+            }
+            reducedParts.push({ text: `=== Document: ${doc.fileName} ===\n${content}` });
+          } else if (doc.type === 'fileData') {
+            reducedParts.push({ fileData: { mimeType: doc.mimeType, fileUri: doc.fileUri } });
+          }
+        }
+        // Re-add prompt/context parts (last 3-5 parts are prompt, focus, etc.)
+        const nonDocParts = contentParts.slice(1 + selectedDocs.length);
+        reducedParts.push(...nonDocParts);
+        requestPayload.contents[0].parts = reducedParts;
+        console.log(`    Reduced to ${reducedDocs.length} docs (budget: ${(reducedBudget / 1000).toFixed(0)}K tokens)`);
+        try {
+          response = await withRetry(
+            () => ai.models.generateContent(requestPayload),
+            { label: `Gemini segment analysis — reduced docs (${displayName})`, maxRetries: 1, baseDelay: 5000 }
+          );
+          console.log(`    ${c.success('Reduced-context retry succeeded')}`);
+        } catch (reduceErr) {
+          console.error(`    ${c.error(`Reduced-context retry also failed: ${reduceErr.message}`)}`);
+          throw reduceErr;
+        }
+      } else {
+        // Log request diagnostics for other errors to aid debugging
+        const partSummary = contentParts.map((p, i) => {
+          if (p.fileData) return `  [${i}] fileData: ${p.fileData.mimeType} → ${(p.fileData.fileUri || '').substring(0, 120)}`;
+          if (p.text) return `  [${i}] text: ${p.text.length} chars → ${p.text.substring(0, 80).replace(/\n/g, ' ')}...`;
+          return `  [${i}] unknown part`;
+        });
+        console.error(`    ${c.error('Request diagnostics:')}`);
+        console.error(`    Model: ${config.GEMINI_MODEL} | Parts: ${contentParts.length} | maxOutput: 65536`);
+        partSummary.forEach(s => console.error(`    ${s}`));
+        throw apiErr;
+      }
     }
   }
   const durationMs = Date.now() - t0;
@@ -628,6 +665,60 @@ ${segmentDumps}`;
 
   const contentParts = [{ text: compilationPrompt }];
 
+  // ------- Pre-flight context window check -------
+  const estimatedInputTokens = estimateTokens(compilationPrompt);
+  const safeLimit = Math.floor(config.GEMINI_CONTEXT_WINDOW * 0.80); // 80% of context window
+  if (estimatedInputTokens > safeLimit) {
+    console.warn(`  ${c.warn(`Compilation input (~${(estimatedInputTokens / 1000).toFixed(0)}K tokens) exceeds 80% of context window (${(safeLimit / 1000).toFixed(0)}K). Trimming older segment detail...`)}`);
+    // Re-build segment dumps with aggressive compression: keep only first & last 2 segments
+    // at full detail, compress the middle ones to IDs + statuses only.
+    const trimmedDumps = allSegmentAnalyses.map((analysis, idx) => {
+      const clean = { ...analysis };
+      delete clean._geminiMeta;
+      delete clean.seg;
+      delete clean.conversation_transcript;
+      const isEdge = idx < 2 || idx >= allSegmentAnalyses.length - 2;
+      if (!isEdge) {
+        // Aggressive compression for middle segments
+        if (clean.tickets) {
+          clean.tickets = clean.tickets.map(t => ({
+            ticket_id: t.ticket_id, status: t.status, title: t.title,
+            assignee: t.assignee, source_segment: t.source_segment,
+          }));
+        }
+        if (clean.change_requests) {
+          clean.change_requests = clean.change_requests.map(cr => ({
+            id: cr.id, status: cr.status, title: cr.title,
+            assigned_to: cr.assigned_to, source_segment: cr.source_segment,
+          }));
+        }
+        if (clean.action_items) {
+          clean.action_items = clean.action_items.map(ai => ({
+            id: ai.id, description: ai.description, assigned_to: ai.assigned_to,
+            status: ai.status, source_segment: ai.source_segment,
+          }));
+        }
+        delete clean.file_references;
+        clean.summary = (clean.summary || '').substring(0, 200);
+      } else {
+        if (clean.tickets) {
+          clean.tickets = clean.tickets.map(t => {
+            const tc = { ...t };
+            if (tc.comments && tc.comments.length > 5) {
+              tc.comments = tc.comments.slice(0, 5);
+              tc.comments.push({ note: `...${t.comments.length - 5} more comments omitted` });
+            }
+            return tc;
+          });
+        }
+      }
+      return `=== SEGMENT ${idx + 1} OF ${allSegmentAnalyses.length} ===\n${JSON.stringify(clean, null, 2)}`;
+    }).join('\n\n');
+    contentParts[0] = { text: compilationPrompt.replace(segmentDumps, trimmedDumps) };
+    const newEstimate = estimateTokens(contentParts[0].text);
+    console.log(`  Trimmed compilation input to ~${(newEstimate / 1000).toFixed(0)}K tokens`);
+  }
+
   const requestPayload = {
     model: config.GEMINI_MODEL,
     contents: [{ role: 'user', parts: contentParts }],
@@ -640,10 +731,44 @@ ${segmentDumps}`;
 
   const t0 = Date.now();
   console.log(`  Compiling with ${config.GEMINI_MODEL}...`);
-  const response = await withRetry(
-    () => ai.models.generateContent(requestPayload),
-    { label: 'Gemini final compilation', maxRetries: 2, baseDelay: 5000 }
-  );
+  let response;
+  try {
+    response = await withRetry(
+      () => ai.models.generateContent(requestPayload),
+      { label: 'Gemini final compilation', maxRetries: 2, baseDelay: 5000 }
+    );
+  } catch (compileErr) {
+    const errMsg = compileErr.message || '';
+    if (errMsg.includes('RESOURCE_EXHAUSTED') || errMsg.includes('429') || errMsg.includes('quota')) {
+      console.warn(`  ${c.warn('Context window or quota exceeded during compilation — waiting 30s and retrying with reduced input...')}`);
+      await new Promise(r => setTimeout(r, 30000));
+      // Halve the compilation prompt by keeping only edge segments
+      const miniDumps = allSegmentAnalyses.map((analysis, idx) => {
+        const clean = { tickets: (analysis.tickets || []).map(t => ({ ticket_id: t.ticket_id, status: t.status, title: t.title, assignee: t.assignee })),
+          change_requests: (analysis.change_requests || []).map(cr => ({ id: cr.id, status: cr.status, title: cr.title })),
+          action_items: (analysis.action_items || []).map(ai => ({ id: ai.id, description: ai.description, assigned_to: ai.assigned_to, status: ai.status })),
+          blockers: (analysis.blockers || []).map(b => ({ id: b.id, description: b.description, status: b.status })),
+          scope_changes: analysis.scope_changes || [],
+          your_tasks: analysis.your_tasks || {},
+          summary: (analysis.summary || '').substring(0, 300),
+        };
+        return `=== SEGMENT ${idx + 1} OF ${allSegmentAnalyses.length} ===\n${JSON.stringify(clean, null, 2)}`;
+      }).join('\n\n');
+      requestPayload.contents[0].parts = [{ text: compilationPrompt.replace(/SEGMENT ANALYSES:\n[\s\S]*$/, `SEGMENT ANALYSES:\n${miniDumps}`) }];
+      try {
+        response = await withRetry(
+          () => ai.models.generateContent(requestPayload),
+          { label: 'Gemini compilation (reduced)', maxRetries: 1, baseDelay: 5000 }
+        );
+        console.log(`  ${c.success('Reduced compilation succeeded')}`);
+      } catch (reduceErr) {
+        console.error(`  ${c.error(`Reduced compilation also failed: ${reduceErr.message}`)}`);
+        throw reduceErr;
+      }
+    } else {
+      throw compileErr;
+    }
+  }
   const durationMs = Date.now() - t0;
   const rawText = response.text;
 

package/src/utils/cli.js

@@ -35,7 +35,7 @@ function parseArgs(argv) {
     'skip-upload', 'force-upload', 'no-storage-url',
     'skip-compression', 'skip-gemini',
     'resume', 'reanalyze', 'dry-run',
-    'dynamic', 'deep-dive', 'update-progress',
+    'dynamic', 'deep-dive', 'deep-summary', 'update-progress',
     'no-focused-pass', 'no-learning', 'no-diff',
     'no-html',
   ]);
@@ -371,6 +371,8 @@ ${f('(default)', 'Video/audio analysis — compress, analyze, compile')}
 ${f('--dynamic', 'Document generation — no media required')}
 ${f('--update-progress', 'Track item completion via git changes')}
 ${f('--deep-dive', 'Generate explanatory docs per topic')}
+${f('--deep-summary', 'Pre-summarize context docs (saves ~60-80% input tokens)')}
+${f('--exclude-docs <list>', 'Comma-separated doc names to keep full (use with --deep-summary)')}
 
   ${h('CORE OPTIONS')}
 ${f('--name <name>', 'Your name (skip interactive prompt)')}
@@ -435,6 +437,8 @@ ${f('--version, -v', 'Show version')}
     ${c.dim('$')} taskex --format pdf "call 1" ${c.dim('# PDF report')}
     ${c.dim('$')} taskex --format docx "call 1" ${c.dim('# Word document')}
     ${c.dim('$')} taskex --resume "call 1" ${c.dim('# Resume interrupted run')}
+    ${c.dim('$')} taskex --deep-summary "call 1" ${c.dim('# Pre-summarize docs, save tokens')}
+    ${c.dim('$')} taskex --deep-summary --exclude-docs "board.md,spec.md" "call 1" ${c.dim('# Keep specific docs full')}
     ${c.dim('$')} taskex --update-progress --repo ./my-project "call 1"
   `);
   // Signal early exit — pipeline checks for help flag before calling this
@@ -444,6 +448,7 @@ ${f('--version, -v', 'Show version')}
 module.exports = {
   parseArgs, showHelp, discoverFolders, selectFolder, selectModel,
   promptUser, promptUserText, selectRunMode, selectFormats, selectConfidence,
+  selectDocsToExclude,
 };
 
 // ======================== INTERACTIVE PROMPTS ========================
@@ -527,6 +532,9 @@ const RUN_PRESETS = {
   },
 };
 
+// Attach RUN_PRESETS to exports (defined after module.exports due to const ordering)
+module.exports.RUN_PRESETS = RUN_PRESETS;
+
 /**
  * Interactive run-mode selector. Shows preset options and returns the chosen
  * preset key. The caller applies overrides to opts.
@@ -721,3 +729,83 @@ async function selectConfidence() {
     });
   });
 }
+
+// ======================== DEEP SUMMARY DOC EXCLUSION PICKER ========================
+
+/**
+ * Interactive picker: let user select documents to EXCLUDE from deep summary.
+ * Excluded docs stay at full fidelity; the summary pass focuses on their topics.
+ *
+ * @param {Array<{fileName: string, type: string, content?: string}>} contextDocs - Prepared docs
+ * @returns {Promise<string[]>} Array of excluded fileName strings
+ */
+async function selectDocsToExclude(contextDocs) {
+  const readline = require('readline');
+
+  // Only show inlineText docs with actual content
+  const eligible = contextDocs
+    .filter(d => d.type === 'inlineText' && d.content && d.content.length > 0)
+    .map(d => ({
+      fileName: d.fileName,
+      chars: d.content.length,
+      tokensEst: Math.ceil(d.content.length * 0.3),
+    }));
+
+  if (eligible.length === 0) return [];
+
+  console.log('');
+  console.log(`  ${c.bold('📋 Deep Summary — Choose What to Keep in Full')}`);
+  console.log(c.dim('  ' + '─'.repeat(60)));
+  console.log('');
+  console.log(`  ${c.dim('To save processing time, we can create short summaries of your')}`);
+  console.log(`  ${c.dim('reference documents. The AI will still read them — just faster.')}`);
+  console.log('');
+  console.log(`  ${c.bold('If a document is especially important to you, select it below')}`);
+  console.log(`  ${c.bold('to keep it in full.')} The rest will be smartly condensed.`);
+  console.log('');
+
+  eligible.forEach((d, i) => {
+    const num = c.cyan(`[${i + 1}]`);
+    const size = d.tokensEst >= 1000
+      ? c.dim(`~${(d.tokensEst / 1000).toFixed(0)}K words`)
+      : c.dim(`~${d.tokensEst} words`);
+    console.log(`    ${num} ${c.bold(d.fileName)} ${size}`);
+  });
+  console.log('');
+  console.log(c.dim('  Tip: Enter = condense all · Type numbers to keep full (e.g. 1,3)'));
+  console.log('');
+
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+  return new Promise(resolve => {
+    rl.question('  Keep full (e.g. 1,3 or Enter = condense all): ', answer => {
+      rl.close();
+      const trimmed = (answer || '').trim();
+
+      if (!trimmed) {
+        console.log(c.success('Got it — all documents will be condensed for faster processing'));
+        resolve([]);
+        return;
+      }
+
+      const parts = trimmed.split(/[\s,]+/).filter(Boolean);
+      const excluded = [];
+
+      for (const p of parts) {
+        const num = parseInt(p, 10);
+        if (num >= 1 && num <= eligible.length) {
+          excluded.push(eligible[num - 1].fileName);
+        }
+      }
+
+      if (excluded.length === 0) {
+        console.log(c.warn('No valid selections — condensing all documents'));
+        resolve([]);
+        return;
+      }
+
+      console.log(c.success(`Keeping ${excluded.length} doc(s) in full — the rest will be condensed:`));
+      excluded.forEach(f => console.log(`    ${c.dim('•')} ${c.cyan(f)}`));
+      resolve(excluded);
+    });
+  });
+}

package/src/utils/context-manager.js

@@ -29,6 +29,14 @@ function estimateDocTokens(doc) {
   return 500;
 }
 
+/**
+ * Hard character limit for VTT fallback.
+ * When VTT parsing fails (0 cues), the full VTT is returned.
+ * Cap it so a huge transcript can't blow the context window.
+ * 500K chars ≈ 150K tokens — leaves plenty of room for docs + prompt.
+ */
+const VTT_FALLBACK_MAX_CHARS = 500000;
+
 // ════════════════════════════════════════════════════════════
 //  Priority Classification
 // ════════════════════════════════════════════════════════════
@@ -100,12 +108,16 @@ function selectDocsByBudget(allDocs, tokenBudget, opts = {}) {
   const excluded = [];
   let usedTokens = 0;
 
+  // Hard cap: even P0/P1 docs may not exceed 2× the budget.
+  // This prevents a handful of huge critical docs from blowing the context window.
+  const hardCap = tokenBudget * 2;
+
   for (const item of classified) {
     if (usedTokens + item.tokens <= tokenBudget) {
       selected.push(item.doc);
       usedTokens += item.tokens;
-    } else if (item.priority <= PRIORITY.HIGH) {
-      // P0 and P1 are always included even if over budget
+    } else if (item.priority <= PRIORITY.HIGH && usedTokens + item.tokens <= hardCap) {
+      // P0 and P1 are always included even if over budget, up to the hard cap
       selected.push(item.doc);
       usedTokens += item.tokens;
     } else {
@@ -171,14 +183,28 @@ function parseVttCues(vttContent) {
  */
 function sliceVttForSegment(vttContent, segStartSec, segEndSec, overlapSec = 30) {
   const cues = parseVttCues(vttContent);
-  if (cues.length === 0) return vttContent; // fallback: return full VTT
+  if (cues.length === 0) {
+    // Fallback: return full VTT but cap size to avoid context window overflow
+    if (vttContent.length > VTT_FALLBACK_MAX_CHARS) {
+      return vttContent.substring(0, VTT_FALLBACK_MAX_CHARS) +
+        `\n\n[TRUNCATED — original VTT was ${(vttContent.length / 1024).toFixed(0)} KB; capped at ${(VTT_FALLBACK_MAX_CHARS / 1024).toFixed(0)} KB]`;
+    }
+    return vttContent;
+  }
 
   const rangeStart = Math.max(0, segStartSec - overlapSec);
   const rangeEnd = segEndSec + overlapSec;
 
   const filtered = cues.filter(c => c.endSec >= rangeStart && c.startSec <= rangeEnd);
 
-  if (filtered.length === 0) return vttContent; // fallback
+  if (filtered.length === 0) {
+    // Fallback with cap
+    if (vttContent.length > VTT_FALLBACK_MAX_CHARS) {
+      return vttContent.substring(0, VTT_FALLBACK_MAX_CHARS) +
+        `\n\n[TRUNCATED — original VTT was ${(vttContent.length / 1024).toFixed(0)} KB; capped at ${(VTT_FALLBACK_MAX_CHARS / 1024).toFixed(0)} KB]`;
+    }
+    return vttContent;
+  }
 
   const header = `WEBVTT\n\n[Segment transcript: ${formatHMS(segStartSec)} — ${formatHMS(segEndSec)}]\n[Showing cues from ${formatHMS(rangeStart)} to ${formatHMS(rangeEnd)} with ${overlapSec}s overlap]\n`;
 
@@ -492,4 +518,5 @@ module.exports = {
   buildProgressiveContext,
   buildSegmentFocus,
   detectBoundaryContext,
+  VTT_FALLBACK_MAX_CHARS,
 };