task-summary-extractor 8.3.0 → 9.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/.env.example +38 -0
  2. package/ARCHITECTURE.md +99 -3
  3. package/EXPLORATION.md +148 -89
  4. package/QUICK_START.md +5 -2
  5. package/README.md +51 -7
  6. package/bin/taskex.js +11 -4
  7. package/package.json +38 -5
  8. package/src/config.js +52 -3
  9. package/src/modes/focused-reanalysis.js +2 -1
  10. package/src/modes/progress-updater.js +1 -1
  11. package/src/phases/_shared.js +43 -0
  12. package/src/phases/compile.js +101 -0
  13. package/src/phases/deep-dive.js +118 -0
  14. package/src/phases/discover.js +178 -0
  15. package/src/phases/init.js +192 -0
  16. package/src/phases/output.js +238 -0
  17. package/src/phases/process-media.js +633 -0
  18. package/src/phases/services.js +104 -0
  19. package/src/phases/summary.js +86 -0
  20. package/src/pipeline.js +431 -1463
  21. package/src/renderers/docx.js +531 -0
  22. package/src/renderers/html.js +672 -0
  23. package/src/renderers/markdown.js +15 -183
  24. package/src/renderers/pdf.js +90 -0
  25. package/src/renderers/shared.js +211 -0
  26. package/src/schemas/analysis-compiled.schema.json +381 -0
  27. package/src/schemas/analysis-segment.schema.json +380 -0
  28. package/src/services/doc-parser.js +346 -0
  29. package/src/services/gemini.js +101 -44
  30. package/src/services/video.js +123 -8
  31. package/src/utils/adaptive-budget.js +6 -4
  32. package/src/utils/checkpoint.js +2 -1
  33. package/src/utils/cli.js +131 -110
  34. package/src/utils/colors.js +83 -0
  35. package/src/utils/confidence-filter.js +138 -0
  36. package/src/utils/diff-engine.js +2 -1
  37. package/src/utils/global-config.js +6 -5
  38. package/src/utils/health-dashboard.js +11 -9
  39. package/src/utils/json-parser.js +4 -2
  40. package/src/utils/learning-loop.js +3 -2
  41. package/src/utils/progress-bar.js +286 -0
  42. package/src/utils/quality-gate.js +4 -2
  43. package/src/utils/retry.js +3 -1
  44. package/src/utils/schema-validator.js +314 -0
package/src/pipeline.js CHANGED
@@ -3,9 +3,9 @@
3
3
  *
4
4
  * Compress → Upload → AI Segment Analysis → AI Final Compilation → JSON + MD output.
5
5
  *
6
- * Architecture: each pipeline phase is a separate function that receives
7
- * a shared `ctx` (context) object. This makes phases independently testable
8
- * and allows the main `run()` to read as a clean sequence of steps.
6
+ * Architecture: each pipeline phase is a separate module under src/phases/.
7
+ * The shared `ctx` (context) object flows through phases. This makes phases
8
+ * independently testable and allows run() to read as a clean sequence of steps.
9
9
  *
10
10
  * v6 improvements:
11
11
  * - Confidence Scoring: every extracted item gets HIGH/MEDIUM/LOW confidence
@@ -24,1398 +24,88 @@ const path = require('path');
24
24
 
25
25
  // --- Config ---
26
26
  const config = require('./config');
27
- const {
28
- VIDEO_EXTS, DOC_EXTS, SPEED, SEG_TIME, PRESET,
29
- LOG_LEVEL, MAX_PARALLEL_UPLOADS, THINKING_BUDGET, COMPILATION_THINKING_BUDGET,
30
- validateConfig, GEMINI_MODELS, setActiveModel, getActiveModelPricing,
31
- } = config;
32
-
33
- // --- Services ---
34
- const { initFirebase, uploadToStorage, storageExists } = require('./services/firebase');
35
- const { initGemini, prepareDocsForGemini, processWithGemini, compileFinalResult, analyzeVideoForContext, cleanupGeminiFiles } = require('./services/gemini');
36
- const { compressAndSegment, probeFormat, verifySegment } = require('./services/video');
37
- const { isGitAvailable, isGitRepo, initRepo } = require('./services/git');
38
-
39
- // --- Utils ---
27
+ const { VIDEO_EXTS, DOC_EXTS, SPEED, SEG_TIME, PRESET, THINKING_BUDGET, validateConfig } = config;
28
+
29
+ // --- Shared state ---
30
+ const { getLog, isShuttingDown, PKG_ROOT, PROJECT_ROOT } = require('./phases/_shared');
31
+
32
+ // --- Pipeline phases ---
33
+ const phaseInit = require('./phases/init');
34
+ const phaseDiscover = require('./phases/discover');
35
+ const phaseServices = require('./phases/services');
36
+ const phaseProcessVideo = require('./phases/process-media');
37
+ const phaseCompile = require('./phases/compile');
38
+ const phaseOutput = require('./phases/output');
39
+ const phaseSummary = require('./phases/summary');
40
+ const phaseDeepDive = require('./phases/deep-dive');
41
+
42
+ // --- Services (for alternative modes — lazy-loaded inside each function) ---
43
+ // initFirebase, uploadToStorage, initGemini, compileFinalResult, analyzeVideoForContext,
44
+ // compressAndSegment, verifySegment, isGitAvailable, isGitRepo, initRepo
45
+
46
+ // --- Utils (for run orchestration + alt modes) ---
47
+ const { c } = require('./utils/colors');
40
48
  const { findDocsRecursive } = require('./utils/fs');
41
- const { fmtDuration, fmtBytes } = require('./utils/format');
42
- const { promptUser, promptUserText, parseArgs, showHelp, selectFolder, selectModel } = require('./utils/cli');
43
- const { parallelMap } = require('./utils/retry');
44
- const Progress = require('./utils/checkpoint');
45
- const CostTracker = require('./utils/cost-tracker');
46
- const { assessQuality, formatQualityLine, getConfidenceStats, THRESHOLDS } = require('./utils/quality-gate');
47
- const { calculateThinkingBudget, calculateCompilationBudget } = require('./utils/adaptive-budget');
48
- const { detectBoundaryContext, sliceVttForSegment } = require('./utils/context-manager');
49
+ const { promptUserText } = require('./utils/cli');
50
+ const { createProgressBar } = require('./utils/progress-bar');
49
51
  const { buildHealthReport, printHealthDashboard } = require('./utils/health-dashboard');
50
- const { loadHistory, saveHistory, buildHistoryEntry, analyzeHistory, printLearningInsights } = require('./utils/learning-loop');
51
- const { loadPreviousCompilation, generateDiff, renderDiffMarkdown } = require('./utils/diff-engine');
52
- const { promptForKey } = require('./utils/global-config');
53
-
54
- // --- Modes ---
55
- const { identifyWeaknesses, runFocusedPass, mergeFocusedResults } = require('./modes/focused-reanalysis');
56
- const { detectAllChanges, serializeReport } = require('./modes/change-detector');
57
- const { assessProgressLocal, assessProgressWithAI, mergeProgressIntoAnalysis, buildProgressSummary, renderProgressMarkdown, STATUS_ICONS } = require('./modes/progress-updater');
58
- const { discoverTopics, generateAllDocuments, writeDeepDiveOutput } = require('./modes/deep-dive');
59
- const { planTopics, generateAllDynamicDocuments, writeDynamicOutput } = require('./modes/dynamic-mode');
60
-
61
- // --- Renderers ---
62
- const { renderResultsMarkdown } = require('./renderers/markdown');
63
-
64
- // --- Logger ---
65
- const Logger = require('./logger');
66
-
67
- // Global reference — set in run()
68
- let log = null;
69
-
70
- // Graceful shutdown flag
71
- let shuttingDown = false;
72
-
73
- // ======================== PROJECT ROOT ========================
74
- // PKG_ROOT = where the package is installed (for reading prompt.json, package.json)
75
- // PROJECT_ROOT = where the user runs from (CWD) — logs, history, gemini_runs go here
76
- const PKG_ROOT = path.resolve(__dirname, '..');
77
- const PROJECT_ROOT = process.cwd();
78
-
79
- // ======================== PHASE HELPERS ========================
80
-
81
- /** Create a timing wrapper for phase profiling — also writes structured log spans */
82
- function phaseTimer(phaseName) {
83
- const t0 = Date.now();
84
- if (log && log.phaseStart) log.phaseStart(phaseName);
85
- return {
86
- end(meta = {}) {
87
- const ms = Date.now() - t0;
88
- if (log && log.phaseEnd) log.phaseEnd({ ...meta, durationMs: ms });
89
- if (log) log.step(`PHASE ${phaseName} completed in ${(ms / 1000).toFixed(1)}s`);
90
- return ms;
91
- },
92
- };
93
- }
94
-
95
- // ======================== PHASE: INIT ========================
96
-
97
- /**
98
- * Parse CLI args, validate config, initialize logger, set up shutdown handlers.
99
- * Returns the pipeline context object shared by all phases.
100
- */
101
- async function phaseInit() {
102
- const { flags, positional } = parseArgs(process.argv.slice(2));
103
-
104
- if (flags.help || flags.h) showHelp();
105
- if (flags.version || flags.v) {
106
- const pkg = JSON.parse(fs.readFileSync(path.join(PKG_ROOT, 'package.json'), 'utf8'));
107
- process.stdout.write(`v${pkg.version}\n`);
108
- throw Object.assign(new Error('VERSION_SHOWN'), { code: 'VERSION_SHOWN' });
109
- }
110
-
111
- const opts = {
112
- skipUpload: !!flags['skip-upload'],
113
- forceUpload: !!flags['force-upload'],
114
- noStorageUrl: !!flags['no-storage-url'],
115
- skipCompression: !!flags['skip-compression'],
116
- skipGemini: !!flags['skip-gemini'],
117
- resume: !!flags.resume,
118
- reanalyze: !!flags.reanalyze,
119
- dryRun: !!flags['dry-run'],
120
- userName: flags.name || null,
121
- parallel: parseInt(flags.parallel, 10) || MAX_PARALLEL_UPLOADS,
122
- logLevel: flags['log-level'] || LOG_LEVEL,
123
- outputDir: flags.output || null,
124
- thinkingBudget: parseInt(flags['thinking-budget'], 10) || THINKING_BUDGET,
125
- compilationThinkingBudget: parseInt(flags['compilation-thinking-budget'], 10) || COMPILATION_THINKING_BUDGET,
126
- parallelAnalysis: parseInt(flags['parallel-analysis'], 10) || 2, // concurrent segment analysis
127
- disableFocusedPass: !!flags['no-focused-pass'],
128
- disableLearning: !!flags['no-learning'],
129
- disableDiff: !!flags['no-diff'],
130
- deepDive: !!flags['deep-dive'],
131
- dynamic: !!flags.dynamic,
132
- request: typeof flags.request === 'string' ? flags.request : null,
133
- updateProgress: !!flags['update-progress'],
134
- repoPath: flags.repo || null,
135
- model: typeof flags.model === 'string' ? flags.model : null,
136
- };
137
-
138
- // --- Resolve folder: positional arg or interactive selection ---
139
- let folderArg = positional[0];
140
- if (!folderArg) {
141
- folderArg = await selectFolder(PROJECT_ROOT);
142
- if (!folderArg) {
143
- showHelp();
144
- }
145
- }
146
-
147
- const targetDir = path.resolve(folderArg);
148
- if (!fs.existsSync(targetDir) || !fs.statSync(targetDir).isDirectory()) {
149
- throw new Error(`"${targetDir}" is not a valid folder. Check the path and try again.`);
150
- }
151
-
152
- // --- Validate configuration (with first-run recovery) ---
153
- let configCheck = validateConfig({
154
- skipFirebase: opts.skipUpload,
155
- skipGemini: opts.skipGemini,
156
- });
157
-
158
- // First-run experience: if GEMINI_API_KEY is missing, prompt interactively
159
- if (!configCheck.valid && !opts.skipGemini && !config.GEMINI_API_KEY) {
160
- const key = await promptForKey('GEMINI_API_KEY');
161
- if (key) {
162
- // Re-validate after user provided the key
163
- configCheck = validateConfig({
164
- skipFirebase: opts.skipUpload,
165
- skipGemini: opts.skipGemini,
166
- });
167
- }
168
- }
169
-
170
- if (!configCheck.valid) {
171
- console.error('\n Configuration errors:');
172
- configCheck.errors.forEach(e => console.error(` ✗ ${e}`));
173
- console.error('\n Fix these via:');
174
- console.error(' • taskex config (save globally for all projects)');
175
- console.error(' • .env file (project-specific config)');
176
- console.error(' • --gemini-key <key> (one-time inline)\n');
177
- throw new Error('Invalid configuration. See errors above.');
178
- }
179
-
180
- // --- Initialize logger ---
181
- const logsDir = path.join(PROJECT_ROOT, 'logs');
182
- log = new Logger(logsDir, path.basename(targetDir), { level: opts.logLevel });
183
- log.patchConsole();
184
- log.step(`START processing "${path.basename(targetDir)}"`);
185
-
186
- // --- Learning Loop: load historical insights ---
187
- let learningInsights = { hasData: false, budgetAdjustment: 0, compilationBudgetAdjustment: 0 };
188
- if (!opts.disableLearning) {
189
- const history = loadHistory(PROJECT_ROOT);
190
- learningInsights = analyzeHistory(history);
191
- if (learningInsights.hasData) {
192
- printLearningInsights(learningInsights);
193
- // Apply budget adjustments from learning
194
- if (learningInsights.budgetAdjustment !== 0) {
195
- opts.thinkingBudget = Math.max(8192, opts.thinkingBudget + learningInsights.budgetAdjustment);
196
- log.step(`Learning: adjusted thinking budget → ${opts.thinkingBudget}`);
197
- }
198
- if (learningInsights.compilationBudgetAdjustment !== 0) {
199
- opts.compilationThinkingBudget = Math.max(8192, opts.compilationThinkingBudget + learningInsights.compilationBudgetAdjustment);
200
- log.step(`Learning: adjusted compilation budget → ${opts.compilationThinkingBudget}`);
201
- }
202
- }
203
- }
204
-
205
- // --- Graceful shutdown handler ---
206
- const shutdown = (signal) => {
207
- if (shuttingDown) return;
208
- shuttingDown = true;
209
- console.warn(`\n ⚠ Received ${signal} — shutting down gracefully...`);
210
- log.step(`SHUTDOWN requested (${signal})`);
211
- log.close();
212
- };
213
- process.on('SIGINT', () => shutdown('SIGINT'));
214
- process.on('SIGTERM', () => shutdown('SIGTERM'));
215
-
216
- // --- Model selection ---
217
- if (opts.model) {
218
- // CLI flag: --model <id> — validate and activate
219
- setActiveModel(opts.model);
220
- log.step(`Model set via flag: ${config.GEMINI_MODEL}`);
221
- } else {
222
- // Interactive model selection
223
- const chosenModel = await selectModel(GEMINI_MODELS, config.GEMINI_MODEL);
224
- setActiveModel(chosenModel);
225
- log.step(`Model selected: ${config.GEMINI_MODEL}`);
226
- }
227
-
228
- // --- Initialize progress tracking ---
229
- const progress = new Progress(targetDir);
230
- const costTracker = new CostTracker(getActiveModelPricing());
231
-
232
- return { opts, targetDir, progress, costTracker };
233
- }
234
-
235
- // ======================== PHASE: DISCOVER ========================
236
-
237
- /**
238
- * Discover videos and documents, resolve user name, show banner.
239
- * Returns augmented ctx with videoFiles, allDocFiles, userName.
240
- */
241
- async function phaseDiscover(ctx) {
242
- const timer = phaseTimer('discover');
243
- const { opts, targetDir, progress } = ctx;
244
-
245
- console.log('');
246
- console.log('==============================================');
247
- console.log(' Video Compress → Upload → AI Process');
248
- console.log('==============================================');
249
-
250
- // Show active flags
251
- const activeFlags = [];
252
- if (opts.skipUpload) activeFlags.push('skip-upload');
253
- if (opts.forceUpload) activeFlags.push('force-upload');
254
- if (opts.noStorageUrl) activeFlags.push('no-storage-url');
255
- if (opts.skipCompression) activeFlags.push('skip-compression');
256
- if (opts.skipGemini) activeFlags.push('skip-gemini');
257
- if (opts.resume) activeFlags.push('resume');
258
- if (opts.reanalyze) activeFlags.push('reanalyze');
259
- if (opts.dryRun) activeFlags.push('dry-run');
260
- if (activeFlags.length > 0) {
261
- console.log(` Flags: ${activeFlags.join(', ')}`);
262
- }
263
- console.log('');
264
-
265
- // --- Resume check ---
266
- if (opts.resume && progress.hasResumableState()) {
267
- progress.printResumeSummary();
268
- console.log('');
269
- }
270
-
271
- // --- Ask for user's name (or use --name flag) ---
272
- let userName = opts.userName;
273
- if (!userName) {
274
- if (opts.resume && progress.state.userName) {
275
- userName = progress.state.userName;
276
- console.log(` Using saved name: ${userName}`);
277
- } else {
278
- userName = await promptUserText(' Your name (for task assignment detection): ');
279
- }
280
- }
281
- if (!userName) {
282
- throw new Error('Name is required for personalized analysis. Use --name "Your Name" or enter it when prompted.');
283
- }
284
- log.step(`User identified as: ${userName}`);
285
-
286
- // --- Find video files ---
287
- let videoFiles = fs.readdirSync(targetDir)
288
- .filter(f => {
289
- const stat = fs.statSync(path.join(targetDir, f));
290
- return stat.isFile() && VIDEO_EXTS.includes(path.extname(f).toLowerCase());
291
- })
292
- .map(f => path.join(targetDir, f));
293
-
294
- if (videoFiles.length === 0) {
295
- throw new Error('No video files found (mp4/mkv/avi/mov/webm). Check that the folder contains video files.');
296
- }
297
-
298
- // --- Find ALL document files recursively ---
299
- const allDocFiles = findDocsRecursive(targetDir, DOC_EXTS);
300
-
301
- console.log('');
302
- console.log(` User : ${userName}`);
303
- console.log(` Source : ${targetDir}`);
304
- console.log(` Videos : ${videoFiles.length}`);
305
- console.log(` Docs : ${allDocFiles.length}`);
306
- console.log(` Speed : ${SPEED}x`);
307
- console.log(` Segments: < 5 min each (${SEG_TIME}s)`);
308
- console.log(` Model : ${config.GEMINI_MODEL}`);
309
- console.log(` Parallel: ${opts.parallel} concurrent uploads`);
310
- console.log(` Thinking: ${opts.thinkingBudget} tokens (analysis) / ${opts.compilationThinkingBudget} tokens (compilation)`);
311
- console.log('');
312
-
313
- // Save progress init
314
- progress.init(path.basename(targetDir), userName);
315
-
316
- console.log(` Found ${videoFiles.length} video file(s):`);
317
- videoFiles.forEach((f, i) => console.log(` [${i + 1}] ${path.basename(f)}`));
318
-
319
- // If multiple video files found, let user select which to process
320
- if (videoFiles.length > 1) {
321
- console.log('');
322
- const selectionInput = await promptUserText(` Which files to process? (comma-separated numbers, or "all", default: all): `);
323
- const trimmed = (selectionInput || '').trim().toLowerCase();
324
- if (trimmed && trimmed !== 'all') {
325
- const indices = trimmed.split(',').map(s => parseInt(s.trim(), 10) - 1).filter(n => !isNaN(n) && n >= 0 && n < videoFiles.length);
326
- if (indices.length > 0) {
327
- videoFiles = indices.map(i => videoFiles[i]);
328
- console.log(` → Processing ${videoFiles.length} selected file(s):`);
329
- videoFiles.forEach(f => console.log(` - ${path.basename(f)}`));
330
- } else {
331
- console.log(' → Invalid selection, processing all files');
332
- }
333
- } else {
334
- console.log(' → Processing all video files');
335
- }
336
- }
337
- log.step(`Found ${videoFiles.length} video(s): ${videoFiles.map(f => path.basename(f)).join(', ')}`);
338
- console.log('');
339
-
340
- if (allDocFiles.length > 0) {
341
- console.log(` Found ${allDocFiles.length} document(s) for context (recursive):`);
342
- allDocFiles.forEach(f => console.log(` - ${f.relPath}`));
343
- console.log('');
344
- }
345
-
346
- timer.end();
347
- return { ...ctx, videoFiles, allDocFiles, userName };
348
- }
349
-
350
- // ======================== PHASE: SERVICES ========================
351
-
352
- /**
353
- * Initialize Firebase and Gemini services, prepare context documents.
354
- * Returns augmented ctx with storage, firebaseReady, ai, contextDocs.
355
- */
356
- async function phaseServices(ctx) {
357
- const timer = phaseTimer('services');
358
- const { opts, allDocFiles } = ctx;
359
- const callName = path.basename(ctx.targetDir);
360
-
361
- console.log('Initializing services...');
362
-
363
- let storage = null;
364
- let firebaseReady = false;
365
- if (!opts.skipUpload && !opts.dryRun) {
366
- const fb = await initFirebase();
367
- storage = fb.storage;
368
- firebaseReady = fb.authenticated;
369
- } else if (opts.skipUpload) {
370
- console.log(' Firebase: skipped (--skip-upload)');
371
- } else {
372
- console.log(' Firebase: skipped (--dry-run)');
373
- }
374
-
375
- let ai = null;
376
- if (!opts.skipGemini && !opts.dryRun) {
377
- ai = await initGemini();
378
- console.log(' Gemini AI: ready');
379
- } else if (opts.skipGemini) {
380
- console.log(' Gemini AI: skipped (--skip-gemini)');
381
- } else {
382
- console.log(' Gemini AI: skipped (--dry-run)');
383
- }
384
-
385
- log.step(`Services: Firebase auth=${firebaseReady}, Gemini=${ai ? 'ready' : 'skipped'}`);
386
-
387
- // --- Prepare documents for Gemini ---
388
- let contextDocs = [];
389
- if (ai) {
390
- contextDocs = await prepareDocsForGemini(ai, allDocFiles);
391
- } else if (allDocFiles.length > 0) {
392
- console.log(` ⚠ Skipping Gemini doc preparation (AI not active)`);
393
- contextDocs = allDocFiles
394
- .filter(({ absPath }) => ['.txt', '.md', '.vtt', '.srt', '.csv', '.json', '.xml', '.html']
395
- .includes(path.extname(absPath).toLowerCase()))
396
- .map(({ absPath, relPath }) => ({
397
- type: 'inlineText',
398
- fileName: relPath,
399
- content: fs.readFileSync(absPath, 'utf8'),
400
- }));
401
- }
402
-
403
- // --- Upload documents to Firebase Storage for archival ---
404
- const docStorageUrls = {};
405
- if (firebaseReady && !opts.skipUpload) {
406
- await parallelMap(allDocFiles, async ({ absPath: docPath, relPath }) => {
407
- if (shuttingDown) return;
408
- const docStoragePath = `calls/${callName}/documents/${relPath}`;
409
- try {
410
- if (!opts.forceUpload) {
411
- const existingUrl = await storageExists(storage, docStoragePath);
412
- if (existingUrl) {
413
- docStorageUrls[relPath] = existingUrl;
414
- console.log(` ✓ Document already in Storage → ${docStoragePath}`);
415
- return;
416
- }
417
- }
418
- const url = await uploadToStorage(storage, docPath, docStoragePath);
419
- docStorageUrls[relPath] = url;
420
- console.log(` ✓ Document ${opts.forceUpload ? '(re-uploaded)' : '→'} ${docStoragePath}`);
421
- } catch (err) {
422
- console.warn(` ⚠ Document upload failed (${relPath}): ${err.message}`);
423
- }
424
- }, opts.parallel);
425
- } else if (opts.skipUpload) {
426
- console.log(' ⚠ Skipping document uploads (--skip-upload)');
427
- } else {
428
- console.log(' ⚠ Skipping document uploads (Firebase auth not configured)');
429
- }
430
- console.log('');
431
-
432
- timer.end();
433
- return { ...ctx, storage, firebaseReady, ai, contextDocs, docStorageUrls, callName };
434
- }
435
-
436
- // ======================== PHASE: PROCESS VIDEO ========================
437
-
438
- /**
439
- * Process a single video: compress → upload segments → analyze with Gemini.
440
- * Returns { fileResult, segmentAnalyses }.
441
- */
442
- async function phaseProcessVideo(ctx, videoPath, videoIndex) {
443
- const {
444
- opts, callName, storage, firebaseReady, ai, contextDocs,
445
- progress, costTracker, userName,
446
- } = ctx;
447
-
448
- const baseName = path.basename(videoPath, path.extname(videoPath));
449
- const compressedDir = path.join(ctx.targetDir, 'compressed');
450
-
451
- console.log('──────────────────────────────────────────────');
452
- console.log(`[${videoIndex + 1}/${ctx.videoFiles.length}] ${path.basename(videoPath)}`);
453
- console.log('──────────────────────────────────────────────');
454
-
455
- // ---- Compress & Segment ----
456
- log.step(`Compressing "${path.basename(videoPath)}"`);
457
- const segmentDir = path.join(compressedDir, baseName);
458
- let segments;
459
- const existingSegments = fs.existsSync(segmentDir)
460
- ? fs.readdirSync(segmentDir).filter(f => f.startsWith('segment_') && f.endsWith('.mp4')).sort()
461
- : [];
462
-
463
- if (opts.skipCompression || opts.dryRun) {
464
- if (existingSegments.length > 0) {
465
- segments = existingSegments.map(f => path.join(segmentDir, f));
466
- console.log(` ✓ Using ${segments.length} existing segment(s) (${opts.dryRun ? '--dry-run' : '--skip-compression'})`);
467
- } else {
468
- console.warn(` ⚠ No existing segments found — cannot skip compression for "${baseName}"`);
469
- if (opts.dryRun) {
470
- console.log(` [DRY-RUN] Would compress "${path.basename(videoPath)}" into segments`);
471
- return { fileResult: null, segmentAnalyses: [] };
472
- }
473
- segments = compressAndSegment(videoPath, segmentDir);
474
- log.step(`Compressed → ${segments.length} segment(s)`);
475
- }
476
- } else if (existingSegments.length > 0) {
477
- segments = existingSegments.map(f => path.join(segmentDir, f));
478
- log.step(`SKIP compression — ${segments.length} segment(s) already on disk`);
479
- console.log(` ✓ Skipped compression — ${segments.length} segment(s) already exist`);
480
- } else {
481
- segments = compressAndSegment(videoPath, segmentDir);
482
- log.step(`Compressed → ${segments.length} segment(s)`);
483
- console.log(` → ${segments.length} segment(s) created`);
484
- }
485
-
486
- progress.markCompressed(baseName, segments.length);
487
- const origSize = fs.statSync(videoPath).size;
488
- log.step(`original=${(origSize / 1048576).toFixed(2)}MB (${fmtBytes(origSize)}) | ${segments.length} segment(s)`);
489
- console.log('');
490
-
491
- const fileResult = {
492
- originalFile: path.basename(videoPath),
493
- originalSizeMB: (origSize / 1048576).toFixed(2),
494
- segmentCount: segments.length,
495
- segments: [],
496
- };
497
-
498
- // ---- Pre-validate all segments before sending to Gemini ----
499
- if (!opts.skipGemini && !opts.dryRun) {
500
- const invalidSegs = segments.filter(s => !verifySegment(s));
501
- if (invalidSegs.length > 0) {
502
- console.warn(` ⚠ Pre-validation: ${invalidSegs.length}/${segments.length} segment(s) are corrupt:`);
503
- invalidSegs.forEach(s => console.warn(` ✗ ${path.basename(s)}`));
504
- console.warn(` → Corrupt segments will be skipped during analysis.`);
505
- console.warn(` → Delete "${segmentDir}" and re-run to re-compress.`);
506
- log.warn(`Pre-validation: ${invalidSegs.length} corrupt segments in ${baseName}`);
507
- }
508
- }
509
-
510
- // ---- Upload all segments to Firebase (parallel) ----
511
- progress.setPhase('upload');
512
- const segmentMeta = [];
513
-
514
- if (!opts.skipUpload && firebaseReady && !opts.dryRun) {
515
- const metaList = segments.map((segPath) => {
516
- const segName = path.basename(segPath);
517
- const storagePath = `calls/${callName}/segments/${baseName}/${segName}`;
518
- const durStr = probeFormat(segPath, 'duration');
519
- const durSec = durStr ? parseFloat(durStr) : null;
520
- const sizeMB = (fs.statSync(segPath).size / 1048576).toFixed(2);
521
- return { segPath, segName, storagePath, durSec, sizeMB, storageUrl: null };
522
- });
523
-
524
- await parallelMap(metaList, async (meta, j) => {
525
- if (shuttingDown) return;
526
- console.log(` ── Segment ${j + 1}/${segments.length}: ${meta.segName} (upload) ──`);
527
- console.log(` Duration: ${fmtDuration(meta.durSec)} | Size: ${meta.sizeMB} MB`);
528
-
529
- const resumedUrl = progress.getUploadUrl(meta.storagePath);
530
- if (resumedUrl && opts.resume) {
531
- meta.storageUrl = resumedUrl;
532
- console.log(` ✓ Upload resumed from checkpoint`);
533
- return;
534
- }
535
-
536
- try {
537
- if (!opts.forceUpload) {
538
- const existingUrl = await storageExists(storage, meta.storagePath);
539
- if (existingUrl) {
540
- meta.storageUrl = existingUrl;
541
- log.step(`SKIP upload — ${meta.segName} already in Storage`);
542
- console.log(` ✓ Already in Storage → ${meta.storagePath}`);
543
- progress.markUploaded(meta.storagePath, meta.storageUrl);
544
- return;
545
- }
546
- }
547
- console.log(` ${opts.forceUpload ? 'Re-uploading' : 'Uploading'} to Firebase Storage...`);
548
- meta.storageUrl = await uploadToStorage(storage, meta.segPath, meta.storagePath);
549
- console.log(` ✓ ${opts.forceUpload ? 'Re-uploaded' : 'Uploaded'} → ${meta.storagePath}`);
550
- log.step(`Upload OK: ${meta.segName} → ${meta.storagePath}`);
551
- progress.markUploaded(meta.storagePath, meta.storageUrl);
552
- } catch (err) {
553
- console.error(` ✗ Firebase upload failed: ${err.message}`);
554
- log.error(`Upload FAIL: ${meta.segName} — ${err.message}`);
555
- }
556
- }, opts.parallel);
557
-
558
- segmentMeta.push(...metaList);
559
- } else {
560
- for (let j = 0; j < segments.length; j++) {
561
- const segPath = segments[j];
562
- const segName = path.basename(segPath);
563
- const storagePath = `calls/${callName}/segments/${baseName}/${segName}`;
564
- const durStr = probeFormat(segPath, 'duration');
565
- const durSec = durStr ? parseFloat(durStr) : null;
566
- const sizeMB = (fs.statSync(segPath).size / 1048576).toFixed(2);
567
-
568
- console.log(` ── Segment ${j + 1}/${segments.length}: ${segName} ──`);
569
- console.log(` Duration: ${fmtDuration(durSec)} | Size: ${sizeMB} MB`);
570
- if (opts.skipUpload) console.log(` ⚠ Upload skipped (--skip-upload)`);
571
-
572
- segmentMeta.push({ segPath, segName, storagePath, storageUrl: null, durSec, sizeMB });
573
- }
574
- }
575
-
576
- // Calculate cumulative time offsets for VTT time-slicing
577
- let cumulativeTimeSec = 0;
578
- for (const meta of segmentMeta) {
579
- meta.startTimeSec = cumulativeTimeSec;
580
- meta.endTimeSec = cumulativeTimeSec + (meta.durSec || 0) * SPEED;
581
- cumulativeTimeSec = meta.endTimeSec;
582
- }
583
-
584
- console.log('');
585
- log.step(`All ${segments.length} segment(s) processed. Starting Gemini analysis...`);
586
- console.log('');
587
-
588
- // ---- Analyze all segments with Gemini ----
589
- progress.setPhase('analyze');
590
- const geminiRunsDir = path.join(PROJECT_ROOT, 'gemini_runs', callName, baseName);
591
- fs.mkdirSync(geminiRunsDir, { recursive: true });
592
-
593
- let forceReanalyze = opts.reanalyze;
594
- if (!forceReanalyze && !opts.skipGemini && !opts.dryRun) {
595
- const allExistingRuns = fs.readdirSync(geminiRunsDir).filter(f => f.endsWith('.json'));
596
- if (allExistingRuns.length > 0) {
597
- console.log(` Found ${allExistingRuns.length} existing Gemini run file(s) in:`);
598
- console.log(` ${geminiRunsDir}`);
599
- console.log('');
600
- if (!opts.resume) {
601
- forceReanalyze = await promptUser(' Re-analyze all segments? (y/n, default: n): ');
602
- }
603
- if (forceReanalyze) {
604
- console.log(' → Will re-analyze all segments (previous runs preserved with timestamps)');
605
- log.step('User chose to re-analyze all segments');
606
- } else {
607
- console.log(' → Using cached results where available');
608
- }
609
- console.log('');
610
- }
611
- }
612
-
613
- const previousAnalyses = [];
614
- const segmentAnalyses = [];
615
- const segmentReports = []; // Quality reports for health dashboard
616
-
617
- for (let j = 0; j < segments.length; j++) {
618
- if (shuttingDown) break;
619
-
620
- const { segPath, segName, storagePath, storageUrl, durSec, sizeMB } = segmentMeta[j];
621
-
622
- console.log(` ── Segment ${j + 1}/${segments.length}: ${segName} (AI) ──`);
623
-
624
- if (opts.skipGemini) {
625
- console.log(` ⚠ Skipped (--skip-gemini)`);
626
- fileResult.segments.push({
627
- segmentFile: segName, segmentIndex: j,
628
- storagePath, storageUrl,
629
- duration: fmtDuration(durSec), durationSeconds: durSec,
630
- fileSizeMB: parseFloat(sizeMB), geminiRunFile: null, analysis: null,
631
- });
632
- console.log('');
633
- continue;
634
- }
635
-
636
- if (opts.dryRun) {
637
- console.log(` [DRY-RUN] Would analyze with ${config.GEMINI_MODEL}`);
638
- fileResult.segments.push({
639
- segmentFile: segName, segmentIndex: j,
640
- storagePath, storageUrl,
641
- duration: fmtDuration(durSec), durationSeconds: durSec,
642
- fileSizeMB: parseFloat(sizeMB), geminiRunFile: null, analysis: null,
643
- });
644
- console.log('');
645
- continue;
646
- }
647
-
648
- const runPrefix = `segment_${String(j).padStart(2, '0')}_`;
649
- const existingRuns = fs.readdirSync(geminiRunsDir)
650
- .filter(f => f.startsWith(runPrefix) && f.endsWith('.json'))
651
- .sort();
652
- const latestRunFile = existingRuns.length > 0 ? existingRuns[existingRuns.length - 1] : null;
653
- const latestRunPath = latestRunFile ? path.join(geminiRunsDir, latestRunFile) : null;
654
-
655
- let analysis = null;
656
- let geminiRunFile = null;
657
-
658
- // Skip if valid run exists and user didn't choose to re-analyze
659
- if (!forceReanalyze && latestRunPath && fs.existsSync(latestRunPath)) {
660
- try {
661
- const existingRun = JSON.parse(fs.readFileSync(latestRunPath, 'utf8'));
662
- geminiRunFile = path.relative(PROJECT_ROOT, path.join(geminiRunsDir, latestRunFile));
663
- analysis = existingRun.output.parsed || { rawResponse: existingRun.output.raw };
664
- analysis._geminiMeta = {
665
- model: existingRun.run.model,
666
- processedAt: existingRun.run.timestamp,
667
- durationMs: existingRun.run.durationMs,
668
- tokenUsage: existingRun.run.tokenUsage || null,
669
- runFile: geminiRunFile,
670
- parseSuccess: existingRun.output.parseSuccess,
671
- skipped: true,
672
- };
673
- previousAnalyses.push(analysis);
674
- // Track cached run costs too
675
- if (existingRun.run.tokenUsage) {
676
- costTracker.addSegment(segName, existingRun.run.tokenUsage, existingRun.run.durationMs, true);
677
- }
678
-
679
- // Quality gate on cached results
680
- const cachedQuality = assessQuality(analysis, {
681
- parseSuccess: existingRun.output.parseSuccess,
682
- rawLength: (existingRun.output.raw || '').length,
683
- });
684
- segmentReports.push({ segmentName: segName, qualityReport: cachedQuality, retried: false, retryImproved: false });
685
- console.log(formatQualityLine(cachedQuality, segName));
686
-
687
- const ticketCount = analysis.tickets ? analysis.tickets.length : 0;
688
- log.step(`SKIP Gemini — ${segName} already analyzed (${ticketCount} ticket(s), quality: ${cachedQuality.score}/100)`);
689
- console.log(` ✓ Already analyzed — loaded from ${latestRunFile}`);
690
- } catch (err) {
691
- console.warn(` ⚠ Existing run file corrupt, re-analyzing: ${err.message}`);
692
- analysis = null;
693
- }
694
- }
695
-
696
- if (!analysis) {
697
- // Pre-flight: verify segment is a valid MP4
698
- if (!verifySegment(segPath)) {
699
- console.error(` ✗ Segment "${segName}" is corrupt (missing moov atom / unreadable).`);
700
- console.error(` → Delete "${path.dirname(segPath)}" and re-run to re-compress.`);
701
- log.error(`Segment corrupt: ${segName} — skipping Gemini`);
702
- analysis = { error: `Segment file corrupt: ${segName}` };
703
- fileResult.segments.push({
704
- segmentFile: segName, segmentIndex: j,
705
- storagePath, storageUrl,
706
- duration: fmtDuration(durSec), durationSeconds: durSec,
707
- fileSizeMB: parseFloat(sizeMB), geminiRunFile: null, analysis,
708
- });
709
- console.log('');
710
- continue;
711
- }
712
-
713
- // === ADAPTIVE THINKING BUDGET ===
714
- // Find VTT content for this segment for complexity analysis
715
- let vttContentForAnalysis = '';
716
- for (const doc of contextDocs) {
717
- if (doc.type === 'inlineText' && (doc.fileName.endsWith('.vtt') || doc.fileName.endsWith('.srt'))) {
718
- if (segmentMeta[j].startTimeSec != null && segmentMeta[j].endTimeSec != null) {
719
- vttContentForAnalysis = sliceVttForSegment(doc.content, segmentMeta[j].startTimeSec, segmentMeta[j].endTimeSec);
720
- } else {
721
- vttContentForAnalysis = doc.content;
722
- }
723
- break;
724
- }
725
- }
726
-
727
- const budgetResult = calculateThinkingBudget({
728
- segmentIndex: j,
729
- totalSegments: segments.length,
730
- previousAnalyses,
731
- contextDocs,
732
- vttContent: vttContentForAnalysis,
733
- baseBudget: opts.thinkingBudget,
734
- });
735
- const adaptiveBudget = budgetResult.budget;
736
- console.log(` Thinking budget: ${adaptiveBudget.toLocaleString()} tokens (${budgetResult.reason})`);
737
- if (budgetResult.complexity.complexityScore > 0) {
738
- log.debug(`Segment ${j} complexity: ${budgetResult.complexity.complexityScore}/100 — words:${budgetResult.complexity.wordCount} speakers:${budgetResult.complexity.speakerCount} tech:${budgetResult.complexity.hasTechnicalTerms}`);
739
- }
740
-
741
- // === SMART BOUNDARY CONTEXT ===
742
- const prevAnalysis = previousAnalyses.length > 0 ? previousAnalyses[previousAnalyses.length - 1] : null;
743
- const boundaryCtx = detectBoundaryContext(
744
- vttContentForAnalysis,
745
- segmentMeta[j].startTimeSec || 0,
746
- segmentMeta[j].endTimeSec || 0,
747
- j,
748
- prevAnalysis
749
- );
750
-
751
- // === FIRST ATTEMPT ===
752
- let retried = false;
753
- let retryImproved = false;
754
- let geminiFileUri = null; // Gemini File API URI — reused for retry + focused pass
755
- let geminiFileMime = null;
756
- let geminiFileName = null; // Gemini resource name — needed for cleanup
757
-
758
- try {
759
- const geminiRun = await processWithGemini(
760
- ai, segPath,
761
- `${callName}_${baseName}_seg${String(j).padStart(2, '0')}`,
762
- contextDocs,
763
- previousAnalyses,
764
- userName,
765
- PKG_ROOT,
766
- {
767
- segmentIndex: j,
768
- totalSegments: segments.length,
769
- segmentStartSec: segmentMeta[j].startTimeSec,
770
- segmentEndSec: segmentMeta[j].endTimeSec,
771
- thinkingBudget: adaptiveBudget,
772
- boundaryContext: boundaryCtx,
773
- storageDownloadUrl: opts.noStorageUrl ? null : (storageUrl || null),
774
- }
775
- );
776
-
777
- const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
778
- const runFileName = `segment_${String(j).padStart(2, '0')}_${ts}.json`;
779
- const runFilePath = path.join(geminiRunsDir, runFileName);
780
- fs.writeFileSync(runFilePath, JSON.stringify(geminiRun, null, 2), 'utf8');
781
- geminiRunFile = path.relative(PROJECT_ROOT, runFilePath);
782
- log.debug(`Gemini model run saved → ${runFilePath}`);
783
-
784
- // Capture Gemini File API URI for reuse in retry / focused pass
785
- // When external URL was used, fileUri IS the storage URL — reuse it the same way
786
- geminiFileUri = geminiRun.input.videoFile.fileUri;
787
- geminiFileMime = geminiRun.input.videoFile.mimeType;
788
- geminiFileName = geminiRun.input.videoFile.geminiFileName || null;
789
- const usedExternalUrl = geminiRun.input.videoFile.usedExternalUrl || false;
790
-
791
- analysis = geminiRun.output.parsed || { rawResponse: geminiRun.output.raw };
792
- analysis._geminiMeta = {
793
- model: geminiRun.run.model,
794
- processedAt: geminiRun.run.timestamp,
795
- durationMs: geminiRun.run.durationMs,
796
- tokenUsage: geminiRun.run.tokenUsage || null,
797
- runFile: geminiRunFile,
798
- parseSuccess: geminiRun.output.parseSuccess,
799
- };
800
-
801
- // Track cost
802
- costTracker.addSegment(segName, geminiRun.run.tokenUsage, geminiRun.run.durationMs, false);
803
-
804
- // === QUALITY GATE ===
805
- const qualityReport = assessQuality(analysis, {
806
- parseSuccess: geminiRun.output.parseSuccess,
807
- rawLength: (geminiRun.output.raw || '').length,
808
- segmentIndex: j,
809
- totalSegments: segments.length,
810
- });
811
- console.log(formatQualityLine(qualityReport, segName));
812
-
813
- // === AUTO-RETRY on FAIL ===
814
- if (qualityReport.shouldRetry && !shuttingDown) {
815
- console.log(` ↻ Quality below threshold (${qualityReport.score}/${THRESHOLDS.PASS}) — retrying with enhanced hints...`);
816
- log.step(`Quality gate FAIL for ${segName} (score: ${qualityReport.score}) — retrying`);
817
- retried = true;
818
-
819
- // Boost thinking budget for retry (+25%)
820
- const retryBudget = Math.min(32768, Math.round(adaptiveBudget * 1.25));
821
-
822
- try {
823
- const retryRun = await processWithGemini(
824
- ai, segPath,
825
- `${callName}_${baseName}_seg${String(j).padStart(2, '0')}_retry`,
826
- contextDocs,
827
- previousAnalyses,
828
- userName,
829
- PKG_ROOT,
830
- {
831
- segmentIndex: j,
832
- totalSegments: segments.length,
833
- segmentStartSec: segmentMeta[j].startTimeSec,
834
- segmentEndSec: segmentMeta[j].endTimeSec,
835
- thinkingBudget: retryBudget,
836
- boundaryContext: boundaryCtx,
837
- retryHints: qualityReport.retryHints,
838
- existingFileUri: geminiFileUri,
839
- existingFileMime: geminiFileMime,
840
- existingGeminiFileName: geminiFileName,
841
- }
842
- );
843
-
844
- const retryTs = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
845
- const retryRunFileName = `segment_${String(j).padStart(2, '0')}_retry_${retryTs}.json`;
846
- const retryRunFilePath = path.join(geminiRunsDir, retryRunFileName);
847
- fs.writeFileSync(retryRunFilePath, JSON.stringify(retryRun, null, 2), 'utf8');
848
-
849
- const retryAnalysis = retryRun.output.parsed || { rawResponse: retryRun.output.raw };
850
- const retryQuality = assessQuality(retryAnalysis, {
851
- parseSuccess: retryRun.output.parseSuccess,
852
- rawLength: (retryRun.output.raw || '').length,
853
- segmentIndex: j,
854
- totalSegments: segments.length,
855
- });
856
-
857
- // Track retry cost
858
- costTracker.addSegment(`${segName}_retry`, retryRun.run.tokenUsage, retryRun.run.durationMs, false);
859
-
860
- // Use retry result if better
861
- if (retryQuality.score > qualityReport.score) {
862
- retryImproved = true;
863
- analysis = retryAnalysis;
864
- analysis._geminiMeta = {
865
- model: retryRun.run.model,
866
- processedAt: retryRun.run.timestamp,
867
- durationMs: retryRun.run.durationMs,
868
- tokenUsage: retryRun.run.tokenUsage || null,
869
- runFile: path.relative(PROJECT_ROOT, retryRunFilePath),
870
- parseSuccess: retryRun.output.parseSuccess,
871
- retryOf: geminiRunFile,
872
- };
873
- geminiRunFile = path.relative(PROJECT_ROOT, retryRunFilePath);
874
- console.log(` ✓ Retry improved quality: ${qualityReport.score} → ${retryQuality.score}`);
875
- console.log(formatQualityLine(retryQuality, segName));
876
- log.step(`Retry improved ${segName}: ${qualityReport.score} → ${retryQuality.score}`);
877
- segmentReports.push({ segmentName: segName, qualityReport: retryQuality, retried: true, retryImproved: true });
878
- } else {
879
- console.log(` ⚠ Retry did not improve (${qualityReport.score} → ${retryQuality.score}), keeping original`);
880
- segmentReports.push({ segmentName: segName, qualityReport, retried: true, retryImproved: false });
881
- }
882
- } catch (retryErr) {
883
- console.warn(` ⚠ Retry failed: ${retryErr.message} — keeping original result`);
884
- segmentReports.push({ segmentName: segName, qualityReport, retried: true, retryImproved: false });
885
- }
886
- } else {
887
- segmentReports.push({ segmentName: segName, qualityReport, retried: false, retryImproved: false });
888
- }
889
-
890
- // === FOCUSED RE-ANALYSIS (v6) ===
891
- if (!opts.disableFocusedPass && ai && !shuttingDown) {
892
- const lastReport = segmentReports[segmentReports.length - 1];
893
- const weakness = identifyWeaknesses(lastReport.qualityReport, analysis);
894
- if (weakness.shouldReanalyze) {
895
- console.log(` 🔍 Focused re-analysis: ${weakness.weakAreas.length} weak area(s) → ${weakness.weakAreas.join(', ')}`);
896
- log.step(`Focused re-analysis for ${segName}: ${weakness.weakAreas.join(', ')}`);
897
- try {
898
- const focusedResult = await runFocusedPass(ai, analysis, weakness.focusPrompt, {
899
- videoUri: geminiFileUri || null,
900
- segmentIndex: j,
901
- totalSegments: segments.length,
902
- thinkingBudget: 12288,
903
- });
904
- if (focusedResult) {
905
- analysis = mergeFocusedResults(analysis, focusedResult);
906
- if (focusedResult._focusedPassMeta) {
907
- costTracker.addSegment(`${segName}_focused`, focusedResult._focusedPassMeta, 0, false);
908
- }
909
- console.log(` ✓ Focused pass enhanced ${weakness.weakAreas.length} area(s)`);
910
- log.step(`Focused re-analysis merged for ${segName}`);
911
- } else {
912
- console.log(` ℹ Focused pass found no additional items`);
913
- }
914
- } catch (focErr) {
915
- console.warn(` ⚠ Focused re-analysis error: ${focErr.message}`);
916
- log.warn(`Focused re-analysis failed for ${segName}: ${focErr.message}`);
917
- }
918
- }
919
- }
920
-
921
- // === CONFIDENCE STATS (v6) ===
922
- const confStats = getConfidenceStats(analysis);
923
- if (confStats.total > 0) {
924
- console.log(` Confidence: ${confStats.high}H/${confStats.medium}M/${confStats.low}L/${confStats.missing}? (${confStats.coverage}% coverage)`);
925
- if (log.metric) log.metric('confidence_coverage', confStats.coverage);
926
- }
927
-
928
- previousAnalyses.push(analysis);
929
-
930
- // === CLEANUP: delete Gemini File API upload after all passes ===
931
- // Skip cleanup when external URL was used — no Gemini file was uploaded
932
- if (geminiFileName && ai && !usedExternalUrl) {
933
- cleanupGeminiFiles(ai, geminiFileName).catch(() => {});
934
- }
935
-
936
- const ticketCount = analysis.tickets ? analysis.tickets.length : 0;
937
- const tok = geminiRun.run.tokenUsage || {};
938
- const sourceLabel = usedExternalUrl ? 'via Storage URL' : (geminiFileName ? 'via File API' : 'direct');
939
- log.step(`Gemini OK: ${segName} (${sourceLabel}) — ${ticketCount} ticket(s) | ${geminiRun.run.durationMs}ms | tokens: ${tok.inputTokens || 0}in/${tok.outputTokens || 0}out/${tok.thoughtTokens || 0}think/${tok.totalTokens || 0}total`);
940
- log.debug(`Gemini parsed: ${JSON.stringify(analysis).substring(0, 500)}`);
941
- console.log(` ✓ AI analysis complete (${(geminiRun.run.durationMs / 1000).toFixed(1)}s)${retried ? (retryImproved ? ' [retry improved]' : ' [retried]') : ''}`);
942
- progress.markAnalyzed(`${baseName}_seg${j}`, geminiRunFile);
943
- } catch (err) {
944
- console.error(` ✗ Gemini failed: ${err.message}`);
945
- log.error(`Gemini FAIL: ${segName} — ${err.message}`);
946
- analysis = { error: err.message };
947
- segmentReports.push({ segmentName: segName, qualityReport: { grade: 'FAIL', score: 0, issues: [err.message] }, retried: false, retryImproved: false });
948
- }
949
- }
950
-
951
- fileResult.segments.push({
952
- segmentFile: segName,
953
- segmentIndex: j,
954
- storagePath,
955
- storageUrl,
956
- duration: fmtDuration(durSec),
957
- durationSeconds: durSec,
958
- fileSizeMB: parseFloat(sizeMB),
959
- geminiRunFile,
960
- analysis,
961
- });
962
-
963
- // Collect for final compilation (skip errored)
964
- if (analysis && !analysis.error) {
965
- const segNum = j + 1;
966
- const tagSeg = (arr) => (arr || []).forEach(item => { item.source_segment = segNum; });
967
- tagSeg(analysis.action_items);
968
- tagSeg(analysis.change_requests);
969
- tagSeg(analysis.blockers);
970
- tagSeg(analysis.scope_changes);
971
- tagSeg(analysis.file_references);
972
- if (analysis.tickets) {
973
- analysis.tickets.forEach(t => {
974
- t.source_segment = segNum;
975
- tagSeg(t.comments);
976
- tagSeg(t.code_changes);
977
- tagSeg(t.video_segments);
978
- });
979
- }
980
- if (analysis.your_tasks) {
981
- tagSeg(analysis.your_tasks.tasks_todo);
982
- tagSeg(analysis.your_tasks.tasks_waiting_on_others);
983
- tagSeg(analysis.your_tasks.decisions_needed);
984
- }
985
- segmentAnalyses.push(analysis);
986
- }
987
-
988
- console.log('');
989
- }
990
-
991
- // Compute totals for this file
992
- fileResult.compressedTotalMB = fileResult.segments
993
- .reduce((sum, s) => sum + s.fileSizeMB, 0).toFixed(2);
994
- fileResult.compressionRatio = (
995
- (1 - parseFloat(fileResult.compressedTotalMB) / parseFloat(fileResult.originalSizeMB)) * 100
996
- ).toFixed(1) + '% reduction';
997
-
998
- return { fileResult, segmentAnalyses, segmentReports };
999
- }
1000
-
1001
- // ======================== PHASE: COMPILE ========================
1002
-
1003
- /**
1004
- * Send all segment analyses to Gemini for final compilation.
1005
- * Returns { compiledAnalysis, compilationRun }.
1006
- */
1007
- async function phaseCompile(ctx, allSegmentAnalyses) {
1008
- const timer = phaseTimer('compile');
1009
- const { opts, ai, userName, callName, costTracker, progress } = ctx;
1010
-
1011
- progress.setPhase('compile');
1012
-
1013
- let compiledAnalysis = null;
1014
- let compilationRun = null;
1015
-
1016
- if (allSegmentAnalyses.length > 0 && !opts.skipGemini && !opts.dryRun && !shuttingDown) {
1017
- try {
1018
- // Adaptive compilation budget
1019
- const compBudget = calculateCompilationBudget(allSegmentAnalyses, opts.compilationThinkingBudget);
1020
- console.log(` Compilation thinking budget: ${compBudget.budget.toLocaleString()} tokens (${compBudget.reason})`);
1021
-
1022
- const compilationResult = await compileFinalResult(
1023
- ai, allSegmentAnalyses, userName, callName, PKG_ROOT,
1024
- { thinkingBudget: compBudget.budget }
1025
- );
1026
-
1027
- compiledAnalysis = compilationResult.compiled;
1028
- compilationRun = compilationResult.run;
1029
-
1030
- // Track compilation cost
1031
- if (compilationRun?.tokenUsage) {
1032
- costTracker.addCompilation(compilationRun.tokenUsage, compilationRun.durationMs);
1033
- }
1034
-
1035
- // Validate compilation output
1036
- if (compiledAnalysis) {
1037
- const hasTickets = Array.isArray(compiledAnalysis.tickets) && compiledAnalysis.tickets.length > 0;
1038
- const hasActions = Array.isArray(compiledAnalysis.action_items) && compiledAnalysis.action_items.length > 0;
1039
- const hasBlockers = Array.isArray(compiledAnalysis.blockers) && compiledAnalysis.blockers.length > 0;
1040
- const hasCRs = Array.isArray(compiledAnalysis.change_requests) && compiledAnalysis.change_requests.length > 0;
1041
-
1042
- if (!hasTickets && !hasActions && !hasBlockers && !hasCRs) {
1043
- console.warn(' ⚠ Compilation parsed OK but is missing structured data (no tickets, actions, blockers, or CRs)');
1044
- console.warn(' → Falling back to raw segment merge for full data');
1045
- log.warn('Compilation incomplete — missing all structured fields, using segment merge fallback');
1046
- compiledAnalysis._incomplete = true;
1047
- }
1048
- }
1049
-
1050
- // Save compilation run
1051
- const compilationDir = path.join(PROJECT_ROOT, 'gemini_runs', callName);
1052
- fs.mkdirSync(compilationDir, { recursive: true });
1053
- const compTs = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
1054
- const compilationFile = path.join(compilationDir, `compilation_${compTs}.json`);
1055
- const compilationPayload = {
1056
- run: compilationRun,
1057
- output: { raw: compilationResult.raw, parsed: compiledAnalysis, parseSuccess: compiledAnalysis !== null },
1058
- };
1059
- fs.writeFileSync(compilationFile, JSON.stringify(compilationPayload, null, 2), 'utf8');
1060
- log.step(`Compilation run saved → ${compilationFile}`);
1061
-
1062
- progress.markCompilationDone();
1063
-
1064
- timer.end();
1065
- return { compiledAnalysis, compilationRun, compilationPayload, compilationFile };
1066
- } catch (err) {
1067
- console.error(` ✗ Final compilation failed: ${err.message}`);
1068
- log.error(`Compilation FAIL — ${err.message}`);
1069
- console.warn(' → Falling back to raw segment merge for MD');
1070
- }
1071
- }
1072
-
1073
- timer.end();
1074
- return { compiledAnalysis, compilationRun, compilationPayload: null, compilationFile: null };
1075
- }
1076
-
1077
- // ======================== PHASE: OUTPUT ========================
1078
-
1079
- /**
1080
- * Write results JSON, generate Markdown, upload final artifacts.
1081
- * Returns { runDir, jsonPath, mdPath }.
1082
- */
1083
- async function phaseOutput(ctx, results, compiledAnalysis, compilationRun, compilationPayload) {
1084
- const timer = phaseTimer('output');
1085
- const { opts, targetDir, storage, firebaseReady, callName, progress, costTracker, userName } = ctx;
1086
-
1087
- progress.setPhase('output');
1088
-
1089
- // Determine output directory
1090
- const runTs = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
1091
- const runDir = opts.outputDir
1092
- ? path.resolve(opts.outputDir)
1093
- : path.join(targetDir, 'runs', runTs);
1094
- fs.mkdirSync(runDir, { recursive: true });
1095
- log.step(`Run folder created → ${runDir}`);
1096
-
1097
- // Copy compilation JSON into run folder
1098
- if (compilationPayload) {
1099
- const runCompFile = path.join(runDir, 'compilation.json');
1100
- fs.writeFileSync(runCompFile, JSON.stringify(compilationPayload, null, 2), 'utf8');
1101
- }
1102
-
1103
- // Attach cost summary to results
1104
- results.costSummary = costTracker.getSummary();
1105
-
1106
- // Write results JSON
1107
- const jsonPath = path.join(runDir, 'results.json');
1108
- fs.writeFileSync(jsonPath, JSON.stringify(results, null, 2), 'utf8');
1109
- log.step(`Results JSON saved → ${jsonPath}`);
1110
-
1111
- // Generate Markdown
1112
- const mdPath = path.join(runDir, 'results.md');
1113
- const totalSegs = results.files.reduce((s, f) => s + f.segmentCount, 0);
1114
-
1115
- if (compiledAnalysis && !compiledAnalysis._incomplete) {
1116
- const mdContent = renderResultsMarkdown({
1117
- compiled: compiledAnalysis,
1118
- meta: {
1119
- callName: results.callName,
1120
- processedAt: results.processedAt,
1121
- geminiModel: config.GEMINI_MODEL,
1122
- userName,
1123
- segmentCount: totalSegs,
1124
- compilation: compilationRun || null,
1125
- costSummary: results.costSummary,
1126
- segments: results.files.flatMap(f => {
1127
- const speed = results.settings?.speed || 1;
1128
- let cum = 0;
1129
- return (f.segments || []).map(s => {
1130
- const startSec = cum;
1131
- cum += (s.durationSeconds || 0) * speed;
1132
- return {
1133
- file: s.segmentFile,
1134
- duration: s.duration,
1135
- durationSeconds: s.durationSeconds,
1136
- sizeMB: s.fileSizeMB,
1137
- video: f.originalFile,
1138
- startTimeSec: startSec,
1139
- endTimeSec: cum,
1140
- segmentNumber: (s.segmentIndex || 0) + 1,
1141
- };
1142
- });
1143
- }),
1144
- settings: results.settings,
1145
- },
1146
- });
1147
- fs.writeFileSync(mdPath, mdContent, 'utf8');
1148
- log.step(`Results MD saved (compiled) → ${mdPath}`);
1149
- console.log(` ✓ Markdown report (AI-compiled) → ${path.basename(mdPath)}`);
1150
- } else {
1151
- const { renderResultsMarkdownLegacy } = require('./renderers/markdown');
1152
- const mdContent = renderResultsMarkdownLegacy(results);
1153
- fs.writeFileSync(mdPath, mdContent, 'utf8');
1154
- log.step(`Results MD saved (legacy merge) → ${mdPath}`);
1155
- console.log(` ✓ Markdown report (legacy merge) → ${path.basename(mdPath)}`);
1156
- }
1157
-
1158
- // === DIFF ENGINE (v6) ===
1159
- let diffResult = null;
1160
- if (!opts.disableDiff && compiledAnalysis) {
1161
- try {
1162
- const prevComp = loadPreviousCompilation(targetDir, runTs);
1163
- if (prevComp && prevComp.compiled) {
1164
- diffResult = generateDiff(compiledAnalysis, prevComp.compiled);
1165
- // Inject the previous run timestamp into the diff
1166
- if (diffResult.hasDiff) {
1167
- diffResult.previousTimestamp = prevComp.timestamp;
1168
- const diffMd = renderDiffMarkdown(diffResult);
1169
- fs.appendFileSync(mdPath, '\n\n' + diffMd, 'utf8');
1170
- fs.writeFileSync(path.join(runDir, 'diff.json'), JSON.stringify(diffResult, null, 2), 'utf8');
1171
- log.step(`Diff report: ${diffResult.totals.newItems} new, ${diffResult.totals.removedItems} removed, ${diffResult.totals.changedItems} changed`);
1172
- console.log(` ✓ Diff report appended (vs ${prevComp.timestamp})`);
1173
- } else {
1174
- console.log(` ℹ No differences vs previous run (${prevComp.timestamp})`);
1175
- }
1176
- } else {
1177
- console.log(` ℹ No previous compilation found for diff comparison`);
1178
- }
1179
- } catch (diffErr) {
1180
- console.warn(` ⚠ Diff generation failed: ${diffErr.message}`);
1181
- log.warn(`Diff generation error: ${diffErr.message}`);
1182
- }
1183
- }
1184
-
1185
- // Upload results to Firebase
1186
- if (firebaseReady && !opts.skipUpload && !opts.dryRun) {
1187
- try {
1188
- const resultsStoragePath = `calls/${callName}/runs/${runTs}/results.json`;
1189
- // Results always upload fresh (never skip-existing) — they change every run
1190
- const url = await uploadToStorage(storage, jsonPath, resultsStoragePath);
1191
- results.storageUrl = url;
1192
- fs.writeFileSync(jsonPath, JSON.stringify(results, null, 2), 'utf8');
1193
- console.log(` ✓ Results JSON uploaded → ${resultsStoragePath}`);
1194
-
1195
- const mdStoragePath = `calls/${callName}/runs/${runTs}/results.md`;
1196
- await uploadToStorage(storage, mdPath, mdStoragePath);
1197
- console.log(` ✓ Results MD uploaded → ${mdStoragePath}`);
1198
- } catch (err) {
1199
- console.warn(` ⚠ Results upload failed: ${err.message}`);
1200
- }
1201
- } else if (opts.skipUpload) {
1202
- console.log(' ⚠ Skipping results upload (--skip-upload)');
1203
- } else {
1204
- console.log(' ⚠ Skipping results upload (Firebase auth not configured)');
1205
- }
1206
-
1207
- timer.end();
1208
- return { runDir, jsonPath, mdPath, runTs };
1209
- }
1210
-
1211
- // ======================== PHASE: SUMMARY ========================
1212
-
1213
- /**
1214
- * Print the final summary with timing, cost, and file locations.
1215
- */
1216
- function phaseSummary(ctx, results, { jsonPath, mdPath, runTs, compilationRun }) {
1217
- const { opts, firebaseReady, callName, docStorageUrls, costTracker } = ctx;
1218
- const totalSegs = results.files.reduce((s, f) => s + f.segmentCount, 0);
1219
-
1220
- console.log('');
1221
- console.log('==============================================');
1222
- console.log(' COMPLETE');
1223
- console.log('==============================================');
1224
- console.log(` Results JSON : ${jsonPath}`);
1225
- console.log(` Results MD : ${mdPath}`);
1226
- console.log(` Files : ${results.files.length}`);
1227
- console.log(` Segments : ${totalSegs}`);
1228
- console.log(` Elapsed : ${log.elapsed()}`);
1229
- if (compilationRun) {
1230
- console.log(` Compilation : ${(compilationRun.durationMs / 1000).toFixed(1)}s | ${compilationRun.tokenUsage?.totalTokens?.toLocaleString() || '?'} tokens`);
1231
- }
1232
- results.files.forEach(f => {
1233
- console.log(` ${f.originalFile}: ${f.originalSizeMB} MB → ${f.compressedTotalMB} MB (${f.compressionRatio})`);
1234
- });
1235
-
1236
- // Cost breakdown
1237
- const cost = costTracker.getSummary();
1238
- if (cost.totalTokens > 0) {
1239
- console.log('');
1240
- console.log(` Cost estimate (${config.GEMINI_MODEL}):`);
1241
- console.log(` Input tokens : ${cost.inputTokens.toLocaleString()} ($${cost.inputCost.toFixed(4)})`);
1242
- console.log(` Output tokens : ${cost.outputTokens.toLocaleString()} ($${cost.outputCost.toFixed(4)})`);
1243
- console.log(` Thinking tokens: ${cost.thinkingTokens.toLocaleString()} ($${cost.thinkingCost.toFixed(4)})`);
1244
- console.log(` Total : ${cost.totalTokens.toLocaleString()} tokens | $${cost.totalCost.toFixed(4)}`);
1245
- console.log(` AI time : ${(cost.totalDurationMs / 1000).toFixed(1)}s`);
1246
- }
1247
-
1248
- if (firebaseReady && !opts.skipUpload) {
1249
- console.log('');
1250
- console.log(' Firebase Storage:');
1251
- console.log(` calls/${callName}/documents/ → ${Object.keys(docStorageUrls).length} doc(s)`);
1252
- console.log(` calls/${callName}/segments/ → ${totalSegs} segment(s)`);
1253
- console.log(` calls/${callName}/runs/${runTs}/ → results.json + results.md`);
1254
- if (results.storageUrl) {
1255
- console.log(` Results URL: ${results.storageUrl}`);
1256
- }
1257
- } else {
1258
- console.log('');
1259
- console.log(' ⚠ Firebase Storage: uploads skipped');
1260
- }
1261
-
1262
- // Log summary
1263
- log.summary([
1264
- `Call: ${callName}`,
1265
- `Videos: ${results.files.length}`,
1266
- `Segments: ${totalSegs}`,
1267
- `Compiled: ${results.compilation ? 'Yes (AI)' : 'No (fallback merge)'}`,
1268
- `Firebase: ${firebaseReady && !opts.skipUpload ? 'OK' : 'skipped'}`,
1269
- `Documents: ${results.contextDocuments.length}`,
1270
- `Cost: $${cost.totalCost.toFixed(4)} (${cost.totalTokens.toLocaleString()} tokens)`,
1271
- `Elapsed: ${log.elapsed()}`,
1272
- ...results.files.map(f => ` ${f.originalFile}: ${f.originalSizeMB}MB → ${f.compressedTotalMB}MB (${f.compressionRatio})`),
1273
- `Results JSON: ${jsonPath}`,
1274
- `Results MD: ${mdPath}`,
1275
- `Logs: ${log.detailedPath}`,
1276
- ]);
1277
- log.step('DONE');
1278
-
1279
- console.log(` Logs: ${log.detailedPath}`);
1280
- console.log(` ${log.minimalPath}`);
1281
- console.log('');
1282
- }
1283
-
1284
- // ======================== PHASE: DEEP DIVE ========================
1285
-
1286
- /**
1287
- * Generate explanatory documents for topics discussed in the meeting.
1288
- * Two-phase: discover topics → generate documents in parallel.
1289
- */
1290
- async function phaseDeepDive(ctx, compiledAnalysis, runDir) {
1291
- const timer = phaseTimer('deep_dive');
1292
- const { ai, callName, userName, costTracker, opts, contextDocs } = ctx;
1293
-
1294
- console.log('');
1295
- console.log('══════════════════════════════════════════════');
1296
- console.log(' DEEP DIVE — Generating Explanatory Documents');
1297
- console.log('══════════════════════════════════════════════');
1298
- console.log('');
1299
-
1300
- const thinkingBudget = opts.thinkingBudget ||
1301
- require('./config').DEEP_DIVE_THINKING_BUDGET;
1302
-
1303
- // Gather context snippets from inline text docs (for richer AI context)
1304
- const contextSnippets = [];
1305
- for (const doc of (contextDocs || [])) {
1306
- if (doc.type === 'inlineText' && doc.content) {
1307
- const snippet = doc.content.length > 3000
1308
- ? doc.content.slice(0, 3000) + '\n... (truncated)'
1309
- : doc.content;
1310
- contextSnippets.push(`[${doc.fileName}]\n${snippet}`);
1311
- }
1312
- }
1313
-
1314
- // Phase 1: Discover topics
1315
- console.log(' Phase 1: Discovering topics...');
1316
- let topicResult;
1317
- try {
1318
- topicResult = await discoverTopics(ai, compiledAnalysis, {
1319
- callName, userName, thinkingBudget, contextSnippets,
1320
- });
1321
- } catch (err) {
1322
- console.error(` ✗ Topic discovery failed: ${err.message}`);
1323
- log.error(`Deep dive topic discovery failed: ${err.message}`);
1324
- timer.end();
1325
- return;
1326
- }
1327
-
1328
- const topics = topicResult.topics;
1329
- if (!topics || topics.length === 0) {
1330
- console.log(' ℹ No topics identified for deep dive');
1331
- log.step('Deep dive: no topics discovered');
1332
- timer.end();
1333
- return;
1334
- }
1335
-
1336
- console.log(` ✓ Found ${topics.length} topic(s):`);
1337
- topics.forEach(t => console.log(` ${t.id} [${t.category}] ${t.title}`));
1338
- console.log('');
1339
-
1340
- if (topicResult.tokenUsage) {
1341
- costTracker.addSegment('deep-dive-discovery', topicResult.tokenUsage, topicResult.durationMs, false);
1342
- }
1343
- log.step(`Deep dive: ${topics.length} topics discovered in ${(topicResult.durationMs / 1000).toFixed(1)}s`);
1344
-
1345
- // Phase 2: Generate documents
1346
- console.log(` Phase 2: Generating ${topics.length} document(s)...`);
1347
- const documents = await generateAllDocuments(ai, topics, compiledAnalysis, {
1348
- callName,
1349
- userName,
1350
- thinkingBudget,
1351
- contextSnippets,
1352
- concurrency: Math.min(opts.parallelAnalysis || 2, 3), // match pipeline parallelism
1353
- onProgress: (done, total, topic) => {
1354
- console.log(` [${done}/${total}] ✓ ${topic.title}`);
1355
- },
1356
- });
1357
-
1358
- // Track cost
1359
- for (const doc of documents) {
1360
- if (doc.tokenUsage && doc.tokenUsage.totalTokens > 0) {
1361
- costTracker.addSegment(`deep-dive-${doc.topic.id}`, doc.tokenUsage, doc.durationMs, false);
1362
- }
1363
- }
1364
-
1365
- // Phase 3: Write output
1366
- const deepDiveDir = path.join(runDir, 'deep-dive');
1367
- const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
1368
- const { indexPath, stats } = writeDeepDiveOutput(deepDiveDir, documents, {
1369
- callName,
1370
- timestamp: ts,
1371
- });
1372
-
1373
- console.log('');
1374
- console.log(` ✓ Deep dive complete: ${stats.successful}/${stats.total} documents generated`);
1375
- console.log(` Output: ${path.relative(PROJECT_ROOT, deepDiveDir)}/`);
1376
- console.log(` Index: ${path.relative(PROJECT_ROOT, indexPath)}`);
1377
- if (stats.failed > 0) {
1378
- console.log(` ⚠ ${stats.failed} document(s) failed`);
1379
- }
1380
- console.log(` Tokens: ${stats.totalTokens.toLocaleString()} | Time: ${(stats.totalDurationMs / 1000).toFixed(1)}s`);
1381
- console.log('');
52
+ const { saveHistory, buildHistoryEntry } = require('./utils/learning-loop');
53
+ const { loadPreviousCompilation } = require('./utils/diff-engine');
1382
54
 
1383
- log.step(`Deep dive complete: ${stats.successful} docs, ${stats.totalTokens} tokens, ${(stats.totalDurationMs / 1000).toFixed(1)}s`);
1384
- timer.end();
1385
- }
55
+ // --- Modes & renderers (lazy-loaded inside each alternative mode function) ---
56
+ // detectAllChanges, serializeReport, assessProgressLocal, assessProgressWithAI, etc.
1386
57
 
1387
58
  // ======================== MAIN PIPELINE ========================
1388
59
 
1389
60
  async function run() {
61
+ // Lazy imports for run() — quality gate
62
+ const { assessQuality } = require('./utils/quality-gate');
63
+
1390
64
  // Phase 1: Init
1391
65
  const initCtx = await phaseInit();
1392
66
  if (!initCtx) return; // --version early exit
67
+ const log = getLog();
68
+ const bar = initCtx.progressBar;
1393
69
 
1394
70
  // --- Smart Change Detection mode ---
1395
71
  if (initCtx.opts.updateProgress) {
72
+ bar.finish();
1396
73
  return await runProgressUpdate(initCtx);
1397
74
  }
1398
75
 
1399
76
  // --- Dynamic document-only mode ---
1400
77
  if (initCtx.opts.dynamic) {
78
+ bar.finish();
1401
79
  return await runDynamic(initCtx);
1402
80
  }
1403
81
 
1404
82
  // Phase 2: Discover
83
+ bar.setPhase('discover');
1405
84
  const ctx = await phaseDiscover(initCtx);
85
+ bar.tick('Files discovered');
86
+
87
+ // --- Document-only mode: skip media processing, go straight to compilation ---
88
+ if (ctx.inputMode === 'document') {
89
+ bar.finish();
90
+ return await runDocOnly(ctx);
91
+ }
1406
92
 
1407
93
  // Phase 3: Services
94
+ bar.setPhase('services');
1408
95
  const fullCtx = await phaseServices(ctx);
96
+ bar.tick('Services ready');
1409
97
 
1410
- // Phase 4: Process each video
98
+ // Phase 4: Process each media file (video or audio)
1411
99
  const allSegmentAnalyses = [];
1412
100
  const allSegmentReports = [];
1413
101
  const pipelineStartMs = Date.now();
102
+ const mediaFiles = ctx.inputMode === 'video' ? fullCtx.videoFiles : fullCtx.audioFiles;
1414
103
  const results = {
1415
104
  processedAt: new Date().toISOString(),
1416
105
  sourceFolder: fullCtx.targetDir,
1417
106
  callName: fullCtx.callName,
1418
107
  userName: fullCtx.userName,
108
+ inputMode: ctx.inputMode,
1419
109
  settings: {
1420
110
  speed: SPEED,
1421
111
  segmentTimeSec: SEG_TIME,
@@ -1431,12 +121,14 @@ async function run() {
1431
121
  };
1432
122
 
1433
123
  fullCtx.progress.setPhase('compress');
124
+ bar.setPhase('analyze', mediaFiles.length);
1434
125
  if (log && log.phaseStart) log.phaseStart('process_videos');
1435
126
 
1436
- for (let i = 0; i < fullCtx.videoFiles.length; i++) {
1437
- if (shuttingDown) break;
127
+ for (let i = 0; i < mediaFiles.length; i++) {
128
+ if (isShuttingDown()) break;
1438
129
 
1439
- const { fileResult, segmentAnalyses, segmentReports } = await phaseProcessVideo(fullCtx, fullCtx.videoFiles[i], i);
130
+ bar.tick(path.basename(mediaFiles[i]));
131
+ const { fileResult, segmentAnalyses, segmentReports } = await phaseProcessVideo(fullCtx, mediaFiles[i], i);
1440
132
  if (fileResult) {
1441
133
  results.files.push(fileResult);
1442
134
  allSegmentAnalyses.push(...segmentAnalyses);
@@ -1444,10 +136,12 @@ async function run() {
1444
136
  }
1445
137
  }
1446
138
 
1447
- if (log && log.phaseEnd) log.phaseEnd({ videoCount: fullCtx.videoFiles.length, segmentCount: allSegmentAnalyses.length });
139
+ if (log && log.phaseEnd) log.phaseEnd({ videoCount: mediaFiles.length, segmentCount: allSegmentAnalyses.length });
1448
140
 
1449
141
  // Phase 5: Compile
142
+ bar.setPhase('compile', 1);
1450
143
  const { compiledAnalysis, compilationRun, compilationPayload, compilationFile } = await phaseCompile(fullCtx, allSegmentAnalyses);
144
+ bar.tick('Compilation done');
1451
145
 
1452
146
  // Quality gate on compilation output
1453
147
  let compilationQuality = null;
@@ -1468,8 +162,10 @@ async function run() {
1468
162
  results._compilationPayload = compilationPayload;
1469
163
 
1470
164
  // Phase 6: Output
165
+ bar.setPhase('output', 3);
1471
166
  const outputResult = await phaseOutput(fullCtx, results, compiledAnalysis, compilationRun, compilationPayload);
1472
167
  delete results._compilationPayload;
168
+ bar.tick('Output files written');
1473
169
 
1474
170
  // Phase 7: Health Dashboard
1475
171
  const healthReport = buildHealthReport({
@@ -1480,6 +176,7 @@ async function run() {
1480
176
  totalDurationMs: Date.now() - pipelineStartMs,
1481
177
  });
1482
178
  printHealthDashboard(healthReport);
179
+ bar.tick('Health dashboard generated');
1483
180
 
1484
181
  // Add health report to results
1485
182
  results.healthReport = healthReport;
@@ -1506,18 +203,263 @@ async function run() {
1506
203
  }
1507
204
 
1508
205
  // Phase 8: Summary
206
+ bar.setPhase('summary', 1);
1509
207
  phaseSummary(fullCtx, results, { ...outputResult, compilationRun });
208
+ bar.tick('Summary displayed');
1510
209
 
1511
210
  // Phase 9 (optional): Deep Dive — generate explanatory documents
1512
- if (fullCtx.opts.deepDive && compiledAnalysis && !fullCtx.opts.skipGemini && !fullCtx.opts.dryRun && !shuttingDown) {
211
+ if (fullCtx.opts.deepDive && compiledAnalysis && !fullCtx.opts.skipGemini && !fullCtx.opts.dryRun && !isShuttingDown()) {
212
+ bar.setPhase('deep-dive', 1);
1513
213
  await phaseDeepDive(fullCtx, compiledAnalysis, outputResult.runDir);
214
+ bar.tick('Deep dive complete');
1514
215
  }
1515
216
 
1516
217
  // Cleanup
218
+ bar.finish();
1517
219
  fullCtx.progress.cleanup();
1518
220
  log.close();
1519
221
  }
1520
222
 
223
+ // ======================== DOCUMENT-ONLY MODE ========================
224
+
225
+ /**
226
+ * Document-only pipeline mode: no media files, analyze documents directly.
227
+ * Sends all documents to Gemini for compilation, skipping segment processing.
228
+ *
229
+ * Triggered automatically when no video/audio files are found.
230
+ */
231
+ async function runDocOnly(ctx) {
232
+ // Lazy imports for doc-only mode
233
+ const { compileFinalResult } = require('./services/gemini');
234
+ const { validateAnalysis, formatSchemaLine } = require('./utils/schema-validator');
235
+ const { renderResultsMarkdown } = require('./renderers/markdown');
236
+ const { renderResultsHtml } = require('./renderers/html');
237
+ const { renderResultsPdf } = require('./renderers/pdf');
238
+ const { renderResultsDocx } = require('./renderers/docx');
239
+
240
+ const { opts, targetDir, allDocFiles, userName, progress, costTracker } = ctx;
241
+ const callName = path.basename(targetDir);
242
+ const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
243
+ const pipelineStartMs = Date.now();
244
+ const log = getLog();
245
+ const bar = createProgressBar({ costTracker, callName });
246
+
247
+ console.log('');
248
+ console.log(c.cyan('══════════════════════════════════════════════'));
249
+ console.log(c.heading(' DOCUMENT-ONLY MODE — Analyzing Documents'));
250
+ console.log(c.cyan('══════════════════════════════════════════════'));
251
+ console.log(` Folder: ${c.cyan(callName)}`);
252
+ console.log(` Documents: ${c.highlight(allDocFiles.length)}`);
253
+ console.log('');
254
+
255
+ // Initialize services
256
+ bar.setPhase('services', 1);
257
+ const serviceCtx = await phaseServices(ctx);
258
+ const { ai, contextDocs, storage, firebaseReady, docStorageUrls } = serviceCtx;
259
+ bar.tick('Services ready');
260
+
261
+ if (!ai) {
262
+ console.error(` ${c.error('Document-only mode requires Gemini AI. Remove --skip-gemini / --dry-run.')}`);
263
+ bar.finish();
264
+ progress.cleanup();
265
+ log.close();
266
+ return;
267
+ }
268
+
269
+ if (contextDocs.length === 0) {
270
+ console.error(` ${c.error('No documents could be loaded for analysis.')}`);
271
+ bar.finish();
272
+ progress.cleanup();
273
+ log.close();
274
+ return;
275
+ }
276
+
277
+ // Build a single analysis from all documents (send as one "segment")
278
+ bar.setPhase('compile', 1);
279
+ console.log(` Analyzing ${c.highlight(contextDocs.length)} document(s) with ${c.cyan(config.GEMINI_MODEL)}...`);
280
+
281
+ let compiledAnalysis = null;
282
+ let compilationRun = null;
283
+ let compilationPayload = null;
284
+ let compilationFile = null;
285
+
286
+ try {
287
+ const compBudget = opts.compilationThinkingBudget;
288
+ console.log(` Thinking budget: ${c.highlight(compBudget.toLocaleString())} tokens`);
289
+
290
+ // Use compileFinalResult with empty segment analyses — it will use contextDocs as primary input
291
+ const compilationResult = await compileFinalResult(
292
+ ai, [], userName, callName, PKG_ROOT,
293
+ {
294
+ thinkingBudget: compBudget,
295
+ contextDocs,
296
+ docOnlyMode: true,
297
+ }
298
+ );
299
+
300
+ compiledAnalysis = compilationResult.compiled;
301
+ compilationRun = compilationResult.run;
302
+
303
+ if (compilationRun?.tokenUsage) {
304
+ costTracker.addCompilation(compilationRun.tokenUsage, compilationRun.durationMs);
305
+ }
306
+
307
+ // Save compilation run
308
+ const compilationDir = path.join(PROJECT_ROOT, 'gemini_runs', callName);
309
+ fs.mkdirSync(compilationDir, { recursive: true });
310
+ const compTs = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
311
+ compilationFile = path.join(compilationDir, `compilation_doconly_${compTs}.json`);
312
+ compilationPayload = {
313
+ run: compilationRun,
314
+ output: { raw: compilationResult.raw, parsed: compiledAnalysis, parseSuccess: compiledAnalysis !== null },
315
+ };
316
+ fs.writeFileSync(compilationFile, JSON.stringify(compilationPayload, null, 2), 'utf8');
317
+ log.step(`Doc-only compilation saved → ${compilationFile}`);
318
+
319
+ console.log(` ${c.success(`Analysis complete (${c.yellow((compilationRun.durationMs / 1000).toFixed(1) + 's')})`)}`);
320
+
321
+ // Schema validation on doc-only compilation
322
+ if (compiledAnalysis) {
323
+ const docSchemaReport = validateAnalysis(compiledAnalysis, 'compiled');
324
+ console.log(formatSchemaLine(docSchemaReport));
325
+ if (!docSchemaReport.valid && docSchemaReport.errorCount > 0) {
326
+ log.warn(`Doc-only schema: ${docSchemaReport.summary}`);
327
+ }
328
+ }
329
+
330
+ bar.tick('Compilation done');
331
+ progress.markCompilationDone();
332
+ } catch (err) {
333
+ console.error(` ${c.error(`Document analysis failed: ${err.message}`)}`);
334
+ log.error(`Doc-only compilation FAIL — ${err.message}`);
335
+ }
336
+
337
+ // Build results structure
338
+ const results = {
339
+ processedAt: new Date().toISOString(),
340
+ sourceFolder: targetDir,
341
+ callName,
342
+ userName,
343
+ inputMode: 'document',
344
+ settings: {
345
+ geminiModel: config.GEMINI_MODEL,
346
+ thinkingBudget: opts.thinkingBudget,
347
+ },
348
+ flags: opts,
349
+ contextDocuments: contextDocs.map(d => d.fileName),
350
+ documentStorageUrls: docStorageUrls || {},
351
+ firebaseAuthenticated: firebaseReady,
352
+ files: [],
353
+ compilation: compilationRun ? {
354
+ runFile: compilationFile ? path.relative(PROJECT_ROOT, compilationFile) : null,
355
+ ...compilationRun,
356
+ } : null,
357
+ };
358
+ results.costSummary = costTracker.getSummary();
359
+
360
+ // Write output
361
+ const runDir = opts.outputDir
362
+ ? path.resolve(opts.outputDir)
363
+ : path.join(targetDir, 'runs', ts);
364
+ fs.mkdirSync(runDir, { recursive: true });
365
+
366
+ if (compilationPayload) {
367
+ fs.writeFileSync(path.join(runDir, 'compilation.json'), JSON.stringify(compilationPayload, null, 2), 'utf8');
368
+ }
369
+
370
+ const jsonPath = path.join(runDir, 'results.json');
371
+ fs.writeFileSync(jsonPath, JSON.stringify(results, null, 2), 'utf8');
372
+
373
+ const shouldRender = (type) => opts.format === 'all' || opts.format === type;
374
+
375
+ if (compiledAnalysis) {
376
+ const mdMeta = {
377
+ callName,
378
+ processedAt: results.processedAt,
379
+ geminiModel: config.GEMINI_MODEL,
380
+ userName,
381
+ segmentCount: 0,
382
+ compilation: compilationRun || null,
383
+ costSummary: results.costSummary,
384
+ segments: [],
385
+ settings: results.settings,
386
+ };
387
+
388
+ if (shouldRender('md')) {
389
+ const mdContent = renderResultsMarkdown({ compiled: compiledAnalysis, meta: mdMeta });
390
+ const mdPath = path.join(runDir, 'results.md');
391
+ fs.writeFileSync(mdPath, mdContent, 'utf8');
392
+ console.log(` ${c.success(`Markdown report → ${c.cyan(path.basename(mdPath))}`)}`);
393
+ }
394
+
395
+ if (shouldRender('html') && !opts.noHtml) {
396
+ const htmlContent = renderResultsHtml({ compiled: compiledAnalysis, meta: mdMeta });
397
+ const htmlPath = path.join(runDir, 'results.html');
398
+ fs.writeFileSync(htmlPath, htmlContent, 'utf8');
399
+ console.log(` ${c.success(`HTML report → ${c.cyan(path.basename(htmlPath))}`)}`);
400
+
401
+ // PDF (uses rendered HTML)
402
+ if (shouldRender('pdf')) {
403
+ try {
404
+ const pdfPath = path.join(runDir, 'results.pdf');
405
+ const pdfInfo = await renderResultsPdf(htmlContent, pdfPath);
406
+ console.log(` ${c.success(`PDF report → ${c.cyan(path.basename(pdfPath))}`)} ${c.dim(`(${(pdfInfo.bytes / 1024).toFixed(0)} KB)`)}`);
407
+ } catch (pdfErr) {
408
+ console.warn(` ${c.warn('PDF generation failed:')} ${pdfErr.message}`);
409
+ }
410
+ }
411
+ } else if (shouldRender('pdf')) {
412
+ // PDF requested without HTML — build HTML in memory
413
+ try {
414
+ const htmlContent = renderResultsHtml({ compiled: compiledAnalysis, meta: mdMeta });
415
+ const pdfPath = path.join(runDir, 'results.pdf');
416
+ const pdfInfo = await renderResultsPdf(htmlContent, pdfPath);
417
+ console.log(` ${c.success(`PDF report → ${c.cyan(path.basename(pdfPath))}`)} ${c.dim(`(${(pdfInfo.bytes / 1024).toFixed(0)} KB)`)}`);
418
+ } catch (pdfErr) {
419
+ console.warn(` ${c.warn('PDF generation failed:')} ${pdfErr.message}`);
420
+ }
421
+ }
422
+
423
+ // DOCX report
424
+ if (shouldRender('docx')) {
425
+ try {
426
+ const docxPath = path.join(runDir, 'results.docx');
427
+ const docxBuffer = await renderResultsDocx({ compiled: compiledAnalysis, meta: mdMeta });
428
+ fs.writeFileSync(docxPath, docxBuffer);
429
+ console.log(` ${c.success(`DOCX report → ${c.cyan(path.basename(docxPath))}`)} ${c.dim(`(${(docxBuffer.length / 1024).toFixed(0)} KB)`)}`);
430
+ } catch (docxErr) {
431
+ console.warn(` ${c.warn('DOCX generation failed:')} ${docxErr.message}`);
432
+ }
433
+ }
434
+ }
435
+
436
+ // Cost summary
437
+ const cost = costTracker.getSummary();
438
+ if (cost.totalTokens > 0) {
439
+ console.log('');
440
+ console.log(` ${c.heading(`Cost estimate (${config.GEMINI_MODEL}):`)}`);
441
+ console.log(` Input: ${c.yellow(cost.inputTokens.toLocaleString())} ${c.dim(`($${cost.inputCost.toFixed(4)})`)}`);
442
+ console.log(` Output: ${c.yellow(cost.outputTokens.toLocaleString())} ${c.dim(`($${cost.outputCost.toFixed(4)})`)}`);
443
+ console.log(` Thinking: ${c.yellow(cost.thinkingTokens.toLocaleString())} ${c.dim(`($${cost.thinkingCost.toFixed(4)})`)}`);
444
+ console.log(` Total: ${c.highlight(cost.totalTokens.toLocaleString())} tokens | ${c.highlight(`$${cost.totalCost.toFixed(4)}`)}`);
445
+ }
446
+
447
+ console.log('');
448
+ console.log(c.cyan(' ══════════════════════════════════════'));
449
+ console.log(c.heading(' Document-Only Analysis Complete'));
450
+ console.log(c.cyan(' ══════════════════════════════════════'));
451
+ console.log(` Documents: ${c.highlight(contextDocs.length)}`);
452
+ console.log(` Output: ${c.cyan(path.relative(PROJECT_ROOT, runDir) + '/')}`);
453
+ console.log(` Elapsed: ${c.yellow(log.elapsed())}`);
454
+ console.log('');
455
+
456
+ log.step('Doc-only mode complete');
457
+ log.step('DONE');
458
+ bar.finish();
459
+ progress.cleanup();
460
+ log.close();
461
+ }
462
+
1521
463
  // ======================== DYNAMIC DOCUMENT-ONLY MODE ========================
1522
464
 
1523
465
  /**
@@ -1527,17 +469,25 @@ async function run() {
1527
469
  * Triggered by --dynamic flag.
1528
470
  */
1529
471
  async function runDynamic(initCtx) {
472
+ // Lazy imports for dynamic mode
473
+ const { initGemini, analyzeVideoForContext } = require('./services/gemini');
474
+ const { initFirebase, uploadToStorage } = require('./services/firebase');
475
+ const { compressAndSegment, verifySegment } = require('./services/video');
476
+ const { planTopics, generateAllDynamicDocuments, writeDynamicOutput } = require('./modes/dynamic-mode');
477
+
1530
478
  const { opts, targetDir } = initCtx;
479
+ const log = getLog();
1531
480
  const folderName = path.basename(targetDir);
1532
481
  const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
482
+ const bar = createProgressBar({ costTracker: initCtx.costTracker, callName: folderName });
1533
483
 
1534
484
  console.log('');
1535
- console.log('══════════════════════════════════════════════');
1536
- console.log(' DYNAMIC MODE — AI Document Generation');
1537
- console.log('══════════════════════════════════════════════');
1538
- console.log(` Folder: ${folderName}`);
1539
- console.log(` Source: ${targetDir}`);
1540
- console.log(` Mode: Video + Documents (auto-detect)`);
485
+ console.log(c.cyan('══════════════════════════════════════════════'));
486
+ console.log(c.heading(' DYNAMIC MODE — AI Document Generation'));
487
+ console.log(c.cyan('══════════════════════════════════════════════'));
488
+ console.log(` Folder: ${c.cyan(folderName)}`);
489
+ console.log(` Source: ${c.dim(targetDir)}`);
490
+ console.log(` Mode: ${c.yellow('Video + Documents (auto-detect)')}`);
1541
491
  console.log('');
1542
492
 
1543
493
  // 1. Get user request (from --request flag or interactive prompt)
@@ -1546,13 +496,13 @@ async function runDynamic(initCtx) {
1546
496
  userRequest = await promptUserText(' What do you want to generate?\n (e.g. "Plan migration from X to Y", "Explain this codebase", "Create learning guide for React")\n\n → ');
1547
497
  }
1548
498
  if (!userRequest || !userRequest.trim()) {
1549
- console.error('\n A request is required for dynamic mode.');
1550
- console.error(' Use --request "your request" or enter it when prompted.');
1551
- initCtx.progress.cleanup();
499
+ console.error(`\n ${c.error('A request is required for dynamic mode.')}`);
500
+ console.error(` ${c.dim('Use --request "your request" or enter it when prompted.')}`);
501
+ bar.finish(); initCtx.progress.cleanup();
1552
502
  log.close();
1553
503
  return;
1554
504
  }
1555
- console.log(`\n Request: "${userRequest}"`);
505
+ console.log(`\n Request: ${c.highlight(`"${userRequest}"`)}`);
1556
506
  log.step(`Dynamic mode: "${userRequest}"`);
1557
507
 
1558
508
  // 2. Ask for user name (for attribution)
@@ -1564,7 +514,7 @@ async function runDynamic(initCtx) {
1564
514
 
1565
515
  // 3. Discover documents AND video files
1566
516
  console.log('');
1567
- console.log(' Discovering content...');
517
+ console.log(` ${c.dim('Discovering content...')}`);
1568
518
  const allDocFiles = findDocsRecursive(targetDir, DOC_EXTS);
1569
519
  const videoFiles = fs.readdirSync(targetDir)
1570
520
  .filter(f => {
@@ -1573,21 +523,22 @@ async function runDynamic(initCtx) {
1573
523
  })
1574
524
  .map(f => path.join(targetDir, f));
1575
525
 
1576
- console.log(` Found ${allDocFiles.length} document(s)`);
526
+ console.log(` Found ${c.highlight(allDocFiles.length)} document(s)`);
1577
527
  if (allDocFiles.length > 0) {
1578
- allDocFiles.forEach(f => console.log(` - ${f.relPath}`));
528
+ allDocFiles.forEach(f => console.log(` ${c.dim('-')} ${c.cyan(f.relPath)}`));
1579
529
  }
1580
- console.log(` Found ${videoFiles.length} video file(s)`);
530
+ console.log(` Found ${c.highlight(videoFiles.length)} video file(s)`);
1581
531
  if (videoFiles.length > 0) {
1582
- videoFiles.forEach(f => console.log(` - ${path.basename(f)}`));
532
+ videoFiles.forEach(f => console.log(` ${c.dim('-')} ${c.cyan(path.basename(f))}`));
1583
533
  }
1584
534
  log.step(`Discovered ${allDocFiles.length} document(s), ${videoFiles.length} video(s)`);
1585
535
 
1586
536
  // 4. Initialize Gemini
1587
537
  console.log('');
1588
- console.log(' Initializing AI...');
538
+ console.log(` ${c.dim('Initializing AI...')}`);
1589
539
  if (opts.skipGemini || opts.dryRun) {
1590
- console.error('Dynamic mode requires Gemini AI. Remove --skip-gemini / --dry-run.');
540
+ console.error(` ${c.error('Dynamic mode requires Gemini AI. Remove --skip-gemini / --dry-run.')}`);
541
+ bar.finish();
1591
542
  initCtx.progress.cleanup();
1592
543
  log.close();
1593
544
  return;
@@ -1596,15 +547,16 @@ async function runDynamic(initCtx) {
1596
547
  // Validate config for Gemini
1597
548
  const configCheck = validateConfig({ skipFirebase: true, skipGemini: false });
1598
549
  if (!configCheck.valid) {
1599
- console.error('\n Configuration errors:');
1600
- configCheck.errors.forEach(e => console.error(` ${e}`));
550
+ console.error(`\n ${c.error('Configuration errors:')}`);
551
+ configCheck.errors.forEach(e => console.error(` ${c.error(e)}`));
552
+ bar.finish();
1601
553
  initCtx.progress.cleanup();
1602
554
  log.close();
1603
555
  return;
1604
556
  }
1605
557
 
1606
558
  const ai = await initGemini();
1607
- console.log('Gemini AI ready');
559
+ console.log(` ${c.success('Gemini AI ready')}`);
1608
560
  const costTracker = initCtx.costTracker;
1609
561
 
1610
562
  // 5. Process video files (compress → segment → analyze for context)
@@ -1619,7 +571,7 @@ async function runDynamic(initCtx) {
1619
571
  const baseName = path.basename(videoPath, path.extname(videoPath));
1620
572
  const segmentDir = path.join(compressedDir, baseName);
1621
573
 
1622
- console.log(`\n [${vi + 1}/${videoFiles.length}] ${path.basename(videoPath)}`);
574
+ console.log(`\n ${c.dim(`[${vi + 1}/${videoFiles.length}]`)} ${c.cyan(path.basename(videoPath))}`);
1623
575
 
1624
576
  // Compress & segment (reuse existing if available)
1625
577
  let segments;
@@ -1629,23 +581,23 @@ async function runDynamic(initCtx) {
1629
581
 
1630
582
  if (existingSegments.length > 0) {
1631
583
  segments = existingSegments.map(f => path.join(segmentDir, f));
1632
- console.log(` Using ${segments.length} existing segment(s)`);
584
+ console.log(` ${c.success(`Using ${c.highlight(segments.length)} existing segment(s)`)}`);
1633
585
  log.step(`SKIP compression — ${segments.length} segment(s) already on disk for "${baseName}"`);
1634
586
  } else {
1635
587
  console.log(' Compressing & segmenting...');
1636
588
  segments = compressAndSegment(videoPath, segmentDir);
1637
- console.log(` → ${segments.length} segment(s) created`);
589
+ console.log(` → ${c.highlight(segments.length)} segment(s) created`);
1638
590
  log.step(`Compressed "${baseName}" → ${segments.length} segment(s)`);
1639
591
  }
1640
592
 
1641
593
  // Validate segments
1642
594
  const validSegments = segments.filter(s => verifySegment(s));
1643
595
  if (validSegments.length < segments.length) {
1644
- console.warn(` ${segments.length - validSegments.length} corrupt segment(s) skipped`);
596
+ console.warn(` ${c.warn(`${segments.length - validSegments.length} corrupt segment(s) skipped`)}`);
1645
597
  }
1646
598
 
1647
599
  // Analyze each segment with Gemini to extract context
1648
- console.log(` Analyzing ${validSegments.length} segment(s) for content...`);
600
+ console.log(` Analyzing ${c.highlight(validSegments.length)} segment(s) for content...`);
1649
601
  for (let si = 0; si < validSegments.length; si++) {
1650
602
  const segPath = validSegments[si];
1651
603
  const segName = path.basename(segPath);
@@ -1670,58 +622,65 @@ async function runDynamic(initCtx) {
1670
622
  costTracker.addSegment(`dynamic-video-${baseName}-${segName}`, result.tokenUsage, result.durationMs, false);
1671
623
  }
1672
624
  } catch (err) {
1673
- console.error(` Failed to analyze ${segName}: ${err.message}`);
625
+ console.error(` ${c.error(`Failed to analyze ${segName}: ${err.message}`)}`);
1674
626
  log.error(`Dynamic video analysis failed for ${displayName}: ${err.message}`);
1675
627
  }
1676
628
  }
1677
629
  }
1678
630
 
1679
631
  console.log('');
1680
- console.log(` Video analysis complete: ${videoSummaries.length} segment summary(ies)`);
632
+ console.log(` ${c.success(`Video analysis complete: ${c.highlight(videoSummaries.length)} segment summary(ies)`)}`);
1681
633
  log.step(`Dynamic video analysis: ${videoSummaries.length} segment summaries extracted`);
1682
634
  }
1683
635
 
1684
- // 6. Load document contents as snippets for AI
1685
- const INLINE_EXTS = ['.vtt', '.srt', '.txt', '.md', '.csv', '.json', '.xml', '.html'];
636
+ // 6. Load document contents as snippets for AI (text files + parsed binary docs)
637
+ const INLINE_EXTS = ['.vtt', '.srt', '.txt', '.md', '.csv', '.json', '.xml'];
638
+ const { parseDocument, canParse } = require('./services/doc-parser');
1686
639
  const docSnippets = [];
1687
640
  for (const { absPath, relPath } of allDocFiles) {
1688
- if (INLINE_EXTS.includes(path.extname(absPath).toLowerCase())) {
1689
- try {
641
+ const ext = path.extname(absPath).toLowerCase();
642
+ try {
643
+ if (INLINE_EXTS.includes(ext)) {
1690
644
  let content = fs.readFileSync(absPath, 'utf8');
1691
645
  if (content.length > 8000) {
1692
646
  content = content.slice(0, 8000) + '\n... (truncated)';
1693
647
  }
1694
648
  docSnippets.push(`[${relPath}]\n${content}`);
1695
- } catch { /* skip unreadable */ }
1696
- }
649
+ } else if (canParse(ext)) {
650
+ const result = await parseDocument(absPath, { maxLength: 8000, silent: true });
651
+ if (result.success && result.text) {
652
+ docSnippets.push(`[${relPath}]\n${result.text}`);
653
+ }
654
+ }
655
+ } catch { /* skip unreadable */ }
1697
656
  }
1698
- console.log(` Loaded ${docSnippets.length} document(s) as context for AI`);
657
+ console.log(` Loaded ${c.highlight(docSnippets.length)} document(s) as context for AI`);
1699
658
  if (videoSummaries.length > 0) {
1700
- console.log(` Plus ${videoSummaries.length} video segment summary(ies) as context`);
659
+ console.log(` Plus ${c.highlight(videoSummaries.length)} video segment summary(ies) as context`);
1701
660
  }
1702
661
  console.log('');
1703
662
 
1704
663
  const thinkingBudget = opts.thinkingBudget || THINKING_BUDGET;
1705
664
 
1706
665
  // 7. Phase 1: Plan topics
1707
- console.log(' Phase 1: Planning documents...');
666
+ console.log(` ${c.dim('Phase 1:')} Planning documents...`);
1708
667
  let planResult;
1709
668
  try {
1710
669
  planResult = await planTopics(ai, userRequest, docSnippets, {
1711
670
  folderName, userName, thinkingBudget, videoSummaries,
1712
671
  });
1713
672
  } catch (err) {
1714
- console.error(` Topic planning failed: ${err.message}`);
1715
- log.error(`Dynamic topic planning failed: ${err.message}`);
1716
- initCtx.progress.cleanup();
673
+ console.error(` ${c.error(`Topic planning failed: ${err.message}`)}`);
674
+ log.error(`Dynamic topic planning failed: ${err.message}`); bar.finish(); initCtx.progress.cleanup();
1717
675
  log.close();
1718
676
  return;
1719
677
  }
1720
678
 
1721
679
  const topics = planResult.topics;
1722
680
  if (!topics || topics.length === 0) {
1723
- console.log('No documents planned request may be too vague.');
1724
- console.log(' Try a more specific request or add context documents to the folder.');
681
+ console.log(` ${c.info('No documents planned \u2014 request may be too vague.')}`);
682
+ console.log(` ${c.dim('Try a more specific request or add context documents to the folder.')}`);
683
+ bar.finish();
1725
684
  initCtx.progress.cleanup();
1726
685
  log.close();
1727
686
  return;
@@ -1731,16 +690,16 @@ async function runDynamic(initCtx) {
1731
690
  costTracker.addSegment('dynamic-planning', planResult.tokenUsage, planResult.durationMs, false);
1732
691
  }
1733
692
 
1734
- console.log(` Planned ${topics.length} document(s) in ${(planResult.durationMs / 1000).toFixed(1)}s:`);
1735
- topics.forEach(t => console.log(` ${t.id} [${t.category}] ${t.title}`));
693
+ console.log(` ${c.success(`Planned ${c.highlight(topics.length)} document(s) in ${c.yellow((planResult.durationMs / 1000).toFixed(1) + 's')}:`)}`);
694
+ topics.forEach(t => console.log(` ${c.dim(t.id)} ${c.dim(`[${t.category}]`)} ${c.cyan(t.title)}`));
1736
695
  if (planResult.projectSummary) {
1737
- console.log(`\n Summary: ${planResult.projectSummary}`);
696
+ console.log(`\n Summary: ${c.dim(planResult.projectSummary)}`);
1738
697
  }
1739
698
  console.log('');
1740
699
  log.step(`Dynamic mode: ${topics.length} topics planned in ${(planResult.durationMs / 1000).toFixed(1)}s`);
1741
700
 
1742
701
  // 8. Phase 2: Generate all documents
1743
- console.log(` Phase 2: Generating ${topics.length} document(s)...`);
702
+ console.log(` ${c.dim('Phase 2:')} Generating ${c.highlight(topics.length)} document(s)...`);
1744
703
  const documents = await generateAllDynamicDocuments(ai, topics, userRequest, docSnippets, {
1745
704
  folderName,
1746
705
  userName,
@@ -1748,7 +707,7 @@ async function runDynamic(initCtx) {
1748
707
  videoSummaries,
1749
708
  concurrency: Math.min(opts.parallelAnalysis || 2, 3),
1750
709
  onProgress: (done, total, topic) => {
1751
- console.log(` [${done}/${total}] ${topic.title}`);
710
+ console.log(` ${c.dim(`[${done}/${total}]`)} ${c.success(topic.title)}`);
1752
711
  },
1753
712
  });
1754
713
 
@@ -1771,23 +730,23 @@ async function runDynamic(initCtx) {
1771
730
  });
1772
731
 
1773
732
  console.log('');
1774
- console.log(` Dynamic generation complete: ${stats.successful}/${stats.total} documents`);
1775
- console.log(` Output: ${path.relative(PROJECT_ROOT, runDir)}/`);
1776
- console.log(` Index: ${path.relative(PROJECT_ROOT, indexPath)}`);
733
+ console.log(` ${c.success(`Dynamic generation complete: ${c.highlight(stats.successful + '/' + stats.total)} documents`)}`);
734
+ console.log(` Output: ${c.cyan(path.relative(PROJECT_ROOT, runDir) + '/')}`);
735
+ console.log(` Index: ${c.cyan(path.relative(PROJECT_ROOT, indexPath))}`);
1777
736
  if (stats.failed > 0) {
1778
- console.log(` ${stats.failed} document(s) failed`);
737
+ console.log(` ${c.warn(`${stats.failed} document(s) failed`)}`);
1779
738
  }
1780
- console.log(` Tokens: ${stats.totalTokens.toLocaleString()} | Time: ${(stats.totalDurationMs / 1000).toFixed(1)}s`);
739
+ console.log(` Tokens: ${c.yellow(stats.totalTokens.toLocaleString())} | Time: ${c.yellow((stats.totalDurationMs / 1000).toFixed(1) + 's')}`);
1781
740
 
1782
741
  // 10. Cost summary
1783
742
  const cost = costTracker.getSummary();
1784
743
  if (cost.totalTokens > 0) {
1785
744
  console.log('');
1786
- console.log(` Cost estimate (${config.GEMINI_MODEL}):`);
1787
- console.log(` Input: ${cost.inputTokens.toLocaleString()} ($${cost.inputCost.toFixed(4)})`);
1788
- console.log(` Output: ${cost.outputTokens.toLocaleString()} ($${cost.outputCost.toFixed(4)})`);
1789
- console.log(` Thinking: ${cost.thinkingTokens.toLocaleString()} ($${cost.thinkingCost.toFixed(4)})`);
1790
- console.log(` Total: ${cost.totalTokens.toLocaleString()} tokens | $${cost.totalCost.toFixed(4)}`);
745
+ console.log(` ${c.heading(`Cost estimate (${config.GEMINI_MODEL}):`)}`);
746
+ console.log(` Input: ${c.yellow(cost.inputTokens.toLocaleString())} ${c.dim(`($${cost.inputCost.toFixed(4)})`)}`);
747
+ console.log(` Output: ${c.yellow(cost.outputTokens.toLocaleString())} ${c.dim(`($${cost.outputCost.toFixed(4)})`)}`);
748
+ console.log(` Thinking: ${c.yellow(cost.thinkingTokens.toLocaleString())} ${c.dim(`($${cost.thinkingCost.toFixed(4)})`)}`);
749
+ console.log(` Total: ${c.highlight(cost.totalTokens.toLocaleString())} tokens | ${c.highlight(`$${cost.totalCost.toFixed(4)}`)}`);
1791
750
  }
1792
751
 
1793
752
  // 11. Firebase upload (optional)
@@ -1798,29 +757,30 @@ async function runDynamic(initCtx) {
1798
757
  const storagePath = `calls/${folderName}/dynamic/${ts}`;
1799
758
  const indexStoragePath = `${storagePath}/INDEX.md`;
1800
759
  await uploadToStorage(storage, indexPath, indexStoragePath);
1801
- console.log(` Uploaded to Firebase: ${storagePath}/`);
760
+ console.log(` ${c.success(`Uploaded to Firebase: ${c.cyan(storagePath + '/')}`)}`);
1802
761
  log.step(`Firebase upload complete: ${storagePath}`);
1803
762
  }
1804
763
  } catch (fbErr) {
1805
- console.warn(` Firebase upload failed: ${fbErr.message}`);
764
+ console.warn(` ${c.warn(`Firebase upload failed: ${fbErr.message}`)}`);
1806
765
  log.warn(`Firebase upload failed: ${fbErr.message}`);
1807
766
  }
1808
767
  }
1809
768
 
1810
769
  console.log('');
1811
- console.log(' ══════════════════════════════════════');
1812
- console.log(' Dynamic Mode Complete');
1813
- console.log(' ══════════════════════════════════════');
770
+ console.log(c.cyan(' ══════════════════════════════════════'));
771
+ console.log(c.heading(' Dynamic Mode Complete'));
772
+ console.log(c.cyan(' ══════════════════════════════════════'));
1814
773
  if (videoSummaries.length > 0) {
1815
- console.log(` Videos: ${videoFiles.length} (${videoSummaries.length} segments analyzed)`);
774
+ console.log(` Videos: ${c.highlight(videoFiles.length)} (${c.yellow(videoSummaries.length)} segments analyzed)`);
1816
775
  }
1817
- console.log(` Documents: ${stats.successful}`);
1818
- console.log(` Output: ${path.relative(PROJECT_ROOT, runDir)}/`);
1819
- console.log(` Elapsed: ${log.elapsed()}`);
776
+ console.log(` Documents: ${c.highlight(stats.successful)}`);
777
+ console.log(` Output: ${c.cyan(path.relative(PROJECT_ROOT, runDir) + '/')}`);
778
+ console.log(` Elapsed: ${c.yellow(log.elapsed())}`);
1820
779
  console.log('');
1821
780
 
1822
781
  log.step(`Dynamic mode complete: ${stats.successful} docs, ${stats.totalTokens} tokens`);
1823
782
  log.step('DONE');
783
+ bar.finish();
1824
784
  initCtx.progress.cleanup();
1825
785
  log.close();
1826
786
  }
@@ -1834,16 +794,24 @@ async function runDynamic(initCtx) {
1834
794
  * Triggered by --update-progress flag.
1835
795
  */
1836
796
  async function runProgressUpdate(initCtx) {
797
+ // Lazy imports for progress-update mode
798
+ const { initGemini } = require('./services/gemini');
799
+ const { initFirebase, uploadToStorage } = require('./services/firebase');
800
+ const { isGitAvailable, isGitRepo, initRepo } = require('./services/git');
801
+ const { detectAllChanges, serializeReport } = require('./modes/change-detector');
802
+ const { assessProgressLocal, assessProgressWithAI, mergeProgressIntoAnalysis, buildProgressSummary, renderProgressMarkdown, STATUS_ICONS } = require('./modes/progress-updater');
803
+
1837
804
  const { opts, targetDir } = initCtx;
805
+ const log = getLog();
1838
806
  const callName = path.basename(targetDir);
1839
807
  const ts = new Date().toISOString().replace(/:/g, '-').replace(/\.\d+Z$/, '');
1840
808
 
1841
809
  console.log('');
1842
- console.log('==============================================');
1843
- console.log(' Smart Change Detection & Progress Update');
1844
- console.log('==============================================');
1845
- console.log(` Call: ${callName}`);
1846
- console.log(` Folder: ${targetDir}`);
810
+ console.log(c.cyan('=============================================='));
811
+ console.log(c.heading(' Smart Change Detection & Progress Update'));
812
+ console.log(c.cyan('=============================================='));
813
+ console.log(` Call: ${c.cyan(callName)}`);
814
+ console.log(` Folder: ${c.dim(targetDir)}`);
1847
815
  console.log('');
1848
816
 
1849
817
  // 0. Ensure a git repo exists for change tracking
@@ -1851,11 +819,11 @@ async function runProgressUpdate(initCtx) {
1851
819
  try {
1852
820
  const { root, created } = initRepo(targetDir);
1853
821
  if (created) {
1854
- console.log(` Initialized git repository in ${root}`);
822
+ console.log(` ${c.success(`Initialized git repository in ${c.cyan(root)}`)}`);
1855
823
  log.step(`Git repo initialized: ${root}`);
1856
824
  }
1857
825
  } catch (gitErr) {
1858
- console.warn(` Could not initialize git: ${gitErr.message}`);
826
+ console.warn(` ${c.warn(`Could not initialize git: ${gitErr.message}`)}`);
1859
827
  log.warn(`Git init failed: ${gitErr.message}`);
1860
828
  }
1861
829
  }
@@ -1863,17 +831,17 @@ async function runProgressUpdate(initCtx) {
1863
831
  // 1. Load previous compilation
1864
832
  const prev = loadPreviousCompilation(targetDir);
1865
833
  if (!prev) {
1866
- console.error('No previous analysis found. Run the full pipeline first.');
1867
- console.error(' Usage: taskex "' + callName + '"');
834
+ console.error(` ${c.error('No previous analysis found. Run the full pipeline first.')}`);
835
+ console.error(` ${c.dim('Usage: taskex "' + callName + '"')}`);
1868
836
  initCtx.progress.cleanup();
1869
837
  log.close();
1870
838
  return;
1871
839
  }
1872
- console.log(` Loaded previous analysis from ${prev.timestamp}`);
840
+ console.log(` ${c.success(`Loaded previous analysis from ${c.yellow(prev.timestamp)}`)}`);
1873
841
  log.step(`Loaded previous compilation: ${prev.timestamp}`);
1874
842
 
1875
843
  // 2. Detect changes
1876
- console.log(' Detecting changes...');
844
+ console.log(` ${c.dim('Detecting changes...')}`);
1877
845
  const changeReport = detectAllChanges({
1878
846
  repoPath: opts.repoPath,
1879
847
  callDir: targetDir,
@@ -1881,17 +849,17 @@ async function runProgressUpdate(initCtx) {
1881
849
  analysis: prev.compiled,
1882
850
  });
1883
851
 
1884
- console.log(` Git: ${changeReport.totals.commits} commits, ${changeReport.totals.filesChanged} files changed`);
1885
- console.log(` Docs: ${changeReport.totals.docsChanged} document(s) updated`);
1886
- console.log(` Items: ${changeReport.items.length} trackable items found`);
1887
- console.log(` Correlations: ${changeReport.totals.itemsWithMatches} items with matches`);
852
+ console.log(` ${c.success(`Git: ${c.highlight(changeReport.totals.commits)} commits, ${c.highlight(changeReport.totals.filesChanged)} files changed`)}`);
853
+ console.log(` ${c.success(`Docs: ${c.highlight(changeReport.totals.docsChanged)} document(s) updated`)}`);
854
+ console.log(` ${c.success(`Items: ${c.highlight(changeReport.items.length)} trackable items found`)}`);
855
+ console.log(` ${c.success(`Correlations: ${c.highlight(changeReport.totals.itemsWithMatches)} items with matches`)}`);
1888
856
  log.step(`Changes detected: ${changeReport.totals.commits} commits, ${changeReport.totals.filesChanged} files, ${changeReport.totals.docsChanged} docs`);
1889
857
  console.log('');
1890
858
 
1891
859
  // 3. Local assessment (always runs)
1892
860
  const localAssessments = assessProgressLocal(changeReport.items, changeReport.correlations);
1893
861
  const localSummary = buildProgressSummary(localAssessments);
1894
- console.log(` Local assessment: ${localSummary.done} done, ${localSummary.inProgress} in progress, ${localSummary.notStarted} not started`);
862
+ console.log(` Local assessment: ${c.green(localSummary.done + ' done')}, ${c.yellow(localSummary.inProgress + ' in progress')}, ${c.dim(localSummary.notStarted + ' not started')}`);
1895
863
 
1896
864
  // 4. AI-enhanced assessment (if Gemini is available)
1897
865
  let finalAssessments = localAssessments;
@@ -1901,7 +869,7 @@ async function runProgressUpdate(initCtx) {
1901
869
 
1902
870
  if (!opts.skipGemini) {
1903
871
  try {
1904
- console.log(' Running AI-enhanced assessment...');
872
+ console.log(` ${c.dim('Running AI-enhanced assessment...')}`);
1905
873
  const ai = await initGemini();
1906
874
  const aiResult = await assessProgressWithAI(ai, changeReport.items, changeReport, localAssessments, {
1907
875
  thinkingBudget: opts.thinkingBudget,
@@ -1912,18 +880,18 @@ async function runProgressUpdate(initCtx) {
1912
880
  aiMode = 'ai-enhanced';
1913
881
 
1914
882
  const aiSummary = buildProgressSummary(finalAssessments);
1915
- console.log(` AI assessment: ${aiSummary.done} done, ${aiSummary.inProgress} in progress, ${aiSummary.notStarted} not started`);
883
+ console.log(` ${c.success(`AI assessment: ${c.green(aiSummary.done + ' done')}, ${c.yellow(aiSummary.inProgress + ' in progress')}, ${c.dim(aiSummary.notStarted + ' not started')}`)}`);
1916
884
 
1917
885
  if (aiResult.tokenUsage) {
1918
886
  initCtx.costTracker.addSegment('progress-assessment', aiResult.tokenUsage, 0, false);
1919
887
  }
1920
888
  log.step(`AI assessment complete (model: ${aiResult.model})`);
1921
889
  } catch (err) {
1922
- console.warn(` AI assessment failed, using local assessment: ${err.message}`);
890
+ console.warn(` ${c.warn(`AI assessment failed, using local assessment: ${err.message}`)}`);
1923
891
  log.warn(`AI assessment failed: ${err.message}`);
1924
892
  }
1925
893
  } else {
1926
- console.log(' Skipping AI assessment (--skip-gemini)');
894
+ console.log(` ${c.dim('Skipping AI assessment (--skip-gemini)')}`);
1927
895
  }
1928
896
  console.log('');
1929
897
 
@@ -1965,8 +933,8 @@ async function runProgressUpdate(initCtx) {
1965
933
  fs.writeFileSync(progressMdPath, progressMd);
1966
934
  log.step(`Wrote ${progressMdPath}`);
1967
935
 
1968
- console.log(` Progress report: ${path.relative(PROJECT_ROOT, progressMdPath)}`);
1969
- console.log(` Progress data: ${path.relative(PROJECT_ROOT, progressJsonPath)}`);
936
+ console.log(` ${c.success(`Progress report: ${c.cyan(path.relative(PROJECT_ROOT, progressMdPath))}`)}`);
937
+ console.log(` ${c.success(`Progress data: ${c.cyan(path.relative(PROJECT_ROOT, progressJsonPath))}`)}`);
1970
938
 
1971
939
  // 7. Firebase upload (if available)
1972
940
  if (!opts.skipUpload) {
@@ -1976,11 +944,11 @@ async function runProgressUpdate(initCtx) {
1976
944
  const storagePath = `calls/${callName}/progress/${ts}`;
1977
945
  await uploadToStorage(storage, progressJsonPath, `${storagePath}/progress.json`);
1978
946
  await uploadToStorage(storage, progressMdPath, `${storagePath}/progress.md`);
1979
- console.log(` Uploaded to Firebase: ${storagePath}/`);
947
+ console.log(` ${c.success(`Uploaded to Firebase: ${c.cyan(storagePath + '/')}`)}`);
1980
948
  log.step(`Firebase upload complete: ${storagePath}`);
1981
949
  }
1982
950
  } catch (fbErr) {
1983
- console.warn(` Firebase upload failed: ${fbErr.message}`);
951
+ console.warn(` ${c.warn(`Firebase upload failed: ${fbErr.message}`)}`);
1984
952
  log.warn(`Firebase upload failed: ${fbErr.message}`);
1985
953
  }
1986
954
  }
@@ -1988,15 +956,15 @@ async function runProgressUpdate(initCtx) {
1988
956
  // 8. Print summary
1989
957
  const finalSummary = buildProgressSummary(finalAssessments);
1990
958
  console.log('');
1991
- console.log(' ══════════════════════════════════════');
1992
- console.log(' Progress Update Complete');
1993
- console.log(' ══════════════════════════════════════');
1994
- console.log(` ${STATUS_ICONS.DONE} Completed: ${finalSummary.done}`);
1995
- console.log(` ${STATUS_ICONS.IN_PROGRESS} In Progress: ${finalSummary.inProgress}`);
1996
- console.log(` ${STATUS_ICONS.NOT_STARTED} Not Started: ${finalSummary.notStarted}`);
1997
- console.log(` ${STATUS_ICONS.SUPERSEDED} Superseded: ${finalSummary.superseded}`);
959
+ console.log(c.cyan(' ══════════════════════════════════════'));
960
+ console.log(c.heading(' Progress Update Complete'));
961
+ console.log(c.cyan(' ══════════════════════════════════════'));
962
+ console.log(` ${STATUS_ICONS.DONE} Completed: ${c.green(finalSummary.done)}`);
963
+ console.log(` ${STATUS_ICONS.IN_PROGRESS} In Progress: ${c.yellow(finalSummary.inProgress)}`);
964
+ console.log(` ${STATUS_ICONS.NOT_STARTED} Not Started: ${c.dim(finalSummary.notStarted)}`);
965
+ console.log(` ${STATUS_ICONS.SUPERSEDED} Superseded: ${c.dim(finalSummary.superseded)}`);
1998
966
  const completionPct = finalSummary.total > 0 ? ((finalSummary.done / finalSummary.total) * 100).toFixed(0) : 0;
1999
- console.log(` Overall: ${completionPct}% complete (${finalSummary.done}/${finalSummary.total})`);
967
+ console.log(` Overall: ${c.highlight(completionPct + '%')} complete (${c.highlight(finalSummary.done + '/' + finalSummary.total)})`);
2000
968
  console.log('');
2001
969
 
2002
970
  // Cleanup
@@ -2004,4 +972,4 @@ async function runProgressUpdate(initCtx) {
2004
972
  log.close();
2005
973
  }
2006
974
 
2007
- module.exports = { run, getLog: () => log };
975
+ module.exports = { run, getLog };