task-summary-extractor 8.1.0

package/src/utils/json-parser.js

@@ -0,0 +1,245 @@
+/**
+ * Robust JSON parsing — handles markdown fences, invalid escapes,
+ * duplicate blocks, truncated output, mid-output malformations,
+ * and other Gemini output quirks.
+ */
+
+'use strict';
+
+/**
+ * Gemini sometimes produces invalid JSON escape sequences (e.g. \d, \s, \w from regex patterns).
+ * Fix them by double-escaping backslashes that aren't valid JSON escapes.
+ */
+function sanitizeJsonEscapes(text) {
+  return text.replace(/\\(?!["\\\/bfnrtu])/g, '\\\\');
+}
+
+/**
+ * Fix common mid-output JSON malformations that Gemini produces:
+ *  - Doubled closing braces/brackets:  }}, ]] → }, ]
+ *  - Doubled commas:  , , → ,
+ *  - Trailing commas before closing:  ,} → }  ,] → ]
+ *  - Lone commas after opening:  {, → {  [, → [
+ * Operates only OUTSIDE of string literals to avoid corrupting string content.
+ */
+function sanitizeMalformedJson(text) {
+  // Process character by character, only fix outside strings
+  const chars = [...text];
+  const result = [];
+  let inString = false, escape = false;
+
+  for (let i = 0; i < chars.length; i++) {
+    const ch = chars[i];
+    if (escape) { escape = false; result.push(ch); continue; }
+    if (ch === '\\' && inString) { escape = true; result.push(ch); continue; }
+    if (ch === '"') { inString = !inString; result.push(ch); continue; }
+    if (inString) { result.push(ch); continue; }
+    // Outside string — apply fixes
+    result.push(ch);
+  }
+
+  let json = result.join('');
+
+  // Now do regex-based fixes on non-string portions
+  // We'll use a split-on-strings approach: split by quoted strings, fix only non-string parts
+  const parts = json.split(/("(?:[^"\\]|\\.)*")/);
+  for (let i = 0; i < parts.length; i += 2) {
+    // Even indices are outside strings
+    let p = parts[i];
+    // Fix doubled closing braces: }} → } (but not inside nested objects — only when preceded by value)
+    // More specifically: fix }} when the first } closes an object and the second is extraneous
+    // Safest approach: fix ,} doubled patterns and trailing issues
+    p = p.replace(/,\s*,/g, ',');          // doubled commas: , , → ,
+    p = p.replace(/,\s*([}\]])/g, '$1');   // trailing comma before close: ,} → }
+    p = p.replace(/([{[\[])\s*,/g, '$1');  // comma after open: {, → {  [, → [
+    parts[i] = p;
+  }
+  json = parts.join('');
+
+  // Fix doubled closing braces/brackets that aren't valid nesting
+  // Pattern: value }} where the inner } closes an object but outer } is extra
+  // Strategy: validate by trying to parse; if it fails, try removing extra closers
+  return json;
+}
+
+/**
+ * Fix doubled closing structures (}}, ]]) by re-scanning and removing extras.
+ * Only called when initial parse fails.
+ */
+function fixDoubledClosers(text) {
+  const parts = text.split(/("(?:[^"\\]|\\.)*")/);
+  for (let i = 0; i < parts.length; i += 2) {
+    // Replace }} with } when preceded by a value (not by another })
+    // This is conservative: only fix cases where we see  value}}
+    parts[i] = parts[i].replace(/\}\s*\}/g, (match) => {
+      // Keep the match but remove one } — let the re-parser handle it
+      return '}';
+    });
+    parts[i] = parts[i].replace(/\]\s*\]/g, (match) => {
+      return ']';
+    });
+  }
+  return parts.join('');
+}
+
+function tryParse(text) {
+  try { return JSON.parse(text); } catch { return undefined; }
+}
+
+function tryParseWithSanitize(text) {
+  let result = tryParse(text);
+  if (result !== undefined) return result;
+  // Try fixing escape sequences
+  result = tryParse(sanitizeJsonEscapes(text));
+  if (result !== undefined) return result;
+  // Try fixing mid-output malformations
+  const sanitized = sanitizeMalformedJson(text);
+  result = tryParse(sanitized);
+  if (result !== undefined) return result;
+  result = tryParse(sanitizeJsonEscapes(sanitized));
+  if (result !== undefined) return result;
+  return undefined;
+}
+
+/**
+ * Attempt to repair truncated JSON.
+ * When Gemini hits the output token limit, the JSON is cut mid-way.
+ * We try to close any open structures so we can recover the data we have.
+ * Returns parsed object or undefined.
+ */
+function repairTruncatedJson(text) {
+  // Strip markdown fences
+  let json = text.replace(/```json\s*/gi, '').replace(/```\s*/g, '').trim();
+
+  const firstBrace = json.indexOf('{');
+  if (firstBrace === -1) return undefined;
+  json = json.substring(firstBrace);
+
+  // Apply mid-output sanitization FIRST (fix doubled commas, trailing commas, etc.)
+  json = sanitizeMalformedJson(json);
+
+  // Find where the valid JSON roughly ends — trim trailing incomplete values
+  // Remove trailing incomplete string value  (e.g. "key": "partial text...)
+  json = json.replace(/:\s*"[^"]*$/, ': null');
+  // Remove trailing incomplete key (e.g.  , "partial_key...)
+  json = json.replace(/,\s*"[^"]*$/, '');
+  // Remove trailing comma or colon
+  json = json.replace(/,\s*$/, '');
+  json = json.replace(/:\s*$/, ': null');
+
+  // Track the STACK of open structures in order, so we close them in reverse
+  const stack = []; // '{' or '['
+  let inString = false, escape = false;
+  for (let i = 0; i < json.length; i++) {
+    const ch = json[i];
+    if (escape) { escape = false; continue; }
+    if (ch === '\\' && inString) { escape = true; continue; }
+    if (ch === '"') { inString = !inString; continue; }
+    if (inString) continue;
+    if (ch === '{') stack.push('{');
+    else if (ch === '}') { if (stack.length && stack[stack.length - 1] === '{') stack.pop(); }
+    else if (ch === '[') stack.push('[');
+    else if (ch === ']') { if (stack.length && stack[stack.length - 1] === '[') stack.pop(); }
+  }
+
+  // Close open structures in reverse order
+  // First remove any trailing comma before closing
+  json = json.replace(/,\s*$/, '');
+  while (stack.length > 0) {
+    const open = stack.pop();
+    json += open === '{' ? '}' : ']';
+  }
+
+  return tryParseWithSanitize(json);
+}
+
+/**
+ * Aggressive repair: try fixing doubled closers structure-wide.
+ * Only used as a last resort when other strategies fail.
+ */
+function repairDoubledClosers(text) {
+  let json = text.replace(/```json\s*/gi, '').replace(/```\s*/g, '').trim();
+  const firstBrace = json.indexOf('{');
+  if (firstBrace === -1) return undefined;
+  json = json.substring(firstBrace);
+
+  // Sanitize mid-output issues
+  json = sanitizeMalformedJson(json);
+
+  // Aggressively fix doubled closers
+  let prevJson = '';
+  let iterations = 0;
+  while (json !== prevJson && iterations < 10) {
+    prevJson = json;
+    json = fixDoubledClosers(json);
+    iterations++;
+  }
+
+  // Try parsing as-is
+  let result = tryParseWithSanitize(json);
+  if (result !== undefined) return result;
+
+  // If still failing, combine with truncation repair
+  return repairTruncatedJson(json);
+}
+
+/**
+ * Extract JSON from raw AI output using multiple strategies:
+ *  1. Strip markdown fences and parse (with escape + malformation sanitization)
+ *  2. Brace-matching for first complete JSON object
+ *  3. Regex extraction between fences
+ *  4. Doubled-closer repair (fix }}, ]] etc.)
+ *  5. Truncation repair — close open structures and recover partial data
+ * Returns parsed object or null.
+ */
+function extractJson(rawText) {
+  // Strategy 1: Strip all markdown fences and try to parse
+  const cleaned = rawText.replace(/```json\s*/gi, '').replace(/```\s*/g, '').trim();
+  let parsed = tryParseWithSanitize(cleaned);
+  if (parsed !== undefined) return parsed;
+
+  // Strategy 2: Extract first complete JSON object using brace matching
+  const firstBrace = rawText.indexOf('{');
+  if (firstBrace !== -1) {
+    let depth = 0, end = -1, inStr = false, esc = false;
+    for (let ci = firstBrace; ci < rawText.length; ci++) {
+      const c = rawText[ci];
+      if (esc) { esc = false; continue; }
+      if (c === '\\' && inStr) { esc = true; continue; }
+      if (c === '"') { inStr = !inStr; continue; }
+      if (inStr) continue;
+      if (c === '{') depth++;
+      else if (c === '}') { depth--; if (depth === 0) { end = ci; break; } }
+    }
+    if (end !== -1) {
+      parsed = tryParseWithSanitize(rawText.substring(firstBrace, end + 1));
+      if (parsed !== undefined) return parsed;
+    }
+  }
+
+  // Strategy 3: Regex extraction of JSON block between fences
+  const fenceMatch = rawText.match(/```(?:json)?\s*([\s\S]*?)```/i);
+  if (fenceMatch) {
+    parsed = tryParseWithSanitize(fenceMatch[1].trim());
+    if (parsed !== undefined) return parsed;
+  }
+
+  // Strategy 4: Truncation repair — Gemini hit token limit mid-JSON
+  // This is common for large compilation outputs (safest repair, stack-based)
+  parsed = repairTruncatedJson(rawText);
+  if (parsed !== undefined) {
+    console.warn('  ⚠ JSON was truncated — recovered partial data by closing open structures');
+    return parsed;
+  }
+
+  // Strategy 5: Fix doubled closers and mid-output structural errors (aggressive, last resort)
+  parsed = repairDoubledClosers(rawText);
+  if (parsed !== undefined) {
+    console.warn('  ⚠ JSON had structural errors (doubled braces/commas) — repaired');
+    return parsed;
+  }
+
+  return null;
+}
+
+module.exports = { extractJson, sanitizeJsonEscapes, sanitizeMalformedJson, fixDoubledClosers, tryParse, tryParseWithSanitize, repairTruncatedJson, repairDoubledClosers };

package/src/utils/learning-loop.js

@@ -0,0 +1,301 @@
+/**
+ * Learning Loop — stores pipeline execution history and uses it to
+ * auto-adjust quality thresholds, thinking budgets, and extraction strategies.
+ *
+ * After each run, the health report + key metrics are appended to history.json.
+ * Before each run, historical data is analyzed to produce recommendations
+ * for the current execution.
+ *
+ * This creates a feedback loop: each run gets smarter based on past performance.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const HISTORY_FILE = 'history.json';
+const MAX_HISTORY_ENTRIES = 50; // Keep last 50 runs
+
+// ======================== HISTORY I/O ========================
+
+/**
+ * Load run history from disk.
+ *
+ * @param {string} projectRoot - Project root directory
+ * @returns {Array} Array of historical run entries
+ */
+function loadHistory(projectRoot) {
+  const historyPath = path.join(projectRoot, HISTORY_FILE);
+  try {
+    if (fs.existsSync(historyPath)) {
+      const data = JSON.parse(fs.readFileSync(historyPath, 'utf8'));
+      return Array.isArray(data) ? data : [];
+    }
+  } catch (err) {
+    console.warn(`  ⚠ Could not load history: ${err.message}`);
+  }
+  return [];
+}
+
+/**
+ * Save a new run entry to history.
+ *
+ * @param {string} projectRoot - Project root directory
+ * @param {object} entry - Run entry to append
+ */
+function saveHistory(projectRoot, entry) {
+  const historyPath = path.join(projectRoot, HISTORY_FILE);
+  try {
+    const history = loadHistory(projectRoot);
+    history.push(entry);
+
+    // Trim to max entries
+    const trimmed = history.slice(-MAX_HISTORY_ENTRIES);
+    fs.writeFileSync(historyPath, JSON.stringify(trimmed, null, 2), 'utf8');
+  } catch (err) {
+    console.warn(`  ⚠ Could not save history: ${err.message}`);
+  }
+}
+
+// ======================== RUN ENTRY BUILDER ========================
+
+/**
+ * Build a compact history entry from pipeline execution data.
+ *
+ * @param {object} params
+ * @param {string} params.callName - Name of the call
+ * @param {object} params.healthReport - From health-dashboard.js
+ * @param {object} params.costSummary - From CostTracker
+ * @param {number} params.segmentCount - Number of segments
+ * @param {object} [params.compilationQuality] - Quality report for compilation
+ * @param {number} [params.baseBudget] - Thinking budget used
+ * @param {number} [params.compilationBudget] - Compilation budget used
+ * @param {boolean} [params.hadFocusedPasses] - Whether focused re-analysis was used
+ * @returns {object} Compact history entry
+ */
+function buildHistoryEntry(params) {
+  const {
+    callName,
+    healthReport,
+    costSummary = {},
+    segmentCount = 0,
+    compilationQuality = null,
+    baseBudget = 0,
+    compilationBudget = 0,
+    hadFocusedPasses = false,
+  } = params;
+
+  const hr = healthReport || {};
+  const summary = hr.summary || {};
+  const extraction = hr.extraction || {};
+  const retry = hr.retry || {};
+  const efficiency = hr.efficiency || {};
+
+  return {
+    timestamp: new Date().toISOString(),
+    callName,
+    segmentCount,
+    quality: {
+      avgScore: summary.avgQualityScore || 0,
+      minScore: summary.minQualityScore || 0,
+      maxScore: summary.maxQualityScore || 0,
+      grades: summary.grades || {},
+      parseSuccessRate: summary.parseSuccessRate || 0,
+    },
+    extraction: {
+      totalItems: extraction.totalItems || 0,
+      tickets: extraction.totalTickets || 0,
+      crs: extraction.totalChangeRequests || 0,
+      actions: extraction.totalActionItems || 0,
+      blockers: extraction.totalBlockers || 0,
+      scopes: extraction.totalScopeChanges || 0,
+    },
+    cost: {
+      totalTokens: costSummary.totalTokens || 0,
+      totalCost: costSummary.totalCost || 0,
+      tokensPerItem: efficiency.tokensPerExtractedItem || 0,
+    },
+    retry: {
+      segmentsRetried: retry.segmentsRetried || 0,
+      retriesImproved: retry.retriesImproved || 0,
+    },
+    budgets: {
+      baseBudget,
+      compilationBudget,
+    },
+    compilation: compilationQuality ? {
+      score: compilationQuality.score,
+      grade: compilationQuality.grade,
+    } : null,
+    focusedPasses: hadFocusedPasses,
+  };
+}
+
+// ======================== TREND ANALYSIS ========================
+
+/**
+ * Analyze historical trends and produce recommendations for the next run.
+ *
+ * @param {Array} history - Array of historical run entries
+ * @returns {object} Recommendations
+ */
+function analyzeHistory(history) {
+  if (!history || history.length === 0) {
+    return {
+      hasData: false,
+      recommendations: [],
+      budgetAdjustment: 0,
+      compilationBudgetAdjustment: 0,
+      qualityThresholdAdjustment: 0,
+      avgQuality: 0,
+      trend: 'none',
+      runCount: 0,
+    };
+  }
+
+  const recent = history.slice(-10); // Last 10 runs
+  const recommendations = [];
+
+  // Quality trend
+  const qualities = recent.map(r => r.quality?.avgScore || 0).filter(q => q > 0);
+  const avgQuality = qualities.length > 0
+    ? qualities.reduce((a, b) => a + b, 0) / qualities.length
+    : 0;
+
+  // Determine trend direction
+  let trend = 'stable';
+  if (qualities.length >= 3) {
+    const firstHalf = qualities.slice(0, Math.floor(qualities.length / 2));
+    const secondHalf = qualities.slice(Math.floor(qualities.length / 2));
+    const firstAvg = firstHalf.reduce((a, b) => a + b, 0) / firstHalf.length;
+    const secondAvg = secondHalf.reduce((a, b) => a + b, 0) / secondHalf.length;
+    if (secondAvg > firstAvg + 5) trend = 'improving';
+    else if (secondAvg < firstAvg - 5) trend = 'declining';
+  }
+
+  // Budget adjustment — if quality is consistently low, boost budget
+  let budgetAdjustment = 0;
+  let compilationBudgetAdjustment = 0;
+  let qualityThresholdAdjustment = 0;
+
+  if (avgQuality < 45 && qualities.length >= 3) {
+    budgetAdjustment = 4096; // +4K tokens
+    recommendations.push(
+      `Low average quality (${avgQuality.toFixed(0)}/100) across ${qualities.length} runs — boosting thinking budget by +4096 tokens`
+    );
+  } else if (avgQuality > 80 && qualities.length >= 3) {
+    budgetAdjustment = -2048; // Save tokens if quality is great
+    recommendations.push(
+      `High average quality (${avgQuality.toFixed(0)}/100) — reducing thinking budget by 2048 tokens to save cost`
+    );
+  }
+
+  // Retry effectiveness
+  const retryRuns = recent.filter(r => r.retry?.segmentsRetried > 0);
+  if (retryRuns.length > 0) {
+    const totalRetried = retryRuns.reduce((s, r) => s + (r.retry?.segmentsRetried || 0), 0);
+    const totalImproved = retryRuns.reduce((s, r) => s + (r.retry?.retriesImproved || 0), 0);
+    const retrySuccessRate = totalRetried > 0 ? (totalImproved / totalRetried * 100).toFixed(0) : 0;
+
+    if (retrySuccessRate < 30 && totalRetried >= 3) {
+      recommendations.push(
+        `Retry success rate is low (${retrySuccessRate}% of ${totalRetried} retries improved) — consider increasing base thinking budget instead of relying on retries`
+      );
+      budgetAdjustment = Math.max(budgetAdjustment, 2048);
+    }
+  }
+
+  // Cost efficiency
+  const costs = recent.map(r => r.cost?.tokensPerItem || 0).filter(c => c > 0);
+  if (costs.length >= 3) {
+    const avgCostPerItem = costs.reduce((a, b) => a + b, 0) / costs.length;
+    if (avgCostPerItem > 50000) {
+      recommendations.push(
+        `High token usage per item (${avgCostPerItem.toFixed(0)} tokens/item) — extraction may be inefficient`
+      );
+    }
+  }
+
+  // Compilation quality
+  const compilationScores = recent.map(r => r.compilation?.score || 0).filter(s => s > 0);
+  if (compilationScores.length >= 2) {
+    const avgCompScore = compilationScores.reduce((a, b) => a + b, 0) / compilationScores.length;
+    if (avgCompScore < 50) {
+      compilationBudgetAdjustment = 4096;
+      recommendations.push(
+        `Low compilation quality (avg ${avgCompScore.toFixed(0)}/100) — boosting compilation budget by +4096`
+      );
+    }
+  }
+
+  // Focused pass effectiveness
+  const focusedRuns = recent.filter(r => r.focusedPasses);
+  if (focusedRuns.length > 0 && focusedRuns.length < recent.length * 0.3) {
+    recommendations.push(
+      `Focused re-analysis was used in ${focusedRuns.length}/${recent.length} runs — system is self-correcting effectively`
+    );
+  }
+
+  // Quality threshold — if everything consistently passes, tighten threshold
+  const failRuns = recent.filter(r => r.quality?.grades?.FAIL > 0);
+  if (failRuns.length === 0 && recent.length >= 5 && avgQuality > 70) {
+    qualityThresholdAdjustment = 5; // Raise PASS threshold by 5
+    recommendations.push(
+      `No quality failures in last ${recent.length} runs (avg ${avgQuality.toFixed(0)}) — consider raising quality threshold`
+    );
+  }
+
+  return {
+    hasData: true,
+    recommendations,
+    budgetAdjustment,
+    compilationBudgetAdjustment,
+    qualityThresholdAdjustment,
+    avgQuality,
+    trend,
+    runCount: history.length,
+  };
+}
+
+// ======================== PRINT INSIGHTS ========================
+
+/**
+ * Print learning insights to the console.
+ *
+ * @param {object} insights - From analyzeHistory()
+ */
+function printLearningInsights(insights) {
+  if (!insights.hasData) return;
+
+  console.log('');
+  console.log('  📈 Learning Insights:');
+  console.log(`    Historical runs : ${insights.runCount}`);
+  console.log(`    Quality trend   : ${insights.trend} (avg: ${insights.avgQuality.toFixed(0)}/100)`);
+
+  if (insights.budgetAdjustment !== 0) {
+    const dir = insights.budgetAdjustment > 0 ? '+' : '';
+    console.log(`    Budget adjust   : ${dir}${insights.budgetAdjustment} tokens (analysis)`);
+  }
+  if (insights.compilationBudgetAdjustment !== 0) {
+    const dir = insights.compilationBudgetAdjustment > 0 ? '+' : '';
+    console.log(`    Budget adjust   : ${dir}${insights.compilationBudgetAdjustment} tokens (compilation)`);
+  }
+
+  if (insights.recommendations.length > 0) {
+    console.log('    Recommendations :');
+    for (const rec of insights.recommendations) {
+      console.log(`      • ${rec}`);
+    }
+  }
+  console.log('');
+}
+
+module.exports = {
+  loadHistory,
+  saveHistory,
+  buildHistoryEntry,
+  analyzeHistory,
+  printLearningInsights,
+  MAX_HISTORY_ENTRIES,
+};