thumbgate 1.29.2 → 1.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/.claude-plugin/plugin.json +1 -1
  2. package/.well-known/mcp/server-card.json +1 -1
  3. package/adapters/claude/.mcp.json +2 -2
  4. package/adapters/forge/forge.yaml +3 -3
  5. package/adapters/mcp/server-stdio.js +78 -7
  6. package/adapters/opencode/opencode.json +1 -1
  7. package/bin/cli.js +7 -5
  8. package/config/mcp-allowlists.json +26 -2
  9. package/config/post-deploy-marketing-pages.json +26 -1
  10. package/package.json +38 -7
  11. package/public/architecture.html +130 -0
  12. package/public/assets/diagrams/agent-integration.png +0 -0
  13. package/public/assets/diagrams/before-after.svg +21 -0
  14. package/public/assets/diagrams/decision.svg +36 -0
  15. package/public/assets/diagrams/feedback-pipeline.png +0 -0
  16. package/public/assets/diagrams/loop.svg +34 -0
  17. package/public/assets/diagrams/plugin-topology.png +0 -0
  18. package/public/assets/diagrams/pre-action-gate-loop.svg +59 -0
  19. package/public/assets/diagrams/stack.svg +18 -0
  20. package/public/assets/diagrams/thumbgate-architecture.png +0 -0
  21. package/public/case-studies.html +151 -0
  22. package/public/eval-scorecard.html +195 -0
  23. package/public/eval-scorecard.json +18 -0
  24. package/public/evaluations.html +168 -0
  25. package/public/index.html +4 -3
  26. package/public/numbers.html +2 -2
  27. package/public/whitepaper.html +189 -0
  28. package/scripts/activation-quickstart.js +1 -0
  29. package/scripts/agent-outcome-monitor.js +71 -1
  30. package/scripts/billing.js +3 -1
  31. package/scripts/claude-feedback-sync.js +3 -2
  32. package/scripts/cli-feedback.js +13 -7
  33. package/scripts/cross-encoder-reranker.js +3 -0
  34. package/scripts/feedback-aggregate.js +5 -2
  35. package/scripts/feedback-loop.js +244 -182
  36. package/scripts/gates-engine.js +81 -4
  37. package/scripts/generate-case-study-outreach.js +253 -0
  38. package/scripts/generate-eval-scorecard.js +276 -0
  39. package/scripts/growth-campaigns.js +183 -0
  40. package/scripts/jsonl-watcher.js +1 -0
  41. package/scripts/lesson-inference.js +23 -4
  42. package/scripts/lesson-retrieval.js +71 -4
  43. package/scripts/lesson-search.js +26 -3
  44. package/scripts/mcp-config.js +26 -5
  45. package/scripts/mcp-oauth.js +37 -2
  46. package/scripts/model-eval.js +308 -0
  47. package/scripts/parallel-workflow-orchestrator.js +86 -22
  48. package/scripts/published-cli.js +11 -1
  49. package/scripts/refresh-proof-pack.js +261 -0
  50. package/scripts/risk-scorer.js +144 -15
  51. package/scripts/statusline-local-stats.js +1 -1
  52. package/scripts/thumbgate-bench.js +13 -0
  53. package/scripts/tool-kpi-tracker.js +124 -0
  54. package/scripts/tool-registry.js +49 -1
  55. package/src/api/server.js +230 -86
@@ -62,6 +62,11 @@ const FEEDBACK_EVENT_CLAIM_STALE_MS = 60 * 1000;
62
62
  const FEEDBACK_EVENT_CLAIM_WAIT_MS = 15 * 1000;
63
63
  const FEEDBACK_EVENT_CLAIM_POLL_MS = 25;
64
64
 
65
+ function normalizeReviewOrigin(value) {
66
+ const origin = String(value || '').trim().toLowerCase();
67
+ return ['human', 'automated', 'imported'].includes(origin) ? origin : 'unverified';
68
+ }
69
+
65
70
  function isSelfHarnessOptimizerEnabled(env = process.env) {
66
71
  return /^(?:1|true)$/i.test(String(env.THUMBGATE_SELF_HARNESS_OPTIMIZER || '').trim());
67
72
  }
@@ -387,6 +392,8 @@ function normalizeAnalysisShape(analysis = {}) {
387
392
  byImportance: Array.isArray(analysis.byImportance) ? analysis.byImportance : [],
388
393
  recentLessons: Array.isArray(analysis.recentLessons) ? analysis.recentLessons : [],
389
394
  sessionCount: Number.isFinite(analysis.sessionCount) ? analysis.sessionCount : 0,
395
+ rawTotal: analysis.rawTotal ?? total,
396
+ excludedTotal: analysis.excludedTotal ?? 0,
390
397
  };
391
398
  }
392
399
 
@@ -1431,6 +1438,7 @@ function captureFeedback(params) {
1431
1438
  structuredRule: structuredRule || null,
1432
1439
  ...(reflection && { reflection }),
1433
1440
  gateAction: params.gateAction || null,
1441
+ reviewOrigin: normalizeReviewOrigin(params.reviewOrigin),
1434
1442
  sourceEvent: publicFeedbackSourceMetadata(params.sourceEvent),
1435
1443
  timestamp: now,
1436
1444
  };
@@ -1562,6 +1570,7 @@ function captureFeedback(params) {
1562
1570
  diagnosis: storedDiagnosis,
1563
1571
  structuredRule: structuredRule || null,
1564
1572
  sourceFeedbackId: feedbackEvent.id,
1573
+ reviewOrigin: feedbackEvent.reviewOrigin,
1565
1574
  timestamp: now,
1566
1575
  };
1567
1576
 
@@ -1885,6 +1894,9 @@ function captureFeedback(params) {
1885
1894
  inferredLesson: memoryRecord ? memoryRecord.title : (feedbackEvent.context || '').slice(0, 200),
1886
1895
  confidence: memoryRecord ? 70 : 40,
1887
1896
  tags: feedbackEvent.tags || [],
1897
+ metadata: {
1898
+ reviewOrigin: feedbackEvent.reviewOrigin,
1899
+ },
1888
1900
  });
1889
1901
  } catch { /* non-critical — lesson creation should never block feedback */ }
1890
1902
  });
@@ -1910,143 +1922,146 @@ function captureFeedbackIdempotent(params = {}) {
1910
1922
  }
1911
1923
  }
1912
1924
 
1913
- function analyzeFeedback(logPath) {
1914
- const { FEEDBACK_LOG_PATH } = getFeedbackPaths();
1915
- const resolvedLogPath = logPath || FEEDBACK_LOG_PATH;
1916
- const feedbackDir = path.dirname(resolvedLogPath);
1917
- const paths = buildFeedbackPathsFromDir(feedbackDir);
1918
- const shouldUseSQLite = !logPath || path.resolve(resolvedLogPath) === path.resolve(FEEDBACK_LOG_PATH);
1919
- const entries = readJSONL(resolvedLogPath, { maxLines: 0 });
1920
- const diagnosticLogPath = path.join(feedbackDir, 'diagnostic-log.jsonl');
1921
- const diagnosticEntries = readDiagnosticEntries(diagnosticLogPath);
1922
-
1923
- // Prefer the JSONL mirror for full analytics fidelity. Fall back to SQLite only
1924
- // when the mirror is unavailable so dashboards and proof paths keep their full shape.
1925
- const db = shouldUseSQLite ? getLessonDB() : null;
1926
- if (db && entries.length === 0) {
1927
- try {
1928
- const { getStatsFromDB } = require('./lesson-db');
1929
- const sqliteStats = getStatsFromDB(db);
1930
- if (sqliteStats.total > 0) return normalizeAnalysisShape(sqliteStats);
1931
- } catch { /* fall through to JSONL scan */ }
1925
+ function incrementBucket(buckets, key, signal) {
1926
+ if (!key) return;
1927
+ if (!buckets[key]) buckets[key] = { positive: 0, negative: 0, total: 0 };
1928
+ buckets[key][signal] += 1;
1929
+ buckets[key].total += 1;
1930
+ }
1931
+
1932
+ function summarizeRubric(entry, summary) {
1933
+ if (entry.actionType === 'no-action' && typeof entry.actionReason === 'string' && entry.actionReason.includes('Rubric gate')) {
1934
+ summary.blocked += 1;
1932
1935
  }
1936
+ if (entry.rubric?.weightedScore != null) summary.samples += 1;
1933
1937
 
1934
- const skills = {};
1935
- const tags = {};
1936
- const rubricCriteria = {};
1937
- let rubricSamples = 0;
1938
- let blockedPromotions = 0;
1938
+ for (const criterion of entry.rubric?.failingCriteria || []) {
1939
+ if (!summary.criteria[criterion]) summary.criteria[criterion] = { failures: 0 };
1940
+ summary.criteria[criterion].failures += 1;
1941
+ }
1942
+ }
1939
1943
 
1940
- let totalPositive = 0;
1941
- let totalNegative = 0;
1944
+ function summarizeFeedbackEntries(entries) {
1945
+ const summary = {
1946
+ skills: {},
1947
+ tags: {},
1948
+ criteria: {},
1949
+ samples: 0,
1950
+ blocked: 0,
1951
+ positive: 0,
1952
+ negative: 0,
1953
+ };
1942
1954
 
1943
1955
  for (const entry of entries) {
1944
- if (entry.signal === 'positive') totalPositive++;
1945
- if (entry.signal === 'negative') totalNegative++;
1946
-
1947
- if (entry.skill) {
1948
- if (!skills[entry.skill]) skills[entry.skill] = { positive: 0, negative: 0, total: 0 };
1949
- skills[entry.skill][entry.signal] += 1;
1950
- skills[entry.skill].total += 1;
1951
- }
1956
+ if (entry.signal === 'positive') summary.positive += 1;
1957
+ if (entry.signal === 'negative') summary.negative += 1;
1952
1958
 
1959
+ incrementBucket(summary.skills, entry.skill, entry.signal);
1953
1960
  for (const tag of entry.tags || []) {
1954
- if (!tags[tag]) tags[tag] = { positive: 0, negative: 0, total: 0 };
1955
- tags[tag][entry.signal] += 1;
1956
- tags[tag].total += 1;
1961
+ incrementBucket(summary.tags, tag, entry.signal);
1957
1962
  }
1963
+ summarizeRubric(entry, summary);
1964
+ }
1958
1965
 
1959
- if (entry.actionType === 'no-action' && typeof entry.actionReason === 'string' && entry.actionReason.includes('Rubric gate')) {
1960
- blockedPromotions += 1;
1961
- }
1966
+ return summary;
1967
+ }
1962
1968
 
1963
- if (entry.rubric && entry.rubric.weightedScore != null) {
1964
- rubricSamples += 1;
1965
- }
1969
+ function roundedRate(numerator, denominator) {
1970
+ return denominator > 0 ? Math.round((numerator / denominator) * 1000) / 1000 : 0;
1971
+ }
1966
1972
 
1967
- if (entry.rubric && Array.isArray(entry.rubric.failingCriteria)) {
1968
- for (const criterion of entry.rubric.failingCriteria) {
1969
- if (!rubricCriteria[criterion]) rubricCriteria[criterion] = { failures: 0 };
1970
- rubricCriteria[criterion].failures += 1;
1971
- }
1972
- }
1973
- }
1973
+ function addRec(output, message, remediation) {
1974
+ output.messages.push(message);
1975
+ output.remediations.push(remediation);
1976
+ }
1974
1977
 
1975
- const total = totalPositive + totalNegative;
1976
- const approvalRate = total > 0 ? Math.round((totalPositive / total) * 1000) / 1000 : 0;
1977
- const recent = entries.slice(-20);
1978
- const recentPos = recent.filter((e) => e.signal === 'positive').length;
1979
- const recentRate = recent.length > 0 ? Math.round((recentPos / recent.length) * 1000) / 1000 : 0;
1978
+ function feedbackTrend(windowStats, rate7d, rate30d) {
1979
+ if (windowStats['7d'].total === 0 || windowStats['30d'].total === 0) return 'stable';
1980
+ if (rate7d > rate30d + 0.05) return 'improving';
1981
+ if (rate7d < rate30d - 0.05) return 'degrading';
1982
+ return 'stable';
1983
+ }
1980
1984
 
1981
- // Rolling windows: 7-day, 30-day, lifetime (#204)
1985
+ function feedbackWindows(entries, positive, total) {
1982
1986
  const now = Date.now();
1983
- const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
1984
- const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
1985
- const windowStats = { '7d': { total: 0, positive: 0 }, '30d': { total: 0, positive: 0 } };
1987
+ const windowStats = {
1988
+ '7d': { total: 0, positive: 0 },
1989
+ '30d': { total: 0, positive: 0 },
1990
+ };
1991
+ const sevenDaysMs = 7 * 24 * 60 * 60 * 1000;
1992
+ const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000;
1993
+
1986
1994
  for (const entry of entries) {
1987
- const ts = entry.timestamp ? new Date(entry.timestamp).getTime() : 0;
1988
- const age = now - ts;
1989
- if (age <= SEVEN_DAYS_MS) {
1990
- windowStats['7d'].total++;
1991
- if (entry.signal === 'positive') windowStats['7d'].positive++;
1995
+ const timestamp = entry.timestamp ? new Date(entry.timestamp).getTime() : 0;
1996
+ const age = now - timestamp;
1997
+ if (age <= sevenDaysMs) {
1998
+ windowStats['7d'].total += 1;
1999
+ if (entry.signal === 'positive') windowStats['7d'].positive += 1;
1992
2000
  }
1993
- if (age <= THIRTY_DAYS_MS) {
1994
- windowStats['30d'].total++;
1995
- if (entry.signal === 'positive') windowStats['30d'].positive++;
2001
+ if (age <= thirtyDaysMs) {
2002
+ windowStats['30d'].total += 1;
2003
+ if (entry.signal === 'positive') windowStats['30d'].positive += 1;
1996
2004
  }
1997
2005
  }
1998
- const rate7d = windowStats['7d'].total > 0
1999
- ? Math.round((windowStats['7d'].positive / windowStats['7d'].total) * 1000) / 1000 : 0;
2000
- const rate30d = windowStats['30d'].total > 0
2001
- ? Math.round((windowStats['30d'].positive / windowStats['30d'].total) * 1000) / 1000 : 0;
2002
- const TREND_THRESHOLD = 0.05;
2003
- const hasTrendData = windowStats['7d'].total > 0 && windowStats['30d'].total > 0;
2004
- const trend = !hasTrendData ? 'stable'
2005
- : rate7d > rate30d + TREND_THRESHOLD ? 'improving'
2006
- : rate7d < rate30d - TREND_THRESHOLD ? 'degrading' : 'stable';
2007
- const windows = {
2008
- '7d': { ...windowStats['7d'], rate: rate7d },
2009
- '30d': { ...windowStats['30d'], rate: rate30d },
2010
- lifetime: { total, positive: totalPositive, rate: approvalRate },
2011
- };
2012
2006
 
2013
- const recommendations = [];
2014
- // Structured counterpart to `recommendations` — machine-actionable shape so
2015
- // hooks/agents can act on each item without regex-parsing prose strings.
2016
- // Each entry: { type, target, evidence, action, rationale }.
2017
- const actionableRemediations = [];
2007
+ const rate7d = roundedRate(windowStats['7d'].positive, windowStats['7d'].total);
2008
+ const rate30d = roundedRate(windowStats['30d'].positive, windowStats['30d'].total);
2009
+ const trend = feedbackTrend(windowStats, rate7d, rate30d);
2010
+
2011
+ return {
2012
+ rate7d,
2013
+ rate30d,
2014
+ trend,
2015
+ windows: {
2016
+ '7d': { ...windowStats['7d'], rate: rate7d },
2017
+ '30d': { ...windowStats['30d'], rate: rate30d },
2018
+ lifetime: { total, positive, rate: roundedRate(positive, total) },
2019
+ },
2020
+ };
2021
+ }
2018
2022
 
2023
+ function addSkillRecs(skills, output) {
2019
2024
  for (const [skill, stat] of Object.entries(skills)) {
2020
2025
  const negRate = stat.total > 0 ? stat.negative / stat.total : 0;
2021
- if (stat.total >= 3 && negRate >= 0.5) {
2022
- recommendations.push(`IMPROVE skill '${skill}' (${stat.negative}/${stat.total} negative)`);
2023
- actionableRemediations.push({
2024
- type: 'skill-improve',
2025
- target: skill,
2026
- evidence: { positive: stat.positive, negative: stat.negative, total: stat.total, negativeRate: Math.round(negRate * 1000) / 1000 },
2027
- action: 'review-and-update-skill',
2028
- rationale: `Skill '${skill}' has ${stat.negative}/${stat.total} negative feedback events (${Math.round(negRate * 100)}% negative rate).`,
2029
- });
2030
- }
2026
+ if (stat.total < 3 || negRate < 0.5) continue;
2027
+ addRec(output, `IMPROVE skill '${skill}' (${stat.negative}/${stat.total} negative)`, {
2028
+ type: 'skill-improve',
2029
+ target: skill,
2030
+ evidence: {
2031
+ positive: stat.positive,
2032
+ negative: stat.negative,
2033
+ total: stat.total,
2034
+ negativeRate: roundedRate(stat.negative, stat.total),
2035
+ },
2036
+ action: 'review-and-update-skill',
2037
+ rationale: `Skill '${skill}' has ${stat.negative}/${stat.total} negative feedback events (${Math.round(negRate * 100)}% negative rate).`,
2038
+ });
2031
2039
  }
2040
+ }
2032
2041
 
2042
+ function addTagRecs(tags, output) {
2033
2043
  for (const [tag, stat] of Object.entries(tags)) {
2034
2044
  const posRate = stat.total > 0 ? stat.positive / stat.total : 0;
2035
- if (stat.total >= 3 && posRate >= 0.8) {
2036
- recommendations.push(`REUSE pattern '${tag}' (${stat.positive}/${stat.total} positive)`);
2037
- actionableRemediations.push({
2038
- type: 'pattern-reuse',
2039
- target: tag,
2040
- evidence: { positive: stat.positive, negative: stat.negative, total: stat.total, positiveRate: Math.round(posRate * 1000) / 1000 },
2041
- action: 'replicate-pattern',
2042
- rationale: `Pattern '${tag}' has ${stat.positive}/${stat.total} positive feedback events (${Math.round(posRate * 100)}% positive rate).`,
2043
- });
2044
- }
2045
+ if (stat.total < 3 || posRate < 0.8) continue;
2046
+ addRec(output, `REUSE pattern '${tag}' (${stat.positive}/${stat.total} positive)`, {
2047
+ type: 'pattern-reuse',
2048
+ target: tag,
2049
+ evidence: {
2050
+ positive: stat.positive,
2051
+ negative: stat.negative,
2052
+ total: stat.total,
2053
+ positiveRate: roundedRate(stat.positive, stat.total),
2054
+ },
2055
+ action: 'replicate-pattern',
2056
+ rationale: `Pattern '${tag}' has ${stat.positive}/${stat.total} positive feedback events (${Math.round(posRate * 100)}% positive rate).`,
2057
+ });
2045
2058
  }
2059
+ }
2046
2060
 
2061
+ function addTrendRecs(metrics, output) {
2062
+ const { recent, recentRate, approvalRate, trend, rate7d, rate30d } = metrics;
2047
2063
  if (recent.length >= 10 && recentRate < approvalRate - 0.1) {
2048
- recommendations.push('DECLINING trend in last 20 signals; tighten verification before response.');
2049
- actionableRemediations.push({
2064
+ addRec(output, 'DECLINING trend in last 20 signals; tighten verification before response.', {
2050
2065
  type: 'trend-declining',
2051
2066
  target: 'recent-signals',
2052
2067
  evidence: { recentRate, approvalRate, sampleSize: recent.length },
@@ -2054,93 +2069,135 @@ function analyzeFeedback(logPath) {
2054
2069
  rationale: `Recent approval rate (${Math.round(recentRate * 100)}%) has dropped ≥10pp below lifetime (${Math.round(approvalRate * 100)}%).`,
2055
2070
  });
2056
2071
  }
2057
- if (trend === 'degrading') {
2058
- recommendations.push(`DEGRADING 7d trend (${rate7d}) vs 30d (${rate30d}); increase prevention rule injection.`);
2059
- actionableRemediations.push({
2060
- type: 'trend-degrading',
2061
- target: '7d-window',
2062
- evidence: { rate7d, rate30d, delta: Math.round((rate7d - rate30d) * 1000) / 1000 },
2063
- action: 'increase-prevention-rule-injection',
2064
- rationale: `7d rate (${rate7d}) is below 30d rate (${rate30d}) by more than threshold.`,
2072
+ if (trend !== 'degrading') return;
2073
+ addRec(output, `DEGRADING 7d trend (${rate7d}) vs 30d (${rate30d}); increase prevention rule injection.`, {
2074
+ type: 'trend-degrading',
2075
+ target: '7d-window',
2076
+ evidence: { rate7d, rate30d, delta: Math.round((rate7d - rate30d) * 1000) / 1000 },
2077
+ action: 'increase-prevention-rule-injection',
2078
+ rationale: `7d rate (${rate7d}) is below 30d rate (${rate30d}) by more than threshold.`,
2079
+ });
2080
+ }
2081
+
2082
+ function addRiskBuckets(buckets, kind, output) {
2083
+ for (const bucket of buckets.slice(0, 2)) {
2084
+ addRec(output, `CHECK high-risk ${kind} '${bucket.key}' (${bucket.highRisk}/${bucket.total} high-risk)`, {
2085
+ type: `high-risk-${kind}`,
2086
+ target: bucket.key,
2087
+ evidence: { highRisk: bucket.highRisk, total: bucket.total, riskRate: bucket.riskRate },
2088
+ action: `audit-${kind}-failures`,
2089
+ rationale: `${kind === 'domain' ? 'Domain' : 'Tag'} '${bucket.key}' has ${bucket.highRisk}/${bucket.total} high-risk events (${Math.round((bucket.riskRate || 0) * 100)}% risk rate).`,
2065
2090
  });
2066
2091
  }
2092
+ }
2067
2093
 
2068
- let boostedRisk = null;
2094
+ function addRiskRecs(feedbackDir, output) {
2069
2095
  try {
2070
2096
  const riskScorer = getRiskScorerModule();
2071
- if (riskScorer) {
2072
- boostedRisk = riskScorer.getRiskSummary(paths.FEEDBACK_DIR);
2073
- if (boostedRisk) {
2074
- boostedRisk.highRiskDomains.slice(0, 2).forEach((bucket) => {
2075
- recommendations.push(`CHECK high-risk domain '${bucket.key}' (${bucket.highRisk}/${bucket.total} high-risk)`);
2076
- actionableRemediations.push({
2077
- type: 'high-risk-domain',
2078
- target: bucket.key,
2079
- evidence: { highRisk: bucket.highRisk, total: bucket.total, riskRate: bucket.riskRate },
2080
- action: 'audit-domain-failures',
2081
- rationale: `Domain '${bucket.key}' has ${bucket.highRisk}/${bucket.total} high-risk events (${Math.round((bucket.riskRate || 0) * 100)}% risk rate).`,
2082
- });
2083
- });
2084
- boostedRisk.highRiskTags.slice(0, 2).forEach((bucket) => {
2085
- recommendations.push(`CHECK high-risk tag '${bucket.key}' (${bucket.highRisk}/${bucket.total} high-risk)`);
2086
- actionableRemediations.push({
2087
- type: 'high-risk-tag',
2088
- target: bucket.key,
2089
- evidence: { highRisk: bucket.highRisk, total: bucket.total, riskRate: bucket.riskRate },
2090
- action: 'audit-tag-failures',
2091
- rationale: `Tag '${bucket.key}' has ${bucket.highRisk}/${bucket.total} high-risk events (${Math.round((bucket.riskRate || 0) * 100)}% risk rate).`,
2092
- });
2093
- });
2094
- }
2095
- }
2097
+ if (!riskScorer) return null;
2098
+ const boostedRisk = riskScorer.getRiskSummary(feedbackDir);
2099
+ if (!boostedRisk) return null;
2100
+ addRiskBuckets(boostedRisk.highRiskDomains, 'domain', output);
2101
+ addRiskBuckets(boostedRisk.highRiskTags, 'tag', output);
2102
+ return boostedRisk;
2096
2103
  } catch {
2097
- boostedRisk = null;
2104
+ return null;
2098
2105
  }
2099
- const diagnostics = aggregateFailureDiagnostics([...entries, ...diagnosticEntries]);
2100
- let delegation = null;
2106
+ }
2107
+
2108
+ function addDelegationRecs(feedbackDir, output) {
2101
2109
  try {
2102
2110
  const delegationRuntime = getDelegationRuntimeModule();
2103
- if (delegationRuntime && typeof delegationRuntime.summarizeDelegation === 'function') {
2104
- delegation = delegationRuntime.summarizeDelegation(paths.FEEDBACK_DIR);
2105
- if (delegation.attemptCount >= 3 && delegation.verificationFailureRate >= 0.5) {
2106
- recommendations.push(`REDUCE delegation: verification failure rate is ${Math.round(delegation.verificationFailureRate * 100)}%`);
2107
- actionableRemediations.push({
2108
- type: 'delegation-reduce',
2109
- target: 'verification-failure-rate',
2110
- evidence: { verificationFailureRate: delegation.verificationFailureRate, attemptCount: delegation.attemptCount },
2111
- action: 'reduce-delegation-use',
2112
- rationale: `Delegation verification failure rate is ${Math.round(delegation.verificationFailureRate * 100)}% across ${delegation.attemptCount} attempts.`,
2113
- });
2114
- }
2115
- if (delegation.avoidedDelegationCount >= 3) {
2116
- recommendations.push(`REVIEW delegation policy: ${delegation.avoidedDelegationCount} handoff starts were blocked before execution`);
2117
- actionableRemediations.push({
2118
- type: 'delegation-policy-review',
2119
- target: 'handoff-blocks',
2120
- evidence: { avoidedDelegationCount: delegation.avoidedDelegationCount },
2121
- action: 'review-delegation-policy',
2122
- rationale: `${delegation.avoidedDelegationCount} handoff starts were blocked before execution.`,
2123
- });
2124
- }
2111
+ if (!delegationRuntime || typeof delegationRuntime.summarizeDelegation !== 'function') return null;
2112
+ const delegation = delegationRuntime.summarizeDelegation(feedbackDir);
2113
+ if (delegation.attemptCount >= 3 && delegation.verificationFailureRate >= 0.5) {
2114
+ addRec(output, `REDUCE delegation: verification failure rate is ${Math.round(delegation.verificationFailureRate * 100)}%`, {
2115
+ type: 'delegation-reduce',
2116
+ target: 'verification-failure-rate',
2117
+ evidence: {
2118
+ verificationFailureRate: delegation.verificationFailureRate,
2119
+ attemptCount: delegation.attemptCount,
2120
+ },
2121
+ action: 'reduce-delegation-use',
2122
+ rationale: `Delegation verification failure rate is ${Math.round(delegation.verificationFailureRate * 100)}% across ${delegation.attemptCount} attempts.`,
2123
+ });
2124
+ }
2125
+ if (delegation.avoidedDelegationCount >= 3) {
2126
+ addRec(output, `REVIEW delegation policy: ${delegation.avoidedDelegationCount} handoff starts were blocked before execution`, {
2127
+ type: 'delegation-policy-review',
2128
+ target: 'handoff-blocks',
2129
+ evidence: { avoidedDelegationCount: delegation.avoidedDelegationCount },
2130
+ action: 'review-delegation-policy',
2131
+ rationale: `${delegation.avoidedDelegationCount} handoff starts were blocked before execution.`,
2132
+ });
2125
2133
  }
2134
+ return delegation;
2126
2135
  } catch {
2127
- delegation = null;
2136
+ return null;
2128
2137
  }
2129
- diagnostics.categories.slice(0, 2).forEach((bucket) => {
2130
- recommendations.push(`DIAGNOSE '${bucket.key}' failures (${bucket.count})`);
2131
- actionableRemediations.push({
2138
+ }
2139
+
2140
+ function addDiagnosticRecs(diagnostics, output) {
2141
+ for (const bucket of diagnostics.categories.slice(0, 2)) {
2142
+ addRec(output, `DIAGNOSE '${bucket.key}' failures (${bucket.count})`, {
2132
2143
  type: 'diagnose-failure-category',
2133
2144
  target: bucket.key,
2134
2145
  evidence: { count: bucket.count },
2135
2146
  action: 'investigate-failure-category',
2136
2147
  rationale: `Failure category '${bucket.key}' has ${bucket.count} diagnosed events.`,
2137
2148
  });
2138
- });
2149
+ }
2150
+ }
2151
+
2152
+ function getSQLiteFallback(useSQLite, entries) {
2153
+ const db = useSQLite ? getLessonDB() : null;
2154
+ if (!db || entries.length > 0) return null;
2155
+ try {
2156
+ const { getStatsFromDB } = require('./lesson-db');
2157
+ const sqliteStats = getStatsFromDB(db);
2158
+ return sqliteStats.total > 0 ? normalizeAnalysisShape(sqliteStats) : null;
2159
+ } catch {
2160
+ return null;
2161
+ }
2162
+ }
2163
+
2164
+ function analyzeFeedback(logPath, options = {}) {
2165
+ const { FEEDBACK_LOG_PATH } = getFeedbackPaths();
2166
+ const resolvedPath = logPath || FEEDBACK_LOG_PATH;
2167
+ const feedbackDir = path.dirname(resolvedPath);
2168
+ const paths = buildFeedbackPathsFromDir(feedbackDir);
2169
+ const useSQLite = !options.humanOnly && (!logPath || path.resolve(resolvedPath) === path.resolve(FEEDBACK_LOG_PATH));
2170
+ let entries = readJSONL(resolvedPath, { maxLines: 0 });
2171
+ const rawTotal = entries.length;
2172
+ if (options.humanOnly) {
2173
+ entries = entries.filter((entry) => normalizeReviewOrigin(entry.reviewOrigin) === 'human');
2174
+ }
2175
+ const fallback = getSQLiteFallback(useSQLite, entries);
2176
+ if (fallback) return fallback;
2177
+
2178
+ const diagnosticEntries = readDiagnosticEntries(path.join(feedbackDir, 'diagnostic-log.jsonl'));
2179
+ const summary = summarizeFeedbackEntries(entries);
2180
+ const { skills, tags, criteria, samples, blocked, positive, negative } = summary;
2181
+ const total = positive + negative;
2182
+ const approvalRate = roundedRate(positive, total);
2183
+ const recent = entries.slice(-20);
2184
+ const recentPos = recent.filter((e) => e.signal === 'positive').length;
2185
+ const recentRate = roundedRate(recentPos, recent.length);
2186
+ const { rate7d, rate30d, trend, windows } = feedbackWindows(entries, positive, total);
2187
+ const recs = { messages: [], remediations: [] };
2188
+
2189
+ addSkillRecs(skills, recs);
2190
+ addTagRecs(tags, recs);
2191
+ addTrendRecs({ recent, recentRate, approvalRate, trend, rate7d, rate30d }, recs);
2192
+ const boostedRisk = addRiskRecs(paths.FEEDBACK_DIR, recs);
2193
+ const diagnostics = aggregateFailureDiagnostics([...entries, ...diagnosticEntries]);
2194
+ const delegation = addDelegationRecs(paths.FEEDBACK_DIR, recs);
2195
+ addDiagnosticRecs(diagnostics, recs);
2139
2196
 
2140
2197
  return normalizeAnalysisShape({
2141
2198
  total,
2142
- totalPositive,
2143
- totalNegative,
2199
+ totalPositive: positive,
2200
+ totalNegative: negative,
2144
2201
  approvalRate,
2145
2202
  recentRate,
2146
2203
  windows,
@@ -2148,15 +2205,17 @@ function analyzeFeedback(logPath) {
2148
2205
  skills,
2149
2206
  tags,
2150
2207
  rubric: {
2151
- samples: rubricSamples,
2152
- blockedPromotions,
2153
- failingCriteria: rubricCriteria,
2208
+ samples,
2209
+ blockedPromotions: blocked,
2210
+ failingCriteria: criteria,
2154
2211
  },
2155
2212
  diagnostics,
2156
2213
  delegation,
2157
2214
  boostedRisk,
2158
- recommendations,
2159
- actionableRemediations,
2215
+ recommendations: recs.messages,
2216
+ actionableRemediations: recs.remediations,
2217
+ rawTotal,
2218
+ excludedTotal: rawTotal - entries.length,
2160
2219
  });
2161
2220
  }
2162
2221
 
@@ -2313,7 +2372,10 @@ function writePreventionRules(filePath, minOccurrences = 2) {
2313
2372
 
2314
2373
  function feedbackSummary(recentN = 20, options = {}) {
2315
2374
  const { FEEDBACK_LOG_PATH } = getFeedbackPaths(options);
2316
- const entries = readJSONL(FEEDBACK_LOG_PATH);
2375
+ let entries = readJSONL(FEEDBACK_LOG_PATH);
2376
+ if (options.humanOnly) {
2377
+ entries = entries.filter((entry) => normalizeReviewOrigin(entry.reviewOrigin) === 'human');
2378
+ }
2317
2379
  if (entries.length === 0) {
2318
2380
  return '## Feedback Summary\nNo feedback recorded yet.';
2319
2381
  }
@@ -2323,7 +2385,7 @@ function feedbackSummary(recentN = 20, options = {}) {
2323
2385
  const negative = recent.filter((e) => e.signal === 'negative').length;
2324
2386
  const pct = Math.round((positive / recent.length) * 100);
2325
2387
 
2326
- const analysis = analyzeFeedback(FEEDBACK_LOG_PATH);
2388
+ const analysis = analyzeFeedback(FEEDBACK_LOG_PATH, { humanOnly: options.humanOnly });
2327
2389
 
2328
2390
  const lines = [
2329
2391
  `## Feedback Summary (last ${recent.length})`,
@@ -2396,7 +2458,7 @@ function runCli() {
2396
2458
  }
2397
2459
 
2398
2460
  if (args.summary) {
2399
- console.log(feedbackSummary(Number(args.recent || 20)));
2461
+ console.log(feedbackSummary(Number(args.recent || 20), { humanOnly: true }));
2400
2462
  return;
2401
2463
  }
2402
2464