thumbgate 1.29.2 → 1.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/mcp/server-card.json +1 -1
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/mcp/server-stdio.js +78 -7
- package/adapters/opencode/opencode.json +1 -1
- package/bin/cli.js +7 -5
- package/config/mcp-allowlists.json +26 -2
- package/config/post-deploy-marketing-pages.json +26 -1
- package/package.json +38 -7
- package/public/architecture.html +130 -0
- package/public/assets/diagrams/agent-integration.png +0 -0
- package/public/assets/diagrams/before-after.svg +21 -0
- package/public/assets/diagrams/decision.svg +36 -0
- package/public/assets/diagrams/feedback-pipeline.png +0 -0
- package/public/assets/diagrams/loop.svg +34 -0
- package/public/assets/diagrams/plugin-topology.png +0 -0
- package/public/assets/diagrams/pre-action-gate-loop.svg +59 -0
- package/public/assets/diagrams/stack.svg +18 -0
- package/public/assets/diagrams/thumbgate-architecture.png +0 -0
- package/public/case-studies.html +151 -0
- package/public/eval-scorecard.html +195 -0
- package/public/eval-scorecard.json +18 -0
- package/public/evaluations.html +168 -0
- package/public/index.html +4 -3
- package/public/numbers.html +2 -2
- package/public/whitepaper.html +189 -0
- package/scripts/activation-quickstart.js +1 -0
- package/scripts/agent-outcome-monitor.js +71 -1
- package/scripts/billing.js +3 -1
- package/scripts/claude-feedback-sync.js +3 -2
- package/scripts/cli-feedback.js +13 -7
- package/scripts/cross-encoder-reranker.js +3 -0
- package/scripts/feedback-aggregate.js +5 -2
- package/scripts/feedback-loop.js +244 -182
- package/scripts/gates-engine.js +81 -4
- package/scripts/generate-case-study-outreach.js +253 -0
- package/scripts/generate-eval-scorecard.js +276 -0
- package/scripts/growth-campaigns.js +183 -0
- package/scripts/jsonl-watcher.js +1 -0
- package/scripts/lesson-inference.js +23 -4
- package/scripts/lesson-retrieval.js +71 -4
- package/scripts/lesson-search.js +26 -3
- package/scripts/mcp-config.js +26 -5
- package/scripts/mcp-oauth.js +37 -2
- package/scripts/model-eval.js +308 -0
- package/scripts/parallel-workflow-orchestrator.js +86 -22
- package/scripts/published-cli.js +11 -1
- package/scripts/refresh-proof-pack.js +261 -0
- package/scripts/risk-scorer.js +144 -15
- package/scripts/statusline-local-stats.js +1 -1
- package/scripts/thumbgate-bench.js +13 -0
- package/scripts/tool-kpi-tracker.js +124 -0
- package/scripts/tool-registry.js +49 -1
- package/src/api/server.js +230 -86
package/scripts/feedback-loop.js
CHANGED
|
@@ -62,6 +62,11 @@ const FEEDBACK_EVENT_CLAIM_STALE_MS = 60 * 1000;
|
|
|
62
62
|
const FEEDBACK_EVENT_CLAIM_WAIT_MS = 15 * 1000;
|
|
63
63
|
const FEEDBACK_EVENT_CLAIM_POLL_MS = 25;
|
|
64
64
|
|
|
65
|
+
function normalizeReviewOrigin(value) {
|
|
66
|
+
const origin = String(value || '').trim().toLowerCase();
|
|
67
|
+
return ['human', 'automated', 'imported'].includes(origin) ? origin : 'unverified';
|
|
68
|
+
}
|
|
69
|
+
|
|
65
70
|
function isSelfHarnessOptimizerEnabled(env = process.env) {
|
|
66
71
|
return /^(?:1|true)$/i.test(String(env.THUMBGATE_SELF_HARNESS_OPTIMIZER || '').trim());
|
|
67
72
|
}
|
|
@@ -387,6 +392,8 @@ function normalizeAnalysisShape(analysis = {}) {
|
|
|
387
392
|
byImportance: Array.isArray(analysis.byImportance) ? analysis.byImportance : [],
|
|
388
393
|
recentLessons: Array.isArray(analysis.recentLessons) ? analysis.recentLessons : [],
|
|
389
394
|
sessionCount: Number.isFinite(analysis.sessionCount) ? analysis.sessionCount : 0,
|
|
395
|
+
rawTotal: analysis.rawTotal ?? total,
|
|
396
|
+
excludedTotal: analysis.excludedTotal ?? 0,
|
|
390
397
|
};
|
|
391
398
|
}
|
|
392
399
|
|
|
@@ -1431,6 +1438,7 @@ function captureFeedback(params) {
|
|
|
1431
1438
|
structuredRule: structuredRule || null,
|
|
1432
1439
|
...(reflection && { reflection }),
|
|
1433
1440
|
gateAction: params.gateAction || null,
|
|
1441
|
+
reviewOrigin: normalizeReviewOrigin(params.reviewOrigin),
|
|
1434
1442
|
sourceEvent: publicFeedbackSourceMetadata(params.sourceEvent),
|
|
1435
1443
|
timestamp: now,
|
|
1436
1444
|
};
|
|
@@ -1562,6 +1570,7 @@ function captureFeedback(params) {
|
|
|
1562
1570
|
diagnosis: storedDiagnosis,
|
|
1563
1571
|
structuredRule: structuredRule || null,
|
|
1564
1572
|
sourceFeedbackId: feedbackEvent.id,
|
|
1573
|
+
reviewOrigin: feedbackEvent.reviewOrigin,
|
|
1565
1574
|
timestamp: now,
|
|
1566
1575
|
};
|
|
1567
1576
|
|
|
@@ -1885,6 +1894,9 @@ function captureFeedback(params) {
|
|
|
1885
1894
|
inferredLesson: memoryRecord ? memoryRecord.title : (feedbackEvent.context || '').slice(0, 200),
|
|
1886
1895
|
confidence: memoryRecord ? 70 : 40,
|
|
1887
1896
|
tags: feedbackEvent.tags || [],
|
|
1897
|
+
metadata: {
|
|
1898
|
+
reviewOrigin: feedbackEvent.reviewOrigin,
|
|
1899
|
+
},
|
|
1888
1900
|
});
|
|
1889
1901
|
} catch { /* non-critical — lesson creation should never block feedback */ }
|
|
1890
1902
|
});
|
|
@@ -1910,143 +1922,146 @@ function captureFeedbackIdempotent(params = {}) {
|
|
|
1910
1922
|
}
|
|
1911
1923
|
}
|
|
1912
1924
|
|
|
1913
|
-
function
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
// Prefer the JSONL mirror for full analytics fidelity. Fall back to SQLite only
|
|
1924
|
-
// when the mirror is unavailable so dashboards and proof paths keep their full shape.
|
|
1925
|
-
const db = shouldUseSQLite ? getLessonDB() : null;
|
|
1926
|
-
if (db && entries.length === 0) {
|
|
1927
|
-
try {
|
|
1928
|
-
const { getStatsFromDB } = require('./lesson-db');
|
|
1929
|
-
const sqliteStats = getStatsFromDB(db);
|
|
1930
|
-
if (sqliteStats.total > 0) return normalizeAnalysisShape(sqliteStats);
|
|
1931
|
-
} catch { /* fall through to JSONL scan */ }
|
|
1925
|
+
function incrementBucket(buckets, key, signal) {
|
|
1926
|
+
if (!key) return;
|
|
1927
|
+
if (!buckets[key]) buckets[key] = { positive: 0, negative: 0, total: 0 };
|
|
1928
|
+
buckets[key][signal] += 1;
|
|
1929
|
+
buckets[key].total += 1;
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
function summarizeRubric(entry, summary) {
|
|
1933
|
+
if (entry.actionType === 'no-action' && typeof entry.actionReason === 'string' && entry.actionReason.includes('Rubric gate')) {
|
|
1934
|
+
summary.blocked += 1;
|
|
1932
1935
|
}
|
|
1936
|
+
if (entry.rubric?.weightedScore != null) summary.samples += 1;
|
|
1933
1937
|
|
|
1934
|
-
const
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1938
|
+
for (const criterion of entry.rubric?.failingCriteria || []) {
|
|
1939
|
+
if (!summary.criteria[criterion]) summary.criteria[criterion] = { failures: 0 };
|
|
1940
|
+
summary.criteria[criterion].failures += 1;
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1939
1943
|
|
|
1940
|
-
|
|
1941
|
-
|
|
1944
|
+
function summarizeFeedbackEntries(entries) {
|
|
1945
|
+
const summary = {
|
|
1946
|
+
skills: {},
|
|
1947
|
+
tags: {},
|
|
1948
|
+
criteria: {},
|
|
1949
|
+
samples: 0,
|
|
1950
|
+
blocked: 0,
|
|
1951
|
+
positive: 0,
|
|
1952
|
+
negative: 0,
|
|
1953
|
+
};
|
|
1942
1954
|
|
|
1943
1955
|
for (const entry of entries) {
|
|
1944
|
-
if (entry.signal === 'positive')
|
|
1945
|
-
if (entry.signal === 'negative')
|
|
1946
|
-
|
|
1947
|
-
if (entry.skill) {
|
|
1948
|
-
if (!skills[entry.skill]) skills[entry.skill] = { positive: 0, negative: 0, total: 0 };
|
|
1949
|
-
skills[entry.skill][entry.signal] += 1;
|
|
1950
|
-
skills[entry.skill].total += 1;
|
|
1951
|
-
}
|
|
1956
|
+
if (entry.signal === 'positive') summary.positive += 1;
|
|
1957
|
+
if (entry.signal === 'negative') summary.negative += 1;
|
|
1952
1958
|
|
|
1959
|
+
incrementBucket(summary.skills, entry.skill, entry.signal);
|
|
1953
1960
|
for (const tag of entry.tags || []) {
|
|
1954
|
-
|
|
1955
|
-
tags[tag][entry.signal] += 1;
|
|
1956
|
-
tags[tag].total += 1;
|
|
1961
|
+
incrementBucket(summary.tags, tag, entry.signal);
|
|
1957
1962
|
}
|
|
1963
|
+
summarizeRubric(entry, summary);
|
|
1964
|
+
}
|
|
1958
1965
|
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
}
|
|
1966
|
+
return summary;
|
|
1967
|
+
}
|
|
1962
1968
|
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1969
|
+
function roundedRate(numerator, denominator) {
|
|
1970
|
+
return denominator > 0 ? Math.round((numerator / denominator) * 1000) / 1000 : 0;
|
|
1971
|
+
}
|
|
1966
1972
|
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
}
|
|
1972
|
-
}
|
|
1973
|
-
}
|
|
1973
|
+
function addRec(output, message, remediation) {
|
|
1974
|
+
output.messages.push(message);
|
|
1975
|
+
output.remediations.push(remediation);
|
|
1976
|
+
}
|
|
1974
1977
|
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1978
|
+
function feedbackTrend(windowStats, rate7d, rate30d) {
|
|
1979
|
+
if (windowStats['7d'].total === 0 || windowStats['30d'].total === 0) return 'stable';
|
|
1980
|
+
if (rate7d > rate30d + 0.05) return 'improving';
|
|
1981
|
+
if (rate7d < rate30d - 0.05) return 'degrading';
|
|
1982
|
+
return 'stable';
|
|
1983
|
+
}
|
|
1980
1984
|
|
|
1981
|
-
|
|
1985
|
+
function feedbackWindows(entries, positive, total) {
|
|
1982
1986
|
const now = Date.now();
|
|
1983
|
-
const
|
|
1984
|
-
|
|
1985
|
-
|
|
1987
|
+
const windowStats = {
|
|
1988
|
+
'7d': { total: 0, positive: 0 },
|
|
1989
|
+
'30d': { total: 0, positive: 0 },
|
|
1990
|
+
};
|
|
1991
|
+
const sevenDaysMs = 7 * 24 * 60 * 60 * 1000;
|
|
1992
|
+
const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000;
|
|
1993
|
+
|
|
1986
1994
|
for (const entry of entries) {
|
|
1987
|
-
const
|
|
1988
|
-
const age = now -
|
|
1989
|
-
if (age <=
|
|
1990
|
-
windowStats['7d'].total
|
|
1991
|
-
if (entry.signal === 'positive') windowStats['7d'].positive
|
|
1995
|
+
const timestamp = entry.timestamp ? new Date(entry.timestamp).getTime() : 0;
|
|
1996
|
+
const age = now - timestamp;
|
|
1997
|
+
if (age <= sevenDaysMs) {
|
|
1998
|
+
windowStats['7d'].total += 1;
|
|
1999
|
+
if (entry.signal === 'positive') windowStats['7d'].positive += 1;
|
|
1992
2000
|
}
|
|
1993
|
-
if (age <=
|
|
1994
|
-
windowStats['30d'].total
|
|
1995
|
-
if (entry.signal === 'positive') windowStats['30d'].positive
|
|
2001
|
+
if (age <= thirtyDaysMs) {
|
|
2002
|
+
windowStats['30d'].total += 1;
|
|
2003
|
+
if (entry.signal === 'positive') windowStats['30d'].positive += 1;
|
|
1996
2004
|
}
|
|
1997
2005
|
}
|
|
1998
|
-
const rate7d = windowStats['7d'].total > 0
|
|
1999
|
-
? Math.round((windowStats['7d'].positive / windowStats['7d'].total) * 1000) / 1000 : 0;
|
|
2000
|
-
const rate30d = windowStats['30d'].total > 0
|
|
2001
|
-
? Math.round((windowStats['30d'].positive / windowStats['30d'].total) * 1000) / 1000 : 0;
|
|
2002
|
-
const TREND_THRESHOLD = 0.05;
|
|
2003
|
-
const hasTrendData = windowStats['7d'].total > 0 && windowStats['30d'].total > 0;
|
|
2004
|
-
const trend = !hasTrendData ? 'stable'
|
|
2005
|
-
: rate7d > rate30d + TREND_THRESHOLD ? 'improving'
|
|
2006
|
-
: rate7d < rate30d - TREND_THRESHOLD ? 'degrading' : 'stable';
|
|
2007
|
-
const windows = {
|
|
2008
|
-
'7d': { ...windowStats['7d'], rate: rate7d },
|
|
2009
|
-
'30d': { ...windowStats['30d'], rate: rate30d },
|
|
2010
|
-
lifetime: { total, positive: totalPositive, rate: approvalRate },
|
|
2011
|
-
};
|
|
2012
2006
|
|
|
2013
|
-
const
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2007
|
+
const rate7d = roundedRate(windowStats['7d'].positive, windowStats['7d'].total);
|
|
2008
|
+
const rate30d = roundedRate(windowStats['30d'].positive, windowStats['30d'].total);
|
|
2009
|
+
const trend = feedbackTrend(windowStats, rate7d, rate30d);
|
|
2010
|
+
|
|
2011
|
+
return {
|
|
2012
|
+
rate7d,
|
|
2013
|
+
rate30d,
|
|
2014
|
+
trend,
|
|
2015
|
+
windows: {
|
|
2016
|
+
'7d': { ...windowStats['7d'], rate: rate7d },
|
|
2017
|
+
'30d': { ...windowStats['30d'], rate: rate30d },
|
|
2018
|
+
lifetime: { total, positive, rate: roundedRate(positive, total) },
|
|
2019
|
+
},
|
|
2020
|
+
};
|
|
2021
|
+
}
|
|
2018
2022
|
|
|
2023
|
+
function addSkillRecs(skills, output) {
|
|
2019
2024
|
for (const [skill, stat] of Object.entries(skills)) {
|
|
2020
2025
|
const negRate = stat.total > 0 ? stat.negative / stat.total : 0;
|
|
2021
|
-
if (stat.total
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2026
|
+
if (stat.total < 3 || negRate < 0.5) continue;
|
|
2027
|
+
addRec(output, `IMPROVE skill '${skill}' (${stat.negative}/${stat.total} negative)`, {
|
|
2028
|
+
type: 'skill-improve',
|
|
2029
|
+
target: skill,
|
|
2030
|
+
evidence: {
|
|
2031
|
+
positive: stat.positive,
|
|
2032
|
+
negative: stat.negative,
|
|
2033
|
+
total: stat.total,
|
|
2034
|
+
negativeRate: roundedRate(stat.negative, stat.total),
|
|
2035
|
+
},
|
|
2036
|
+
action: 'review-and-update-skill',
|
|
2037
|
+
rationale: `Skill '${skill}' has ${stat.negative}/${stat.total} negative feedback events (${Math.round(negRate * 100)}% negative rate).`,
|
|
2038
|
+
});
|
|
2031
2039
|
}
|
|
2040
|
+
}
|
|
2032
2041
|
|
|
2042
|
+
function addTagRecs(tags, output) {
|
|
2033
2043
|
for (const [tag, stat] of Object.entries(tags)) {
|
|
2034
2044
|
const posRate = stat.total > 0 ? stat.positive / stat.total : 0;
|
|
2035
|
-
if (stat.total
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
+
if (stat.total < 3 || posRate < 0.8) continue;
|
|
2046
|
+
addRec(output, `REUSE pattern '${tag}' (${stat.positive}/${stat.total} positive)`, {
|
|
2047
|
+
type: 'pattern-reuse',
|
|
2048
|
+
target: tag,
|
|
2049
|
+
evidence: {
|
|
2050
|
+
positive: stat.positive,
|
|
2051
|
+
negative: stat.negative,
|
|
2052
|
+
total: stat.total,
|
|
2053
|
+
positiveRate: roundedRate(stat.positive, stat.total),
|
|
2054
|
+
},
|
|
2055
|
+
action: 'replicate-pattern',
|
|
2056
|
+
rationale: `Pattern '${tag}' has ${stat.positive}/${stat.total} positive feedback events (${Math.round(posRate * 100)}% positive rate).`,
|
|
2057
|
+
});
|
|
2045
2058
|
}
|
|
2059
|
+
}
|
|
2046
2060
|
|
|
2061
|
+
function addTrendRecs(metrics, output) {
|
|
2062
|
+
const { recent, recentRate, approvalRate, trend, rate7d, rate30d } = metrics;
|
|
2047
2063
|
if (recent.length >= 10 && recentRate < approvalRate - 0.1) {
|
|
2048
|
-
|
|
2049
|
-
actionableRemediations.push({
|
|
2064
|
+
addRec(output, 'DECLINING trend in last 20 signals; tighten verification before response.', {
|
|
2050
2065
|
type: 'trend-declining',
|
|
2051
2066
|
target: 'recent-signals',
|
|
2052
2067
|
evidence: { recentRate, approvalRate, sampleSize: recent.length },
|
|
@@ -2054,93 +2069,135 @@ function analyzeFeedback(logPath) {
|
|
|
2054
2069
|
rationale: `Recent approval rate (${Math.round(recentRate * 100)}%) has dropped ≥10pp below lifetime (${Math.round(approvalRate * 100)}%).`,
|
|
2055
2070
|
});
|
|
2056
2071
|
}
|
|
2057
|
-
if (trend
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2072
|
+
if (trend !== 'degrading') return;
|
|
2073
|
+
addRec(output, `DEGRADING 7d trend (${rate7d}) vs 30d (${rate30d}); increase prevention rule injection.`, {
|
|
2074
|
+
type: 'trend-degrading',
|
|
2075
|
+
target: '7d-window',
|
|
2076
|
+
evidence: { rate7d, rate30d, delta: Math.round((rate7d - rate30d) * 1000) / 1000 },
|
|
2077
|
+
action: 'increase-prevention-rule-injection',
|
|
2078
|
+
rationale: `7d rate (${rate7d}) is below 30d rate (${rate30d}) by more than threshold.`,
|
|
2079
|
+
});
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
function addRiskBuckets(buckets, kind, output) {
|
|
2083
|
+
for (const bucket of buckets.slice(0, 2)) {
|
|
2084
|
+
addRec(output, `CHECK high-risk ${kind} '${bucket.key}' (${bucket.highRisk}/${bucket.total} high-risk)`, {
|
|
2085
|
+
type: `high-risk-${kind}`,
|
|
2086
|
+
target: bucket.key,
|
|
2087
|
+
evidence: { highRisk: bucket.highRisk, total: bucket.total, riskRate: bucket.riskRate },
|
|
2088
|
+
action: `audit-${kind}-failures`,
|
|
2089
|
+
rationale: `${kind === 'domain' ? 'Domain' : 'Tag'} '${bucket.key}' has ${bucket.highRisk}/${bucket.total} high-risk events (${Math.round((bucket.riskRate || 0) * 100)}% risk rate).`,
|
|
2065
2090
|
});
|
|
2066
2091
|
}
|
|
2092
|
+
}
|
|
2067
2093
|
|
|
2068
|
-
|
|
2094
|
+
function addRiskRecs(feedbackDir, output) {
|
|
2069
2095
|
try {
|
|
2070
2096
|
const riskScorer = getRiskScorerModule();
|
|
2071
|
-
if (riskScorer)
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
type: 'high-risk-domain',
|
|
2078
|
-
target: bucket.key,
|
|
2079
|
-
evidence: { highRisk: bucket.highRisk, total: bucket.total, riskRate: bucket.riskRate },
|
|
2080
|
-
action: 'audit-domain-failures',
|
|
2081
|
-
rationale: `Domain '${bucket.key}' has ${bucket.highRisk}/${bucket.total} high-risk events (${Math.round((bucket.riskRate || 0) * 100)}% risk rate).`,
|
|
2082
|
-
});
|
|
2083
|
-
});
|
|
2084
|
-
boostedRisk.highRiskTags.slice(0, 2).forEach((bucket) => {
|
|
2085
|
-
recommendations.push(`CHECK high-risk tag '${bucket.key}' (${bucket.highRisk}/${bucket.total} high-risk)`);
|
|
2086
|
-
actionableRemediations.push({
|
|
2087
|
-
type: 'high-risk-tag',
|
|
2088
|
-
target: bucket.key,
|
|
2089
|
-
evidence: { highRisk: bucket.highRisk, total: bucket.total, riskRate: bucket.riskRate },
|
|
2090
|
-
action: 'audit-tag-failures',
|
|
2091
|
-
rationale: `Tag '${bucket.key}' has ${bucket.highRisk}/${bucket.total} high-risk events (${Math.round((bucket.riskRate || 0) * 100)}% risk rate).`,
|
|
2092
|
-
});
|
|
2093
|
-
});
|
|
2094
|
-
}
|
|
2095
|
-
}
|
|
2097
|
+
if (!riskScorer) return null;
|
|
2098
|
+
const boostedRisk = riskScorer.getRiskSummary(feedbackDir);
|
|
2099
|
+
if (!boostedRisk) return null;
|
|
2100
|
+
addRiskBuckets(boostedRisk.highRiskDomains, 'domain', output);
|
|
2101
|
+
addRiskBuckets(boostedRisk.highRiskTags, 'tag', output);
|
|
2102
|
+
return boostedRisk;
|
|
2096
2103
|
} catch {
|
|
2097
|
-
|
|
2104
|
+
return null;
|
|
2098
2105
|
}
|
|
2099
|
-
|
|
2100
|
-
|
|
2106
|
+
}
|
|
2107
|
+
|
|
2108
|
+
function addDelegationRecs(feedbackDir, output) {
|
|
2101
2109
|
try {
|
|
2102
2110
|
const delegationRuntime = getDelegationRuntimeModule();
|
|
2103
|
-
if (delegationRuntime
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
}
|
|
2124
|
-
}
|
|
2111
|
+
if (!delegationRuntime || typeof delegationRuntime.summarizeDelegation !== 'function') return null;
|
|
2112
|
+
const delegation = delegationRuntime.summarizeDelegation(feedbackDir);
|
|
2113
|
+
if (delegation.attemptCount >= 3 && delegation.verificationFailureRate >= 0.5) {
|
|
2114
|
+
addRec(output, `REDUCE delegation: verification failure rate is ${Math.round(delegation.verificationFailureRate * 100)}%`, {
|
|
2115
|
+
type: 'delegation-reduce',
|
|
2116
|
+
target: 'verification-failure-rate',
|
|
2117
|
+
evidence: {
|
|
2118
|
+
verificationFailureRate: delegation.verificationFailureRate,
|
|
2119
|
+
attemptCount: delegation.attemptCount,
|
|
2120
|
+
},
|
|
2121
|
+
action: 'reduce-delegation-use',
|
|
2122
|
+
rationale: `Delegation verification failure rate is ${Math.round(delegation.verificationFailureRate * 100)}% across ${delegation.attemptCount} attempts.`,
|
|
2123
|
+
});
|
|
2124
|
+
}
|
|
2125
|
+
if (delegation.avoidedDelegationCount >= 3) {
|
|
2126
|
+
addRec(output, `REVIEW delegation policy: ${delegation.avoidedDelegationCount} handoff starts were blocked before execution`, {
|
|
2127
|
+
type: 'delegation-policy-review',
|
|
2128
|
+
target: 'handoff-blocks',
|
|
2129
|
+
evidence: { avoidedDelegationCount: delegation.avoidedDelegationCount },
|
|
2130
|
+
action: 'review-delegation-policy',
|
|
2131
|
+
rationale: `${delegation.avoidedDelegationCount} handoff starts were blocked before execution.`,
|
|
2132
|
+
});
|
|
2125
2133
|
}
|
|
2134
|
+
return delegation;
|
|
2126
2135
|
} catch {
|
|
2127
|
-
|
|
2136
|
+
return null;
|
|
2128
2137
|
}
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
function addDiagnosticRecs(diagnostics, output) {
|
|
2141
|
+
for (const bucket of diagnostics.categories.slice(0, 2)) {
|
|
2142
|
+
addRec(output, `DIAGNOSE '${bucket.key}' failures (${bucket.count})`, {
|
|
2132
2143
|
type: 'diagnose-failure-category',
|
|
2133
2144
|
target: bucket.key,
|
|
2134
2145
|
evidence: { count: bucket.count },
|
|
2135
2146
|
action: 'investigate-failure-category',
|
|
2136
2147
|
rationale: `Failure category '${bucket.key}' has ${bucket.count} diagnosed events.`,
|
|
2137
2148
|
});
|
|
2138
|
-
}
|
|
2149
|
+
}
|
|
2150
|
+
}
|
|
2151
|
+
|
|
2152
|
+
function getSQLiteFallback(useSQLite, entries) {
|
|
2153
|
+
const db = useSQLite ? getLessonDB() : null;
|
|
2154
|
+
if (!db || entries.length > 0) return null;
|
|
2155
|
+
try {
|
|
2156
|
+
const { getStatsFromDB } = require('./lesson-db');
|
|
2157
|
+
const sqliteStats = getStatsFromDB(db);
|
|
2158
|
+
return sqliteStats.total > 0 ? normalizeAnalysisShape(sqliteStats) : null;
|
|
2159
|
+
} catch {
|
|
2160
|
+
return null;
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
|
|
2164
|
+
function analyzeFeedback(logPath, options = {}) {
|
|
2165
|
+
const { FEEDBACK_LOG_PATH } = getFeedbackPaths();
|
|
2166
|
+
const resolvedPath = logPath || FEEDBACK_LOG_PATH;
|
|
2167
|
+
const feedbackDir = path.dirname(resolvedPath);
|
|
2168
|
+
const paths = buildFeedbackPathsFromDir(feedbackDir);
|
|
2169
|
+
const useSQLite = !options.humanOnly && (!logPath || path.resolve(resolvedPath) === path.resolve(FEEDBACK_LOG_PATH));
|
|
2170
|
+
let entries = readJSONL(resolvedPath, { maxLines: 0 });
|
|
2171
|
+
const rawTotal = entries.length;
|
|
2172
|
+
if (options.humanOnly) {
|
|
2173
|
+
entries = entries.filter((entry) => normalizeReviewOrigin(entry.reviewOrigin) === 'human');
|
|
2174
|
+
}
|
|
2175
|
+
const fallback = getSQLiteFallback(useSQLite, entries);
|
|
2176
|
+
if (fallback) return fallback;
|
|
2177
|
+
|
|
2178
|
+
const diagnosticEntries = readDiagnosticEntries(path.join(feedbackDir, 'diagnostic-log.jsonl'));
|
|
2179
|
+
const summary = summarizeFeedbackEntries(entries);
|
|
2180
|
+
const { skills, tags, criteria, samples, blocked, positive, negative } = summary;
|
|
2181
|
+
const total = positive + negative;
|
|
2182
|
+
const approvalRate = roundedRate(positive, total);
|
|
2183
|
+
const recent = entries.slice(-20);
|
|
2184
|
+
const recentPos = recent.filter((e) => e.signal === 'positive').length;
|
|
2185
|
+
const recentRate = roundedRate(recentPos, recent.length);
|
|
2186
|
+
const { rate7d, rate30d, trend, windows } = feedbackWindows(entries, positive, total);
|
|
2187
|
+
const recs = { messages: [], remediations: [] };
|
|
2188
|
+
|
|
2189
|
+
addSkillRecs(skills, recs);
|
|
2190
|
+
addTagRecs(tags, recs);
|
|
2191
|
+
addTrendRecs({ recent, recentRate, approvalRate, trend, rate7d, rate30d }, recs);
|
|
2192
|
+
const boostedRisk = addRiskRecs(paths.FEEDBACK_DIR, recs);
|
|
2193
|
+
const diagnostics = aggregateFailureDiagnostics([...entries, ...diagnosticEntries]);
|
|
2194
|
+
const delegation = addDelegationRecs(paths.FEEDBACK_DIR, recs);
|
|
2195
|
+
addDiagnosticRecs(diagnostics, recs);
|
|
2139
2196
|
|
|
2140
2197
|
return normalizeAnalysisShape({
|
|
2141
2198
|
total,
|
|
2142
|
-
totalPositive,
|
|
2143
|
-
totalNegative,
|
|
2199
|
+
totalPositive: positive,
|
|
2200
|
+
totalNegative: negative,
|
|
2144
2201
|
approvalRate,
|
|
2145
2202
|
recentRate,
|
|
2146
2203
|
windows,
|
|
@@ -2148,15 +2205,17 @@ function analyzeFeedback(logPath) {
|
|
|
2148
2205
|
skills,
|
|
2149
2206
|
tags,
|
|
2150
2207
|
rubric: {
|
|
2151
|
-
samples
|
|
2152
|
-
blockedPromotions,
|
|
2153
|
-
failingCriteria:
|
|
2208
|
+
samples,
|
|
2209
|
+
blockedPromotions: blocked,
|
|
2210
|
+
failingCriteria: criteria,
|
|
2154
2211
|
},
|
|
2155
2212
|
diagnostics,
|
|
2156
2213
|
delegation,
|
|
2157
2214
|
boostedRisk,
|
|
2158
|
-
recommendations,
|
|
2159
|
-
actionableRemediations,
|
|
2215
|
+
recommendations: recs.messages,
|
|
2216
|
+
actionableRemediations: recs.remediations,
|
|
2217
|
+
rawTotal,
|
|
2218
|
+
excludedTotal: rawTotal - entries.length,
|
|
2160
2219
|
});
|
|
2161
2220
|
}
|
|
2162
2221
|
|
|
@@ -2313,7 +2372,10 @@ function writePreventionRules(filePath, minOccurrences = 2) {
|
|
|
2313
2372
|
|
|
2314
2373
|
function feedbackSummary(recentN = 20, options = {}) {
|
|
2315
2374
|
const { FEEDBACK_LOG_PATH } = getFeedbackPaths(options);
|
|
2316
|
-
|
|
2375
|
+
let entries = readJSONL(FEEDBACK_LOG_PATH);
|
|
2376
|
+
if (options.humanOnly) {
|
|
2377
|
+
entries = entries.filter((entry) => normalizeReviewOrigin(entry.reviewOrigin) === 'human');
|
|
2378
|
+
}
|
|
2317
2379
|
if (entries.length === 0) {
|
|
2318
2380
|
return '## Feedback Summary\nNo feedback recorded yet.';
|
|
2319
2381
|
}
|
|
@@ -2323,7 +2385,7 @@ function feedbackSummary(recentN = 20, options = {}) {
|
|
|
2323
2385
|
const negative = recent.filter((e) => e.signal === 'negative').length;
|
|
2324
2386
|
const pct = Math.round((positive / recent.length) * 100);
|
|
2325
2387
|
|
|
2326
|
-
const analysis = analyzeFeedback(FEEDBACK_LOG_PATH);
|
|
2388
|
+
const analysis = analyzeFeedback(FEEDBACK_LOG_PATH, { humanOnly: options.humanOnly });
|
|
2327
2389
|
|
|
2328
2390
|
const lines = [
|
|
2329
2391
|
`## Feedback Summary (last ${recent.length})`,
|
|
@@ -2396,7 +2458,7 @@ function runCli() {
|
|
|
2396
2458
|
}
|
|
2397
2459
|
|
|
2398
2460
|
if (args.summary) {
|
|
2399
|
-
console.log(feedbackSummary(Number(args.recent || 20)));
|
|
2461
|
+
console.log(feedbackSummary(Number(args.recent || 20), { humanOnly: true }));
|
|
2400
2462
|
return;
|
|
2401
2463
|
}
|
|
2402
2464
|
|