thumbgate 0.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/README.md +134 -0
- package/.claude-plugin/bundle/icon.png +0 -0
- package/.claude-plugin/bundle/icon.svg +18 -0
- package/.claude-plugin/bundle/server/index.js +24 -0
- package/.claude-plugin/marketplace.json +36 -0
- package/.claude-plugin/plugin.json +21 -0
- package/.well-known/mcp/server-card.json +231 -0
- package/LICENSE +21 -0
- package/README.md +375 -0
- package/adapters/README.md +9 -0
- package/adapters/amp/skills/rlhf-feedback/SKILL.md +22 -0
- package/adapters/chatgpt/INSTALL.md +83 -0
- package/adapters/chatgpt/openapi.yaml +1281 -0
- package/adapters/claude/.mcp.json +14 -0
- package/adapters/codex/config.toml +9 -0
- package/adapters/gemini/function-declarations.json +224 -0
- package/adapters/mcp/server-stdio.js +788 -0
- package/adapters/opencode/opencode.json +15 -0
- package/bin/cli.js +1483 -0
- package/bin/memory.sh +64 -0
- package/bin/obsidian-sync.sh +20 -0
- package/bin/postinstall.js +37 -0
- package/config/build-metadata.json +4 -0
- package/config/e2e-critical-flows.json +45 -0
- package/config/gate-templates.json +77 -0
- package/config/gates/claim-verification.json +29 -0
- package/config/gates/computer-use.json +39 -0
- package/config/gates/default.json +117 -0
- package/config/github-about.json +25 -0
- package/config/mcp-allowlists.json +135 -0
- package/config/model-tiers.json +33 -0
- package/config/partner-routing.json +132 -0
- package/config/policy-bundles/constrained-v1.json +64 -0
- package/config/policy-bundles/default-v1.json +91 -0
- package/config/rubrics/default-v1.json +52 -0
- package/config/skill-packs/react-testing.json +23 -0
- package/config/skill-packs/stripe-integration/references/api-spec.json +1 -0
- package/config/skill-packs/stripe-integration/references/webhook-guide.md +3 -0
- package/config/skill-specs/pr-reviewer.json +9 -0
- package/config/skill-specs/release-status.json +9 -0
- package/config/skill-specs/ticket-triage.json +9 -0
- package/config/subagent-profiles.json +32 -0
- package/config/tessl-tiles.json +29 -0
- package/config/thumbgate-settings.managed.json +12 -0
- package/openapi/openapi.yaml +1281 -0
- package/package.json +286 -0
- package/plugins/amp-skill/INSTALL.md +52 -0
- package/plugins/amp-skill/SKILL.md +64 -0
- package/plugins/claude-codex-bridge/.claude-plugin/plugin.json +22 -0
- package/plugins/claude-codex-bridge/.mcp.json +12 -0
- package/plugins/claude-codex-bridge/INSTALL.md +43 -0
- package/plugins/claude-codex-bridge/README.md +46 -0
- package/plugins/claude-codex-bridge/scripts/codex-bridge.js +288 -0
- package/plugins/claude-codex-bridge/skills/adversarial-review/SKILL.md +24 -0
- package/plugins/claude-codex-bridge/skills/result/SKILL.md +22 -0
- package/plugins/claude-codex-bridge/skills/review/SKILL.md +28 -0
- package/plugins/claude-codex-bridge/skills/second-pass/SKILL.md +27 -0
- package/plugins/claude-codex-bridge/skills/setup/SKILL.md +21 -0
- package/plugins/claude-codex-bridge/skills/status/SKILL.md +19 -0
- package/plugins/claude-skill/INSTALL.md +55 -0
- package/plugins/claude-skill/SKILL.md +46 -0
- package/plugins/codex-profile/.codex-plugin/plugin.json +43 -0
- package/plugins/codex-profile/.mcp.json +12 -0
- package/plugins/codex-profile/AGENTS.md +20 -0
- package/plugins/codex-profile/INSTALL.md +66 -0
- package/plugins/codex-profile/README.md +37 -0
- package/plugins/cursor-marketplace/.cursor-plugin/plugin.json +23 -0
- package/plugins/cursor-marketplace/CHANGELOG.md +30 -0
- package/plugins/cursor-marketplace/LICENSE +21 -0
- package/plugins/cursor-marketplace/README.md +124 -0
- package/plugins/cursor-marketplace/agents/reliability-reviewer.md +31 -0
- package/plugins/cursor-marketplace/assets/logo-400x400.png +0 -0
- package/plugins/cursor-marketplace/commands/capture-feedback.md +33 -0
- package/plugins/cursor-marketplace/commands/check-gates.md +25 -0
- package/plugins/cursor-marketplace/commands/show-lessons.md +27 -0
- package/plugins/cursor-marketplace/hooks/hooks.json +10 -0
- package/plugins/cursor-marketplace/mcp.json +12 -0
- package/plugins/cursor-marketplace/rules/feedback-capture.mdc +34 -0
- package/plugins/cursor-marketplace/rules/pre-action-gates.mdc +30 -0
- package/plugins/cursor-marketplace/rules/session-continuity.mdc +28 -0
- package/plugins/cursor-marketplace/scripts/gate-check.sh +11 -0
- package/plugins/cursor-marketplace/skills/capture-feedback/SKILL.md +47 -0
- package/plugins/cursor-marketplace/skills/prevention-rules/SKILL.md +31 -0
- package/plugins/cursor-marketplace/skills/recall-context/SKILL.md +30 -0
- package/plugins/cursor-marketplace/skills/search-lessons/SKILL.md +33 -0
- package/plugins/gemini-extension/INSTALL.md +92 -0
- package/plugins/gemini-extension/gemini_prompt.txt +14 -0
- package/plugins/gemini-extension/tool_contract.json +45 -0
- package/plugins/opencode-profile/INSTALL.md +57 -0
- package/public/assets/instagram-card.png +0 -0
- package/public/assets/tiktok-agent-memory.mp4 +0 -0
- package/public/blog.html +400 -0
- package/public/dashboard.html +1093 -0
- package/public/guide.html +317 -0
- package/public/index.html +1195 -0
- package/public/learn/agent-harness-pattern.html +180 -0
- package/public/learn/ai-agent-persistent-memory.html +202 -0
- package/public/learn/learn.css +45 -0
- package/public/learn/mcp-pre-action-gates-explained.html +172 -0
- package/public/learn/stop-ai-agent-force-push.html +134 -0
- package/public/learn/vibe-coding-safety-net.html +142 -0
- package/public/learn.html +213 -0
- package/public/lessons.html +650 -0
- package/public/vercel.json +8 -0
- package/scripts/__pycache__/train_from_feedback.cpython-314.pyc +0 -0
- package/scripts/a2ui-engine.js +73 -0
- package/scripts/access-anomaly-detector.js +12 -0
- package/scripts/adk-consolidator.js +266 -0
- package/scripts/agent-readiness.js +220 -0
- package/scripts/agent-security-hardening.js +227 -0
- package/scripts/agentic-data-pipeline.js +847 -0
- package/scripts/analytics-report.js +328 -0
- package/scripts/analytics-window.js +158 -0
- package/scripts/async-job-runner.js +1001 -0
- package/scripts/audit-trail.js +398 -0
- package/scripts/auto-promote-gates.js +293 -0
- package/scripts/auto-wire-hooks.js +316 -0
- package/scripts/autonomous-sales-agent.js +39 -0
- package/scripts/autoresearch-runner.js +216 -0
- package/scripts/background-agent-governance.js +237 -0
- package/scripts/behavioral-extraction.js +93 -0
- package/scripts/belief-update.js +84 -0
- package/scripts/billing.js +2438 -0
- package/scripts/bot-detector.js +50 -0
- package/scripts/budget-guard.js +173 -0
- package/scripts/build-claude-mcpb.js +189 -0
- package/scripts/build-metadata.js +97 -0
- package/scripts/check-congruence.js +322 -0
- package/scripts/cli-feedback.js +135 -0
- package/scripts/cli-telemetry.js +87 -0
- package/scripts/cloudflare-dynamic-sandbox.js +315 -0
- package/scripts/code-reasoning.js +350 -0
- package/scripts/codegraph-context.js +466 -0
- package/scripts/commercial-offer.js +56 -0
- package/scripts/computer-use-firewall.js +250 -0
- package/scripts/context-engine.js +694 -0
- package/scripts/contextfs.js +1287 -0
- package/scripts/conversation-context.js +119 -0
- package/scripts/creator-campaigns.js +239 -0
- package/scripts/daemon-manager.js +108 -0
- package/scripts/daily-digest.js +11 -0
- package/scripts/dashboard-render-spec.js +395 -0
- package/scripts/dashboard.js +1058 -0
- package/scripts/data-governance.js +173 -0
- package/scripts/delegation-runtime.js +900 -0
- package/scripts/deploy-gcp.sh +44 -0
- package/scripts/deploy-policy.js +231 -0
- package/scripts/disagreement-mining.js +315 -0
- package/scripts/dispatch-brief.js +159 -0
- package/scripts/distribution-surfaces.js +44 -0
- package/scripts/dpo-optimizer.js +206 -0
- package/scripts/ensure-repo-bootstrap.js +129 -0
- package/scripts/ephemeral-agent-store.js +219 -0
- package/scripts/eval-harness.js +56 -0
- package/scripts/evolution-state.js +241 -0
- package/scripts/experiment-tracker.js +267 -0
- package/scripts/export-databricks-bundle.js +242 -0
- package/scripts/export-dpo-pairs.js +344 -0
- package/scripts/export-kto-pairs.js +309 -0
- package/scripts/export-training.js +450 -0
- package/scripts/failure-diagnostics.js +558 -0
- package/scripts/feedback-attribution.js +313 -0
- package/scripts/feedback-fallback.js +110 -0
- package/scripts/feedback-history-distiller.js +391 -0
- package/scripts/feedback-inbox-read.js +162 -0
- package/scripts/feedback-loop.js +1887 -0
- package/scripts/feedback-paths.js +145 -0
- package/scripts/feedback-quality.js +139 -0
- package/scripts/feedback-root-consolidator.js +238 -0
- package/scripts/feedback-schema.js +426 -0
- package/scripts/feedback-session.js +286 -0
- package/scripts/feedback-to-memory.js +185 -0
- package/scripts/feedback-to-rules.js +164 -0
- package/scripts/filesystem-search.js +405 -0
- package/scripts/funnel-analytics.js +35 -0
- package/scripts/gate-satisfy.js +42 -0
- package/scripts/gate-stats.js +116 -0
- package/scripts/gate-templates.js +70 -0
- package/scripts/gates-engine.js +816 -0
- package/scripts/generate-paperbanana-diagrams.sh +99 -0
- package/scripts/generate-pretool-hook.sh +40 -0
- package/scripts/github-about.js +350 -0
- package/scripts/github-outreach.js +65 -0
- package/scripts/gtm-revenue-loop.js +520 -0
- package/scripts/hallucination-detector.js +226 -0
- package/scripts/hf-papers.js +317 -0
- package/scripts/history-distiller.js +200 -0
- package/scripts/hook-auto-capture.sh +100 -0
- package/scripts/hook-stop-pr-thread-check.sh +68 -0
- package/scripts/hook-stop-self-score.sh +51 -0
- package/scripts/hook-stop-verify-deploy.sh +31 -0
- package/scripts/hook-thumbgate-cache-updater.js +48 -0
- package/scripts/hook-verify-before-done.sh +20 -0
- package/scripts/hosted-config.js +156 -0
- package/scripts/hybrid-feedback-context.js +675 -0
- package/scripts/install-mcp.js +159 -0
- package/scripts/intent-router.js +392 -0
- package/scripts/internal-agent-bootstrap.js +490 -0
- package/scripts/jsonl-watcher.js +155 -0
- package/scripts/lesson-db.js +613 -0
- package/scripts/lesson-inference.js +310 -0
- package/scripts/lesson-retrieval.js +95 -0
- package/scripts/lesson-rotation.js +137 -0
- package/scripts/lesson-search.js +644 -0
- package/scripts/lesson-synthesis.js +196 -0
- package/scripts/license.js +50 -0
- package/scripts/local-model-profile.js +384 -0
- package/scripts/markdown-escape.js +12 -0
- package/scripts/marketing-experiment.js +671 -0
- package/scripts/mcp-config.js +149 -0
- package/scripts/mcp-policy.js +99 -0
- package/scripts/memalign-recall.js +111 -0
- package/scripts/memory-firewall.js +222 -0
- package/scripts/memory-migration.js +296 -0
- package/scripts/meta-policy.js +190 -0
- package/scripts/metered-billing.js +16 -0
- package/scripts/model-tier-router.js +301 -0
- package/scripts/money-watcher.js +71 -0
- package/scripts/multi-hop-recall.js +240 -0
- package/scripts/natural-language-harness.js +330 -0
- package/scripts/obsidian-export.js +713 -0
- package/scripts/operational-dashboard.js +103 -0
- package/scripts/operational-summary.js +93 -0
- package/scripts/optimize-context.js +17 -0
- package/scripts/org-dashboard.js +201 -0
- package/scripts/partner-orchestration.js +146 -0
- package/scripts/per-step-scoring.js +165 -0
- package/scripts/perplexity-marketing.js +466 -0
- package/scripts/pii-scanner.js +153 -0
- package/scripts/plan-gate.js +154 -0
- package/scripts/post-everywhere.js +308 -0
- package/scripts/post-to-x-retry.sh +22 -0
- package/scripts/post-to-x.js +369 -0
- package/scripts/pr-manager.js +236 -0
- package/scripts/predictive-insights.js +356 -0
- package/scripts/principle-extractor.js +162 -0
- package/scripts/pro-features.js +40 -0
- package/scripts/pro-local-dashboard.js +174 -0
- package/scripts/problem-detail.js +53 -0
- package/scripts/product-feedback.js +134 -0
- package/scripts/profile-router.js +245 -0
- package/scripts/prompt-dlp.js +221 -0
- package/scripts/prompt-guard.js +83 -0
- package/scripts/prove-adapters.js +863 -0
- package/scripts/prove-attribution.js +365 -0
- package/scripts/prove-automation.js +653 -0
- package/scripts/prove-autoresearch.js +304 -0
- package/scripts/prove-claim-verification.js +277 -0
- package/scripts/prove-cloudflare-sandbox.js +163 -0
- package/scripts/prove-data-pipeline.js +410 -0
- package/scripts/prove-data-quality.js +227 -0
- package/scripts/prove-evolution.js +352 -0
- package/scripts/prove-harnesses.js +287 -0
- package/scripts/prove-intelligence.js +259 -0
- package/scripts/prove-lancedb.js +371 -0
- package/scripts/prove-local-intelligence.js +342 -0
- package/scripts/prove-loop-closure.js +263 -0
- package/scripts/prove-predictive-insights.js +357 -0
- package/scripts/prove-runtime.js +350 -0
- package/scripts/prove-seo-gsd.js +234 -0
- package/scripts/prove-settings.js +279 -0
- package/scripts/prove-subway-upgrades.js +277 -0
- package/scripts/prove-tessl.js +229 -0
- package/scripts/prove-training-export.js +327 -0
- package/scripts/prove-workflow-contract.js +116 -0
- package/scripts/prove-xmemory.js +332 -0
- package/scripts/publish-decision.js +133 -0
- package/scripts/pulse.js +80 -0
- package/scripts/rate-limiter.js +125 -0
- package/scripts/reddit-dm-outreach.js +182 -0
- package/scripts/reddit-monitor-cron.sh +26 -0
- package/scripts/reflector-agent.js +221 -0
- package/scripts/reminder-engine.js +132 -0
- package/scripts/revenue-status.js +472 -0
- package/scripts/risk-scorer.js +459 -0
- package/scripts/rlaif-self-audit.js +129 -0
- package/scripts/rlhf_session_start.sh +32 -0
- package/scripts/rubric-engine.js +230 -0
- package/scripts/schedule-manager.js +251 -0
- package/scripts/secret-scanner.js +414 -0
- package/scripts/self-heal.js +147 -0
- package/scripts/self-healing-check.js +188 -0
- package/scripts/semantic-layer.js +98 -0
- package/scripts/seo-gsd.js +1153 -0
- package/scripts/settings-hierarchy.js +214 -0
- package/scripts/shieldcortex-memory-firewall-runner.mjs +53 -0
- package/scripts/skill-exporter.js +262 -0
- package/scripts/skill-generator.js +446 -0
- package/scripts/skill-materializer.js +134 -0
- package/scripts/skill-packs.js +136 -0
- package/scripts/skill-proposer.js +99 -0
- package/scripts/skill-quality-tracker.js +282 -0
- package/scripts/slo-alert-engine.js +14 -0
- package/scripts/slow-loop.js +72 -0
- package/scripts/social-analytics/db/schema.sql +32 -0
- package/scripts/social-analytics/db/social-analytics.db +0 -0
- package/scripts/social-analytics/digest.js +256 -0
- package/scripts/social-analytics/generate-instagram-card.js +97 -0
- package/scripts/social-analytics/instagram-thumbgate-post.js +107 -0
- package/scripts/social-analytics/load-env.js +46 -0
- package/scripts/social-analytics/mcp-server.js +289 -0
- package/scripts/social-analytics/normalizer.js +580 -0
- package/scripts/social-analytics/notify.js +162 -0
- package/scripts/social-analytics/poll-all.js +92 -0
- package/scripts/social-analytics/pollers/github.js +195 -0
- package/scripts/social-analytics/pollers/instagram.js +253 -0
- package/scripts/social-analytics/pollers/linkedin.js +330 -0
- package/scripts/social-analytics/pollers/plausible.js +247 -0
- package/scripts/social-analytics/pollers/reddit.js +306 -0
- package/scripts/social-analytics/pollers/threads.js +233 -0
- package/scripts/social-analytics/pollers/tiktok.js +203 -0
- package/scripts/social-analytics/pollers/x.js +227 -0
- package/scripts/social-analytics/pollers/youtube.js +304 -0
- package/scripts/social-analytics/pollers/zernio.js +183 -0
- package/scripts/social-analytics/publish-instagram-thumbgate.js +98 -0
- package/scripts/social-analytics/publish-thumbgate-launch.js +316 -0
- package/scripts/social-analytics/publishers/devto.js +122 -0
- package/scripts/social-analytics/publishers/instagram.js +317 -0
- package/scripts/social-analytics/publishers/linkedin.js +294 -0
- package/scripts/social-analytics/publishers/reddit.js +390 -0
- package/scripts/social-analytics/publishers/threads.js +275 -0
- package/scripts/social-analytics/publishers/tiktok.js +217 -0
- package/scripts/social-analytics/publishers/x.js +259 -0
- package/scripts/social-analytics/publishers/youtube.js +223 -0
- package/scripts/social-analytics/publishers/zernio.js +378 -0
- package/scripts/social-analytics/run-digest.js +34 -0
- package/scripts/social-analytics/store.js +257 -0
- package/scripts/social-analytics/utm.js +143 -0
- package/scripts/social-pipeline.js +2628 -0
- package/scripts/social-quality-gate.js +18 -0
- package/scripts/social-reply-monitor.js +445 -0
- package/scripts/status-dashboard.js +155 -0
- package/scripts/statusline-lesson.js +16 -0
- package/scripts/statusline-tower.js +8 -0
- package/scripts/statusline.sh +116 -0
- package/scripts/stripe-live-status.js +115 -0
- package/scripts/subagent-profiles.js +79 -0
- package/scripts/sync-gh-secrets-from-env.sh +70 -0
- package/scripts/sync-github-about.js +52 -0
- package/scripts/sync-version.js +447 -0
- package/scripts/synthetic-dpo.js +234 -0
- package/scripts/telemetry-analytics.js +821 -0
- package/scripts/tessl-export.js +371 -0
- package/scripts/test-coverage.js +120 -0
- package/scripts/thompson-sampling.js +417 -0
- package/scripts/thumbgate-search.js +189 -0
- package/scripts/tool-kpi-tracker.js +12 -0
- package/scripts/tool-registry.js +811 -0
- package/scripts/train_from_feedback.py +933 -0
- package/scripts/user-profile.js +78 -0
- package/scripts/validate-feedback.js +581 -0
- package/scripts/validate-workflow-contract.js +287 -0
- package/scripts/vector-store.js +197 -0
- package/scripts/verification-loop.js +291 -0
- package/scripts/verify-obsidian-setup.sh +269 -0
- package/scripts/verify-run.js +269 -0
- package/scripts/webhook-delivery.js +62 -0
- package/scripts/weekly-auto-post.js +124 -0
- package/scripts/workflow-runs.js +154 -0
- package/scripts/workflow-sprint-intake.js +475 -0
- package/scripts/workspace-evolver.js +374 -0
- package/scripts/x-autonomous-marketing.js +139 -0
- package/scripts/xmemory-lite.js +405 -0
- package/skills/agent-memory/SKILL.md +97 -0
- package/skills/rlhf-feedback/SKILL.md +49 -0
- package/skills/solve-architecture-autonomy/SKILL.md +17 -0
- package/skills/solve-architecture-autonomy/tool.js +33 -0
- package/skills/thumbgate/SKILL.md +114 -0
- package/src/api/server.js +4206 -0
|
@@ -0,0 +1,1887 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* ThumbGate (local-first)
|
|
4
|
+
*
|
|
5
|
+
* Pipeline:
|
|
6
|
+
* thumbs up/down -> resolve action -> validate memory -> append logs
|
|
7
|
+
* -> compute analytics -> generate prevention rules
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const {
|
|
13
|
+
resolveFeedbackAction,
|
|
14
|
+
prepareForStorage,
|
|
15
|
+
parseTimestamp,
|
|
16
|
+
GENERIC_TAGS,
|
|
17
|
+
} = require('./feedback-schema');
|
|
18
|
+
const {
|
|
19
|
+
buildClarificationMessage,
|
|
20
|
+
isGenericFeedbackText,
|
|
21
|
+
} = require('./feedback-quality');
|
|
22
|
+
const {
|
|
23
|
+
buildRubricEvaluation,
|
|
24
|
+
} = require('./rubric-engine');
|
|
25
|
+
const { recordAction, attributeFeedback } = require('./feedback-attribution');
|
|
26
|
+
const {
|
|
27
|
+
distillFeedbackHistory,
|
|
28
|
+
} = require('./feedback-history-distiller');
|
|
29
|
+
const {
|
|
30
|
+
extractFilePaths: extractConversationPaths,
|
|
31
|
+
extractErrors: extractConversationErrors,
|
|
32
|
+
normalizeConversationWindow,
|
|
33
|
+
} = require('./conversation-context');
|
|
34
|
+
const {
|
|
35
|
+
diagnoseFailure,
|
|
36
|
+
aggregateFailureDiagnostics,
|
|
37
|
+
} = require('./failure-diagnostics');
|
|
38
|
+
const { getEffectiveSetting } = require('./evolution-state');
|
|
39
|
+
const {
|
|
40
|
+
buildFeedbackPathsFromDir,
|
|
41
|
+
getFeedbackPaths: resolveFeedbackPaths,
|
|
42
|
+
} = require('./feedback-paths');
|
|
43
|
+
|
|
44
|
+
// Lesson DB — SQLite+FTS5 backing store (dual-write alongside JSONL)
|
|
45
|
+
let _lessonDB = null;
|
|
46
|
+
let _lessonDBPath = null;
|
|
47
|
+
|
|
48
|
+
function resolveLessonDbPath() {
|
|
49
|
+
if (process.env.LESSON_DB_PATH) return process.env.LESSON_DB_PATH;
|
|
50
|
+
if (process.env.THUMBGATE_FEEDBACK_DIR || process.env.RAILWAY_VOLUME_MOUNT_PATH) {
|
|
51
|
+
return path.join(getFeedbackPaths().FEEDBACK_DIR, 'lessons.sqlite');
|
|
52
|
+
}
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function getLessonDB() {
|
|
57
|
+
const desiredPath = resolveLessonDbPath();
|
|
58
|
+
if (_lessonDB && _lessonDBPath === desiredPath) return _lessonDB;
|
|
59
|
+
|
|
60
|
+
if (_lessonDB && _lessonDBPath !== desiredPath) {
|
|
61
|
+
try {
|
|
62
|
+
_lessonDB.close();
|
|
63
|
+
} catch {
|
|
64
|
+
// Non-critical; reopen on the new path below.
|
|
65
|
+
}
|
|
66
|
+
_lessonDB = null;
|
|
67
|
+
_lessonDBPath = null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
try {
|
|
71
|
+
const { initDB } = require('./lesson-db');
|
|
72
|
+
_lessonDB = desiredPath ? initDB(desiredPath) : initDB();
|
|
73
|
+
_lessonDBPath = desiredPath;
|
|
74
|
+
return _lessonDB;
|
|
75
|
+
} catch (_err) {
|
|
76
|
+
// Keep the DB path scoped to the active feedback root even when SQLite
|
|
77
|
+
// cannot open (for example, native module ABI drift in local dev).
|
|
78
|
+
if (desiredPath) {
|
|
79
|
+
try {
|
|
80
|
+
fs.mkdirSync(path.dirname(desiredPath), { recursive: true });
|
|
81
|
+
fs.closeSync(fs.openSync(desiredPath, 'a'));
|
|
82
|
+
_lessonDBPath = desiredPath;
|
|
83
|
+
} catch {
|
|
84
|
+
// Ignore file materialization failures and degrade gracefully below.
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return null; // SQLite unavailable — degrade gracefully
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// ML sequence tracking constants (ML-03)
|
|
92
|
+
const SEQUENCE_WINDOW = 10;
|
|
93
|
+
const DOMAIN_CATEGORIES = [
|
|
94
|
+
'testing', 'security', 'performance', 'ui-components', 'api-integration',
|
|
95
|
+
'git-workflow', 'documentation', 'debugging', 'architecture', 'data-modeling',
|
|
96
|
+
'behavioral',
|
|
97
|
+
];
|
|
98
|
+
|
|
99
|
+
const HOME = process.env.HOME || process.env.USERPROFILE || '';
|
|
100
|
+
const pendingBackgroundSideEffects = new Set();
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Update the statusline cache with latest lesson info after feedback capture.
|
|
104
|
+
* The statusline.sh script reads this cache to display lesson context in Claude Code's status bar.
|
|
105
|
+
*/
|
|
106
|
+
function updateStatuslineWithLesson({ accepted, signal, memoryId, feedbackId, lesson, turnCount }) {
|
|
107
|
+
try {
|
|
108
|
+
const cacheDir = process.env.THUMBGATE_FEEDBACK_DIR || HOME || '.';
|
|
109
|
+
const cachePath = path.join(cacheDir, '.thumbgate', 'statusline_cache.json');
|
|
110
|
+
let cache = {};
|
|
111
|
+
try {
|
|
112
|
+
cache = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
|
|
113
|
+
} catch { /* cache may not exist yet */ }
|
|
114
|
+
|
|
115
|
+
if (accepted) {
|
|
116
|
+
const icon = signal === 'positive' ? '\u2705' : '\u274C';
|
|
117
|
+
const summary = (lesson || '').slice(0, 80).replace(/\n/g, ' ');
|
|
118
|
+
cache.last_lesson = {
|
|
119
|
+
icon,
|
|
120
|
+
memoryId: memoryId || null,
|
|
121
|
+
feedbackId: feedbackId || null,
|
|
122
|
+
signal: signal || null,
|
|
123
|
+
summary,
|
|
124
|
+
turnCount: turnCount || 0,
|
|
125
|
+
timestamp: Math.floor(Date.now() / 1000),
|
|
126
|
+
};
|
|
127
|
+
} else {
|
|
128
|
+
cache.last_lesson = {
|
|
129
|
+
icon: '\u26A0\uFE0F',
|
|
130
|
+
memoryId: null,
|
|
131
|
+
feedbackId: feedbackId || null,
|
|
132
|
+
signal: signal || null,
|
|
133
|
+
summary: 'Feedback needs detail \u2014 describe what worked/failed',
|
|
134
|
+
turnCount: 0,
|
|
135
|
+
timestamp: Math.floor(Date.now() / 1000),
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
cache.updated_at = String(Math.floor(Date.now() / 1000));
|
|
139
|
+
fs.mkdirSync(path.dirname(cachePath), { recursive: true });
|
|
140
|
+
fs.writeFileSync(cachePath, JSON.stringify(cache));
|
|
141
|
+
} catch { /* statusline update is best-effort */ }
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function getFeedbackPaths() {
|
|
145
|
+
return resolveFeedbackPaths();
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function getContextFsModule() {
|
|
149
|
+
try {
|
|
150
|
+
return require('./contextfs');
|
|
151
|
+
} catch {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function getVectorStoreModule() {
|
|
157
|
+
// Prefer filesystem search (no embeddings, no LanceDB binary dependency).
|
|
158
|
+
// Falls back to vector-store.js if filesystem-search.js is missing.
|
|
159
|
+
try {
|
|
160
|
+
return require('./filesystem-search');
|
|
161
|
+
} catch {
|
|
162
|
+
try {
|
|
163
|
+
return require('./vector-store');
|
|
164
|
+
} catch {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function getRiskScorerModule() {
|
|
171
|
+
try {
|
|
172
|
+
return require('./risk-scorer');
|
|
173
|
+
} catch {
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function getSelfAuditModule() {
|
|
179
|
+
try {
|
|
180
|
+
return require('./rlaif-self-audit');
|
|
181
|
+
} catch (_) {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function getDelegationRuntimeModule() {
|
|
187
|
+
try {
|
|
188
|
+
return require('./delegation-runtime');
|
|
189
|
+
} catch {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function getMemoryFirewallModule() {
|
|
195
|
+
try {
|
|
196
|
+
return require('./memory-firewall');
|
|
197
|
+
} catch {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function ensureDir(dirPath) {
|
|
203
|
+
if (!fs.existsSync(dirPath)) {
|
|
204
|
+
fs.mkdirSync(dirPath, { recursive: true });
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function appendJSONL(filePath, record) {
|
|
209
|
+
ensureDir(path.dirname(filePath));
|
|
210
|
+
fs.appendFileSync(filePath, `${JSON.stringify(record)}\n`);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function normalizeAnalysisShape(analysis = {}) {
|
|
214
|
+
const total = Number.isFinite(analysis.total) ? analysis.total : 0;
|
|
215
|
+
const totalPositive = Number.isFinite(analysis.totalPositive)
|
|
216
|
+
? analysis.totalPositive
|
|
217
|
+
: Number.isFinite(analysis.positive) ? analysis.positive : 0;
|
|
218
|
+
const totalNegative = Number.isFinite(analysis.totalNegative)
|
|
219
|
+
? analysis.totalNegative
|
|
220
|
+
: Number.isFinite(analysis.negative) ? analysis.negative : Math.max(0, total - totalPositive);
|
|
221
|
+
const approvalRate = Number.isFinite(analysis.approvalRate)
|
|
222
|
+
? analysis.approvalRate
|
|
223
|
+
: Number.isFinite(analysis.positiveRate)
|
|
224
|
+
? Number((analysis.positiveRate / 100).toFixed(3))
|
|
225
|
+
: total > 0 ? Number((totalPositive / total).toFixed(3)) : 0;
|
|
226
|
+
const recentRate = Number.isFinite(analysis.recentRate) ? analysis.recentRate : approvalRate;
|
|
227
|
+
|
|
228
|
+
return {
|
|
229
|
+
total,
|
|
230
|
+
totalPositive,
|
|
231
|
+
totalNegative,
|
|
232
|
+
approvalRate,
|
|
233
|
+
recentRate,
|
|
234
|
+
windows: analysis.windows || {
|
|
235
|
+
'7d': { total: 0, positive: 0, rate: 0 },
|
|
236
|
+
'30d': { total: 0, positive: 0, rate: 0 },
|
|
237
|
+
lifetime: { total, positive: totalPositive, rate: approvalRate },
|
|
238
|
+
},
|
|
239
|
+
trend: analysis.trend || 'stable',
|
|
240
|
+
skills: analysis.skills || {},
|
|
241
|
+
tags: analysis.tags || {},
|
|
242
|
+
rubric: {
|
|
243
|
+
samples: 0,
|
|
244
|
+
blockedPromotions: 0,
|
|
245
|
+
failingCriteria: {},
|
|
246
|
+
...(analysis.rubric || {}),
|
|
247
|
+
},
|
|
248
|
+
diagnostics: analysis.diagnostics || {
|
|
249
|
+
totalDiagnosed: 0,
|
|
250
|
+
categories: [],
|
|
251
|
+
criticalFailureSteps: [],
|
|
252
|
+
repeatedViolations: [],
|
|
253
|
+
},
|
|
254
|
+
delegation: analysis.delegation || null,
|
|
255
|
+
boostedRisk: analysis.boostedRisk || null,
|
|
256
|
+
recommendations: Array.isArray(analysis.recommendations) ? analysis.recommendations : [],
|
|
257
|
+
source: analysis.source,
|
|
258
|
+
byDomain: Array.isArray(analysis.byDomain) ? analysis.byDomain : [],
|
|
259
|
+
byImportance: Array.isArray(analysis.byImportance) ? analysis.byImportance : [],
|
|
260
|
+
recentLessons: Array.isArray(analysis.recentLessons) ? analysis.recentLessons : [],
|
|
261
|
+
sessionCount: Number.isFinite(analysis.sessionCount) ? analysis.sessionCount : 0,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Check if a memory from the same feedback event already exists (retry/race dedup).
|
|
267
|
+
* Only blocks true duplicates (same sourceFeedbackId). Different feedback events
|
|
268
|
+
* that produce identical content are allowed — they represent real repeated signal.
|
|
269
|
+
*/
|
|
270
|
+
function findDuplicateMemory(memoryLogPath, newRecord) {
|
|
271
|
+
const feedbackId = newRecord.sourceFeedbackId;
|
|
272
|
+
if (!feedbackId) return null;
|
|
273
|
+
|
|
274
|
+
const existing = readJSONL(memoryLogPath, { maxLines: 0 });
|
|
275
|
+
for (let i = existing.length - 1; i >= 0; i--) {
|
|
276
|
+
if (existing[i].sourceFeedbackId === feedbackId) return existing[i];
|
|
277
|
+
}
|
|
278
|
+
return null;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function toStoredDiagnosis(diagnosis) {
|
|
282
|
+
if (!diagnosis || diagnosis.diagnosed === false || !diagnosis.rootCauseCategory) {
|
|
283
|
+
return null;
|
|
284
|
+
}
|
|
285
|
+
return {
|
|
286
|
+
rootCauseCategory: diagnosis.rootCauseCategory,
|
|
287
|
+
criticalFailureStep: diagnosis.criticalFailureStep,
|
|
288
|
+
violations: Array.isArray(diagnosis.violations) ? diagnosis.violations : [],
|
|
289
|
+
evidence: Array.isArray(diagnosis.evidence) ? diagnosis.evidence : [],
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function appendRejectionLedger(feedbackEvent, reason) {
|
|
294
|
+
const { REJECTION_LEDGER_PATH } = getFeedbackPaths();
|
|
295
|
+
appendJSONL(REJECTION_LEDGER_PATH, {
|
|
296
|
+
id: feedbackEvent.id,
|
|
297
|
+
signal: feedbackEvent.signal,
|
|
298
|
+
context: feedbackEvent.context || '',
|
|
299
|
+
reason,
|
|
300
|
+
tags: feedbackEvent.tags || [],
|
|
301
|
+
revivalCondition: feedbackEvent.signal === 'negative'
|
|
302
|
+
? 'Re-submit with whatWentWrong and whatToChange fields populated'
|
|
303
|
+
: 'Re-submit with whatWorked field and at least one domain-specific tag',
|
|
304
|
+
timestamp: feedbackEvent.timestamp || new Date().toISOString(),
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function listEnforcementMatrix() {
|
|
309
|
+
const paths = getFeedbackPaths();
|
|
310
|
+
const feedbackEntries = readJSONL(paths.FEEDBACK_LOG_PATH);
|
|
311
|
+
const memoryEntries = readJSONL(paths.MEMORY_LOG_PATH);
|
|
312
|
+
const rejections = readJSONL(paths.REJECTION_LEDGER_PATH);
|
|
313
|
+
|
|
314
|
+
let autoGates = { gates: [], promotionLog: [] };
|
|
315
|
+
try {
|
|
316
|
+
const apg = require('./auto-promote-gates');
|
|
317
|
+
autoGates = apg.loadAutoGates();
|
|
318
|
+
} catch { /* auto-promote-gates not available */ }
|
|
319
|
+
|
|
320
|
+
const totalFeedback = feedbackEntries.length;
|
|
321
|
+
const promoted = memoryEntries.length;
|
|
322
|
+
const rejected = rejections.length;
|
|
323
|
+
|
|
324
|
+
const reasonCounts = {};
|
|
325
|
+
for (const r of rejections) {
|
|
326
|
+
const key = r.reason || 'unknown';
|
|
327
|
+
reasonCounts[key] = (reasonCounts[key] || 0) + 1;
|
|
328
|
+
}
|
|
329
|
+
const topRejections = Object.entries(reasonCounts)
|
|
330
|
+
.sort((a, b) => b[1] - a[1])
|
|
331
|
+
.slice(0, 5)
|
|
332
|
+
.map(([reason, count]) => ({ reason, count }));
|
|
333
|
+
|
|
334
|
+
return {
|
|
335
|
+
pipeline: {
|
|
336
|
+
totalFeedback,
|
|
337
|
+
promoted,
|
|
338
|
+
rejected,
|
|
339
|
+
promotionRate: totalFeedback > 0 ? Math.round((promoted / totalFeedback) * 100) : 0,
|
|
340
|
+
},
|
|
341
|
+
gates: {
|
|
342
|
+
active: autoGates.gates.length,
|
|
343
|
+
blocking: autoGates.gates.filter((g) => g.action === 'block').length,
|
|
344
|
+
warning: autoGates.gates.filter((g) => g.action === 'warn').length,
|
|
345
|
+
rules: autoGates.gates.map((g) => ({
|
|
346
|
+
id: g.id, action: g.action, pattern: g.pattern,
|
|
347
|
+
occurrences: g.occurrences, promotedAt: g.promotedAt,
|
|
348
|
+
})),
|
|
349
|
+
},
|
|
350
|
+
rejectionLedger: {
|
|
351
|
+
total: rejected,
|
|
352
|
+
topReasons: topRejections,
|
|
353
|
+
recentRejections: rejections.slice(-5).reverse(),
|
|
354
|
+
},
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function appendDiagnosticRecord(params = {}) {
|
|
359
|
+
const { DIAGNOSTIC_LOG_PATH } = getFeedbackPaths();
|
|
360
|
+
const storedDiagnosis = toStoredDiagnosis(params.diagnosis);
|
|
361
|
+
if (!storedDiagnosis) {
|
|
362
|
+
return null;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
const record = {
|
|
366
|
+
id: `diag_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
|
367
|
+
source: params.source || 'system',
|
|
368
|
+
step: params.step || storedDiagnosis.criticalFailureStep || null,
|
|
369
|
+
context: params.context || '',
|
|
370
|
+
metadata: params.metadata && typeof params.metadata === 'object' ? params.metadata : {},
|
|
371
|
+
diagnosis: storedDiagnosis,
|
|
372
|
+
timestamp: params.timestamp || new Date().toISOString(),
|
|
373
|
+
};
|
|
374
|
+
appendJSONL(DIAGNOSTIC_LOG_PATH, record);
|
|
375
|
+
return record;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function buildMemoryFirewallViolations(decision = {}) {
|
|
379
|
+
const findingViolations = Array.isArray(decision.findings)
|
|
380
|
+
? decision.findings.map((finding) => ({
|
|
381
|
+
constraintId: `security:${finding.id || 'credential_leak'}`,
|
|
382
|
+
description: finding.reason || finding.label || 'Blocked by memory-ingress firewall',
|
|
383
|
+
metadata: {
|
|
384
|
+
label: finding.label || finding.id || null,
|
|
385
|
+
line: finding.line || null,
|
|
386
|
+
source: finding.source || null,
|
|
387
|
+
},
|
|
388
|
+
}))
|
|
389
|
+
: [];
|
|
390
|
+
|
|
391
|
+
if (findingViolations.length > 0) {
|
|
392
|
+
return findingViolations;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
return (decision.threatIndicators || []).map((indicator) => ({
|
|
396
|
+
constraintId: `security:${indicator}`,
|
|
397
|
+
description: `Blocked by memory-ingress firewall (${indicator})`,
|
|
398
|
+
metadata: {
|
|
399
|
+
provider: decision.provider || null,
|
|
400
|
+
mode: decision.mode || null,
|
|
401
|
+
},
|
|
402
|
+
}));
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
function maybeBlockMemoryIngress({ feedbackEvent, memoryRecord = null, summary, now }) {
|
|
406
|
+
const memoryFirewall = getMemoryFirewallModule();
|
|
407
|
+
if (!memoryFirewall || typeof memoryFirewall.evaluateMemoryIngress !== 'function') {
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
const decision = memoryFirewall.evaluateMemoryIngress({
|
|
412
|
+
feedbackEvent,
|
|
413
|
+
memoryRecord,
|
|
414
|
+
sourceIdentifier: 'feedback-loop',
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
if (!decision || decision.allowed) {
|
|
418
|
+
return null;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
appendDiagnosticRecord({
|
|
422
|
+
source: 'memory_firewall',
|
|
423
|
+
step: 'memory_ingress',
|
|
424
|
+
context: decision.redactedPreview || '',
|
|
425
|
+
metadata: {
|
|
426
|
+
provider: decision.provider || 'unknown',
|
|
427
|
+
mode: decision.mode || null,
|
|
428
|
+
degraded: Boolean(decision.degraded),
|
|
429
|
+
firewallResult: decision.firewallResult || null,
|
|
430
|
+
blockedPatterns: Array.isArray(decision.blockedPatterns) ? decision.blockedPatterns : [],
|
|
431
|
+
requestedProvider: decision.requestedProvider || null,
|
|
432
|
+
},
|
|
433
|
+
diagnosis: {
|
|
434
|
+
diagnosed: true,
|
|
435
|
+
rootCauseCategory: 'guardrail_triggered',
|
|
436
|
+
criticalFailureStep: 'memory_ingress',
|
|
437
|
+
violations: buildMemoryFirewallViolations(decision),
|
|
438
|
+
evidence: [
|
|
439
|
+
decision.reason || 'Memory ingress blocked',
|
|
440
|
+
...(decision.threatIndicators || []),
|
|
441
|
+
].filter(Boolean),
|
|
442
|
+
},
|
|
443
|
+
});
|
|
444
|
+
|
|
445
|
+
summary.rejected += 1;
|
|
446
|
+
summary.lastUpdated = now;
|
|
447
|
+
saveSummary(summary);
|
|
448
|
+
|
|
449
|
+
return {
|
|
450
|
+
accepted: false,
|
|
451
|
+
status: 'blocked',
|
|
452
|
+
reason: decision.reason,
|
|
453
|
+
message: 'Feedback blocked by memory-ingress security checks.',
|
|
454
|
+
feedbackEvent,
|
|
455
|
+
security: {
|
|
456
|
+
provider: decision.provider || 'unknown',
|
|
457
|
+
mode: decision.mode || null,
|
|
458
|
+
threatIndicators: decision.threatIndicators || [],
|
|
459
|
+
degraded: Boolean(decision.degraded),
|
|
460
|
+
},
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
function readDiagnosticEntries(logPath) {
|
|
465
|
+
const { DIAGNOSTIC_LOG_PATH } = getFeedbackPaths();
|
|
466
|
+
return readJSONL(logPath || DIAGNOSTIC_LOG_PATH);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function trackBackgroundSideEffect(taskPromise) {
|
|
470
|
+
if (!taskPromise || typeof taskPromise.then !== 'function') {
|
|
471
|
+
return null;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
let tracked;
|
|
475
|
+
tracked = Promise.resolve(taskPromise)
|
|
476
|
+
.catch(() => {
|
|
477
|
+
// Non-critical side effects should never fail the primary feedback write.
|
|
478
|
+
})
|
|
479
|
+
.finally(() => {
|
|
480
|
+
pendingBackgroundSideEffects.delete(tracked);
|
|
481
|
+
});
|
|
482
|
+
|
|
483
|
+
pendingBackgroundSideEffects.add(tracked);
|
|
484
|
+
return tracked;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
async function waitForBackgroundSideEffects() {
|
|
488
|
+
while (pendingBackgroundSideEffects.size > 0) {
|
|
489
|
+
await Promise.allSettled([...pendingBackgroundSideEffects]);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function getPendingBackgroundSideEffectCount() {
|
|
494
|
+
return pendingBackgroundSideEffects.size;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
function readJSONL(filePath, { maxLines = 500 } = {}) {
|
|
498
|
+
if (!fs.existsSync(filePath)) return [];
|
|
499
|
+
const content = fs.readFileSync(filePath, 'utf8');
|
|
500
|
+
const lines = content.split('\n').filter(Boolean);
|
|
501
|
+
const tail = maxLines > 0 ? lines.slice(-maxLines) : lines;
|
|
502
|
+
const results = [];
|
|
503
|
+
for (const line of tail) {
|
|
504
|
+
try { results.push(JSON.parse(line)); } catch { /* skip malformed */ }
|
|
505
|
+
}
|
|
506
|
+
return results;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
function normalizeSignal(signal) {
|
|
510
|
+
const value = String(signal || '').trim().toLowerCase();
|
|
511
|
+
if (['up', 'thumbsup', 'thumbs-up', 'positive', 'good'].includes(value)) return 'positive';
|
|
512
|
+
if (['down', 'thumbsdown', 'thumbs-down', 'negative', 'bad'].includes(value)) return 'negative';
|
|
513
|
+
if (value === 'thumbs_up') return 'positive';
|
|
514
|
+
if (value === 'thumbs_down') return 'negative';
|
|
515
|
+
return null;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
function parseOptionalObject(input, name) {
|
|
519
|
+
if (input == null) return {};
|
|
520
|
+
if (typeof input === 'object' && !Array.isArray(input)) return input;
|
|
521
|
+
if (typeof input === 'string') {
|
|
522
|
+
const trimmed = input.trim();
|
|
523
|
+
if (!trimmed) return {};
|
|
524
|
+
const parsed = JSON.parse(trimmed);
|
|
525
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
526
|
+
throw new Error(`${name} must be an object`);
|
|
527
|
+
}
|
|
528
|
+
return parsed;
|
|
529
|
+
}
|
|
530
|
+
throw new Error(`${name} must be object or JSON string`);
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function loadSummary() {
|
|
534
|
+
const { SUMMARY_PATH } = getFeedbackPaths();
|
|
535
|
+
if (!fs.existsSync(SUMMARY_PATH)) {
|
|
536
|
+
return {
|
|
537
|
+
total: 0,
|
|
538
|
+
positive: 0,
|
|
539
|
+
negative: 0,
|
|
540
|
+
accepted: 0,
|
|
541
|
+
rejected: 0,
|
|
542
|
+
lastUpdated: null,
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
return JSON.parse(fs.readFileSync(SUMMARY_PATH, 'utf-8'));
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
function saveSummary(summary) {
|
|
549
|
+
const { SUMMARY_PATH } = getFeedbackPaths();
|
|
550
|
+
ensureDir(path.dirname(SUMMARY_PATH));
|
|
551
|
+
fs.writeFileSync(SUMMARY_PATH, `${JSON.stringify(summary, null, 2)}\n`);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
// ============================================================
|
|
555
|
+
// ML Side-Effect Helpers — Sequence Tracking (ML-03) and
|
|
556
|
+
// Diversity Tracking (ML-04). Inline per Subway architecture.
|
|
557
|
+
// ============================================================
|
|
558
|
+
|
|
559
|
+
function inferDomain(tags, context) {
|
|
560
|
+
const tagSet = new Set((tags || []).map((t) => t.toLowerCase()));
|
|
561
|
+
const ctx = (context || '').toLowerCase();
|
|
562
|
+
if (tagSet.has('test') || tagSet.has('testing') || ctx.includes('test')) return 'testing';
|
|
563
|
+
if (tagSet.has('security') || ctx.includes('secret')) return 'security';
|
|
564
|
+
if (tagSet.has('perf') || tagSet.has('performance') || ctx.includes('performance')) return 'performance';
|
|
565
|
+
if (tagSet.has('ui') || tagSet.has('component') || ctx.includes('component')) return 'ui-components';
|
|
566
|
+
if (tagSet.has('api') || tagSet.has('endpoint') || ctx.includes('endpoint')) return 'api-integration';
|
|
567
|
+
if (tagSet.has('git') || tagSet.has('commit') || ctx.includes('commit')) return 'git-workflow';
|
|
568
|
+
if (tagSet.has('doc') || tagSet.has('readme') || ctx.includes('readme')) return 'documentation';
|
|
569
|
+
if (tagSet.has('debug') || tagSet.has('debugging') || ctx.includes('error')) return 'debugging';
|
|
570
|
+
if (tagSet.has('arch') || tagSet.has('architecture') || ctx.includes('design')) return 'architecture';
|
|
571
|
+
if (tagSet.has('data') || tagSet.has('schema') || ctx.includes('schema')) return 'data-modeling';
|
|
572
|
+
return 'general';
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* Infer granular outcome category from signal + context.
|
|
577
|
+
* Satisfies QUAL-03 — beyond binary up/down.
|
|
578
|
+
* @param {string} signal - 'positive' or 'negative'
|
|
579
|
+
* @param {string} context - feedback context string
|
|
580
|
+
* @returns {string} granular outcome category
|
|
581
|
+
*/
|
|
582
|
+
function inferOutcome(signal, context) {
|
|
583
|
+
const cl = (context || '').toLowerCase();
|
|
584
|
+
if (signal === 'positive') {
|
|
585
|
+
if (cl.includes('first try') || cl.includes('immediately') || cl.includes('right away')) return 'quick-success';
|
|
586
|
+
if (cl.includes('thorough') || cl.includes('comprehensive') || cl.includes('in-depth')) return 'deep-success';
|
|
587
|
+
if (cl.includes('creative') || cl.includes('novel') || cl.includes('elegant')) return 'creative-success';
|
|
588
|
+
if (cl.includes('partial') || cl.includes('mostly') || cl.includes('some issues')) return 'partial-success';
|
|
589
|
+
return 'standard-success';
|
|
590
|
+
} else {
|
|
591
|
+
if (cl.includes('wrong') || cl.includes('incorrect') || cl.includes('factual')) return 'factual-error';
|
|
592
|
+
if (cl.includes('shallow') || cl.includes('surface') || cl.includes('superficial')) return 'insufficient-depth';
|
|
593
|
+
if (cl.includes('slow') || cl.includes('took too long') || cl.includes('inefficient')) return 'efficiency-issue';
|
|
594
|
+
if (cl.includes('assumption') || cl.includes('guessed') || cl.includes('assumed')) return 'false-assumption';
|
|
595
|
+
if (cl.includes('partial') || cl.includes('incomplete') || cl.includes('missing')) return 'incomplete';
|
|
596
|
+
return 'standard-failure';
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* Enrich feedbackEvent with richContext metadata.
|
|
602
|
+
* Satisfies QUAL-02 — domain, filePaths, errorType, outcomeCategory.
|
|
603
|
+
* Non-throwing: returns original event on any error.
|
|
604
|
+
* @param {object} feedbackEvent - base feedback event
|
|
605
|
+
* @param {object} params - original captureFeedback params
|
|
606
|
+
* @returns {object} enriched feedbackEvent
|
|
607
|
+
*/
|
|
608
|
+
function enrichFeedbackContext(feedbackEvent, params) {
|
|
609
|
+
try {
|
|
610
|
+
const domain = inferDomain(feedbackEvent.tags, feedbackEvent.context);
|
|
611
|
+
const outcomeCategory = inferOutcome(feedbackEvent.signal, feedbackEvent.context);
|
|
612
|
+
const filePaths = Array.isArray(params.filePaths)
|
|
613
|
+
? params.filePaths
|
|
614
|
+
: typeof params.filePaths === 'string' && params.filePaths.trim()
|
|
615
|
+
? params.filePaths.split(',').map((f) => f.trim()).filter(Boolean)
|
|
616
|
+
: [];
|
|
617
|
+
const errorType = params.errorType || null;
|
|
618
|
+
|
|
619
|
+
return {
|
|
620
|
+
...feedbackEvent,
|
|
621
|
+
richContext: {
|
|
622
|
+
domain,
|
|
623
|
+
filePaths,
|
|
624
|
+
errorType,
|
|
625
|
+
outcomeCategory,
|
|
626
|
+
},
|
|
627
|
+
};
|
|
628
|
+
} catch (_err) {
|
|
629
|
+
return feedbackEvent;
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
function calculateTrend(rewards) {
|
|
634
|
+
if (rewards.length < 2) return 0;
|
|
635
|
+
const recent = rewards.slice(-3);
|
|
636
|
+
return recent.reduce((a, b) => a + b, 0) / recent.length;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
function calculateTimeGaps(sequence) {
|
|
640
|
+
const gaps = [];
|
|
641
|
+
for (let i = 1; i < sequence.length; i++) {
|
|
642
|
+
const prev = parseTimestamp(sequence[i - 1].timestamp);
|
|
643
|
+
const curr = parseTimestamp(sequence[i].timestamp);
|
|
644
|
+
if (prev && curr) {
|
|
645
|
+
gaps.push((curr - prev) / 1000 / 60); // minutes
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
return gaps;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
function extractActionPatterns(sequence) {
|
|
652
|
+
const patterns = {};
|
|
653
|
+
sequence.forEach((f) => {
|
|
654
|
+
(f.tags || []).forEach((tag) => {
|
|
655
|
+
if (!patterns[tag]) patterns[tag] = { positive: 0, negative: 0 };
|
|
656
|
+
if (f.signal === 'positive') patterns[tag].positive++;
|
|
657
|
+
else patterns[tag].negative++;
|
|
658
|
+
});
|
|
659
|
+
});
|
|
660
|
+
return patterns;
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
function buildSequenceFeatures(recentEntries, currentEntry) {
|
|
664
|
+
const sequence = [...recentEntries, currentEntry];
|
|
665
|
+
return {
|
|
666
|
+
rewardSequence: sequence.map((f) => (f.signal === 'positive' ? 1 : -1)),
|
|
667
|
+
tagFrequency: sequence.reduce((acc, f) => {
|
|
668
|
+
(f.tags || []).forEach((tag) => {
|
|
669
|
+
acc[tag] = (acc[tag] || 0) + 1;
|
|
670
|
+
});
|
|
671
|
+
return acc;
|
|
672
|
+
}, {}),
|
|
673
|
+
recentTrend: calculateTrend(sequence.slice(-5).map((f) => (f.signal === 'positive' ? 1 : -1))),
|
|
674
|
+
timeGaps: calculateTimeGaps(sequence),
|
|
675
|
+
actionPatterns: extractActionPatterns(sequence),
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
function appendSequence(historyEntries, feedbackEvent, paths, outcome = {}) {
|
|
680
|
+
const sequencePath = path.join(paths.FEEDBACK_DIR, 'feedback-sequences.jsonl');
|
|
681
|
+
const recent = Array.isArray(historyEntries) ? historyEntries.slice(-SEQUENCE_WINDOW) : [];
|
|
682
|
+
const features = buildSequenceFeatures(recent, feedbackEvent);
|
|
683
|
+
const rubric = feedbackEvent.rubric || null;
|
|
684
|
+
const filePaths = feedbackEvent.richContext && Array.isArray(feedbackEvent.richContext.filePaths)
|
|
685
|
+
? feedbackEvent.richContext.filePaths
|
|
686
|
+
: [];
|
|
687
|
+
const accepted = outcome.accepted === true;
|
|
688
|
+
const targetRisk = feedbackEvent.signal === 'negative' || !accepted ? 1 : 0;
|
|
689
|
+
const entry = {
|
|
690
|
+
id: `seq_${Date.now()}`,
|
|
691
|
+
timestamp: new Date().toISOString(),
|
|
692
|
+
targetReward: feedbackEvent.signal === 'positive' ? 1 : -1,
|
|
693
|
+
targetTags: feedbackEvent.tags,
|
|
694
|
+
accepted,
|
|
695
|
+
actionType: feedbackEvent.actionType || null,
|
|
696
|
+
actionReason: feedbackEvent.actionReason || null,
|
|
697
|
+
context: feedbackEvent.context || '',
|
|
698
|
+
skill: feedbackEvent.skill || null,
|
|
699
|
+
domain: feedbackEvent.richContext ? feedbackEvent.richContext.domain : 'general',
|
|
700
|
+
outcomeCategory: feedbackEvent.richContext ? feedbackEvent.richContext.outcomeCategory : 'unknown',
|
|
701
|
+
filePathCount: filePaths.length,
|
|
702
|
+
errorType: feedbackEvent.richContext ? feedbackEvent.richContext.errorType : null,
|
|
703
|
+
rubric: rubric
|
|
704
|
+
? {
|
|
705
|
+
rubricId: rubric.rubricId || null,
|
|
706
|
+
weightedScore: rubric.weightedScore,
|
|
707
|
+
failingCriteria: rubric.failingCriteria || [],
|
|
708
|
+
failingGuardrails: rubric.failingGuardrails || [],
|
|
709
|
+
judgeDisagreements: rubric.judgeDisagreements || [],
|
|
710
|
+
}
|
|
711
|
+
: null,
|
|
712
|
+
targetRisk,
|
|
713
|
+
riskLabel: targetRisk === 1 ? 'high-risk' : 'low-risk',
|
|
714
|
+
features,
|
|
715
|
+
label: feedbackEvent.signal === 'positive' ? 'positive' : 'negative',
|
|
716
|
+
};
|
|
717
|
+
appendJSONL(sequencePath, entry);
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
function updateDiversityTracking(feedbackEvent, paths) {
|
|
721
|
+
const diversityPath = path.join(paths.FEEDBACK_DIR, 'diversity-tracking.json');
|
|
722
|
+
let diversity = { domains: {}, lastUpdated: null, diversityScore: 0 };
|
|
723
|
+
if (fs.existsSync(diversityPath)) {
|
|
724
|
+
try {
|
|
725
|
+
diversity = JSON.parse(fs.readFileSync(diversityPath, 'utf-8'));
|
|
726
|
+
} catch {
|
|
727
|
+
// start fresh on parse error
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
const domain = inferDomain(feedbackEvent.tags, feedbackEvent.context);
|
|
732
|
+
if (!diversity.domains[domain]) {
|
|
733
|
+
diversity.domains[domain] = { count: 0, positive: 0, negative: 0, lastSeen: null };
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
diversity.domains[domain].count++;
|
|
737
|
+
diversity.domains[domain].lastSeen = feedbackEvent.timestamp;
|
|
738
|
+
if (feedbackEvent.signal === 'positive') diversity.domains[domain].positive++;
|
|
739
|
+
else diversity.domains[domain].negative++;
|
|
740
|
+
|
|
741
|
+
const totalFeedback = Object.values(diversity.domains).reduce((s, d) => s + d.count, 0);
|
|
742
|
+
const domainCount = Object.keys(diversity.domains).length;
|
|
743
|
+
const idealPerDomain = totalFeedback / DOMAIN_CATEGORIES.length;
|
|
744
|
+
const variance = Object.values(diversity.domains).reduce((s, d) => {
|
|
745
|
+
return s + Math.pow(d.count - idealPerDomain, 2);
|
|
746
|
+
}, 0) / Math.max(domainCount, 1);
|
|
747
|
+
|
|
748
|
+
diversity.diversityScore = Math.max(0, 100 - Math.sqrt(variance) * 10).toFixed(1);
|
|
749
|
+
diversity.lastUpdated = new Date().toISOString();
|
|
750
|
+
diversity.recommendation = Number(diversity.diversityScore) < 50
|
|
751
|
+
? `Low diversity (${diversity.diversityScore}%). Try feedback in: ${DOMAIN_CATEGORIES.filter((d) => !diversity.domains[d]).join(', ')}`
|
|
752
|
+
: `Good diversity (${diversity.diversityScore}%)`;
|
|
753
|
+
|
|
754
|
+
fs.writeFileSync(diversityPath, JSON.stringify(diversity, null, 2) + '\n');
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
function extractAndSetConstraints(context) {
|
|
758
|
+
if (!context) return;
|
|
759
|
+
try {
|
|
760
|
+
const { setConstraint } = require('./gates-engine');
|
|
761
|
+
const lower = context.toLowerCase();
|
|
762
|
+
|
|
763
|
+
// Extraction heuristics
|
|
764
|
+
if (lower.includes('local only') || lower.includes('not in git') || lower.includes("don't push") || lower.includes("no push")) {
|
|
765
|
+
setConstraint('local_only', true);
|
|
766
|
+
}
|
|
767
|
+
} catch (err) {
|
|
768
|
+
// Non-critical if gates engine not loaded
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
function inferSemanticTags(context = '') {
|
|
773
|
+
const lower = context.toLowerCase();
|
|
774
|
+
const tags = new Set();
|
|
775
|
+
|
|
776
|
+
if (lower.includes('revenue') || lower.includes('paid') || lower.includes('dollar') || lower.includes('cent') || lower.includes('price')) {
|
|
777
|
+
tags.add('entity:Revenue');
|
|
778
|
+
}
|
|
779
|
+
if (lower.includes('customer') || lower.includes('user') || lower.includes('pro') || lower.includes('tier')) {
|
|
780
|
+
tags.add('entity:Customer');
|
|
781
|
+
}
|
|
782
|
+
if (lower.includes('funnel') || lower.includes('conversion') || lower.includes('visitor') || lower.includes('checkout') || lower.includes('lead')) {
|
|
783
|
+
tags.add('entity:Funnel');
|
|
784
|
+
}
|
|
785
|
+
if (lower.includes('roi') || lower.includes('campaign') || lower.includes('attribution')) {
|
|
786
|
+
tags.add('metric:ROI');
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
return Array.from(tags);
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
function inferLessonFromConversation(conversationWindow, signal) {
|
|
793
|
+
const normalizedWindow = normalizeConversationWindow(conversationWindow);
|
|
794
|
+
if (normalizedWindow.length === 0) return null;
|
|
795
|
+
|
|
796
|
+
const userMessages = normalizedWindow.filter(m => m.role === 'user');
|
|
797
|
+
const assistantMessages = normalizedWindow.filter(m => m.role === 'assistant');
|
|
798
|
+
|
|
799
|
+
const lastUserMsg = userMessages[userMessages.length - 1]?.content || '';
|
|
800
|
+
const lastAssistantMsg = assistantMessages[assistantMessages.length - 1]?.content || '';
|
|
801
|
+
|
|
802
|
+
const userIntent = lastUserMsg.slice(0, 200);
|
|
803
|
+
const assistantAction = lastAssistantMsg.slice(0, 200);
|
|
804
|
+
|
|
805
|
+
const lesson = signal === 'negative'
|
|
806
|
+
? `User asked: "${userIntent}" → Assistant did: "${assistantAction}" → User rejected this`
|
|
807
|
+
: `User asked: "${userIntent}" → Assistant did: "${assistantAction}" → User approved this`;
|
|
808
|
+
|
|
809
|
+
const filePaths = extractConversationPaths(normalizedWindow);
|
|
810
|
+
const errorPatterns = extractConversationErrors(normalizedWindow);
|
|
811
|
+
|
|
812
|
+
const tags = [];
|
|
813
|
+
if (filePaths.length > 0) tags.push('has-file-context');
|
|
814
|
+
if (errorPatterns.length > 0) tags.push('has-error-context');
|
|
815
|
+
|
|
816
|
+
return {
|
|
817
|
+
lesson,
|
|
818
|
+
whatWentWrong: signal === 'negative' ? `Assistant response to "${userIntent.slice(0, 60)}..." was rejected` : null,
|
|
819
|
+
whatWorked: signal === 'positive' ? `Assistant response to "${userIntent.slice(0, 60)}..." was approved` : null,
|
|
820
|
+
tags,
|
|
821
|
+
filePaths,
|
|
822
|
+
errorPatterns,
|
|
823
|
+
messageCount: normalizedWindow.length,
|
|
824
|
+
};
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
function captureFeedback(params) {
|
|
828
|
+
const _captureStart = Date.now();
|
|
829
|
+
const { FEEDBACK_LOG_PATH, MEMORY_LOG_PATH, FEEDBACK_DIR } = getFeedbackPaths();
|
|
830
|
+
const signal = normalizeSignal(params.signal);
|
|
831
|
+
if (!signal) {
|
|
832
|
+
return {
|
|
833
|
+
accepted: false,
|
|
834
|
+
reason: `Invalid signal "${params.signal}". Use up/down or positive/negative.`,
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
const submittedContext = params.context || '';
|
|
839
|
+
const distillation = distillFeedbackHistory({
|
|
840
|
+
signal,
|
|
841
|
+
context: submittedContext,
|
|
842
|
+
whatWentWrong: params.whatWentWrong,
|
|
843
|
+
whatToChange: params.whatToChange,
|
|
844
|
+
whatWorked: params.whatWorked,
|
|
845
|
+
relatedFeedbackId: params.relatedFeedbackId,
|
|
846
|
+
chatHistory: params.chatHistory || params.messages,
|
|
847
|
+
allowLocalConversationFallback: params.allowLocalConversationFallback === true,
|
|
848
|
+
lastAction: params.lastAction,
|
|
849
|
+
feedbackDir: FEEDBACK_DIR,
|
|
850
|
+
});
|
|
851
|
+
|
|
852
|
+
const shouldUseDistilledContext = !submittedContext || isGenericFeedbackText(submittedContext, signal);
|
|
853
|
+
const context = shouldUseDistilledContext && distillation.inferredFields.context
|
|
854
|
+
? distillation.inferredFields.context
|
|
855
|
+
: submittedContext;
|
|
856
|
+
const whatWentWrong = params.whatWentWrong || distillation.inferredFields.whatWentWrong || null;
|
|
857
|
+
const whatToChange = params.whatToChange || distillation.inferredFields.whatToChange || null;
|
|
858
|
+
const whatWorked = params.whatWorked || distillation.inferredFields.whatWorked || null;
|
|
859
|
+
extractAndSetConstraints(context);
|
|
860
|
+
|
|
861
|
+
const providedTags = Array.isArray(params.tags)
|
|
862
|
+
? params.tags
|
|
863
|
+
: String(params.tags || '')
|
|
864
|
+
.split(',')
|
|
865
|
+
.map((t) => t.trim())
|
|
866
|
+
.filter(Boolean);
|
|
867
|
+
|
|
868
|
+
const semanticTags = inferSemanticTags(context);
|
|
869
|
+
const tags = Array.from(new Set([...providedTags, ...semanticTags]));
|
|
870
|
+
|
|
871
|
+
// Infer lesson from conversation window if provided
|
|
872
|
+
let inferredContext = context;
|
|
873
|
+
if (Array.isArray(params.conversationWindow) && params.conversationWindow.length > 0) {
|
|
874
|
+
const windowSummary = inferLessonFromConversation(params.conversationWindow, signal);
|
|
875
|
+
if (windowSummary) {
|
|
876
|
+
inferredContext = windowSummary.lesson;
|
|
877
|
+
if (windowSummary.tags) {
|
|
878
|
+
tags.push(...windowSummary.tags.filter(t => !tags.includes(t)));
|
|
879
|
+
}
|
|
880
|
+
if (!params.whatWentWrong && windowSummary.whatWentWrong) {
|
|
881
|
+
params.whatWentWrong = windowSummary.whatWentWrong;
|
|
882
|
+
}
|
|
883
|
+
if (!params.whatWorked && windowSummary.whatWorked) {
|
|
884
|
+
params.whatWorked = windowSummary.whatWorked;
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
// Infer structured IF/THEN rule from conversation
|
|
890
|
+
let structuredRule = null;
|
|
891
|
+
if (Array.isArray(params.conversationWindow) && params.conversationWindow.length >= 2) {
|
|
892
|
+
try {
|
|
893
|
+
const { inferStructuredLesson } = require('./lesson-inference');
|
|
894
|
+
structuredRule = inferStructuredLesson(params.conversationWindow, signal, inferredContext);
|
|
895
|
+
} catch (_err) { /* non-critical */ }
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
// Reflector agent: auto-propose rules on negative feedback
|
|
899
|
+
let reflection = null;
|
|
900
|
+
if (signal === 'negative' && Array.isArray(params.conversationWindow) && params.conversationWindow.length >= 2) {
|
|
901
|
+
try {
|
|
902
|
+
const { reflect } = require('./reflector-agent');
|
|
903
|
+
reflection = reflect({
|
|
904
|
+
conversationWindow: params.conversationWindow,
|
|
905
|
+
context: inferredContext,
|
|
906
|
+
whatWentWrong: params.whatWentWrong,
|
|
907
|
+
structuredRule,
|
|
908
|
+
feedbackEvent: null, // not yet constructed
|
|
909
|
+
});
|
|
910
|
+
} catch (_err) { /* non-critical */ }
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
let rubricEvaluation = null;
|
|
914
|
+
try {
|
|
915
|
+
if (params.rubricScores != null || params.guardrails != null) {
|
|
916
|
+
rubricEvaluation = buildRubricEvaluation({
|
|
917
|
+
rubricScores: params.rubricScores,
|
|
918
|
+
guardrails: parseOptionalObject(params.guardrails, 'guardrails'),
|
|
919
|
+
});
|
|
920
|
+
}
|
|
921
|
+
} catch (err) {
|
|
922
|
+
return {
|
|
923
|
+
accepted: false,
|
|
924
|
+
reason: `Invalid rubric payload: ${err.message}`,
|
|
925
|
+
};
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
const action = resolveFeedbackAction({
|
|
929
|
+
signal,
|
|
930
|
+
context,
|
|
931
|
+
whatWentWrong,
|
|
932
|
+
whatToChange,
|
|
933
|
+
whatWorked,
|
|
934
|
+
reasoning: params.reasoning,
|
|
935
|
+
visualEvidence: params.visualEvidence,
|
|
936
|
+
tags,
|
|
937
|
+
rubricEvaluation,
|
|
938
|
+
});
|
|
939
|
+
|
|
940
|
+
// Tool-call attribution: link feedback to specific action (#203)
|
|
941
|
+
const lastAction = params.lastAction
|
|
942
|
+
? {
|
|
943
|
+
tool: params.lastAction.tool || 'unknown',
|
|
944
|
+
contextKey: params.lastAction.contextKey || null,
|
|
945
|
+
file: params.lastAction.file || null,
|
|
946
|
+
timestamp: params.lastAction.timestamp || null,
|
|
947
|
+
}
|
|
948
|
+
: null;
|
|
949
|
+
|
|
950
|
+
const now = new Date().toISOString();
|
|
951
|
+
const rawFeedbackEvent = {
|
|
952
|
+
id: `fb_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
|
953
|
+
signal,
|
|
954
|
+
context,
|
|
955
|
+
submittedContext,
|
|
956
|
+
relatedFeedbackId: params.relatedFeedbackId || null,
|
|
957
|
+
lastAction,
|
|
958
|
+
whatWentWrong,
|
|
959
|
+
whatToChange,
|
|
960
|
+
whatWorked,
|
|
961
|
+
reasoning: params.reasoning || null,
|
|
962
|
+
visualEvidence: params.visualEvidence || null,
|
|
963
|
+
conversationWindow: Array.isArray(distillation.conversationWindow) && distillation.conversationWindow.length > 0 ? distillation.conversationWindow : null,
|
|
964
|
+
distillation: distillation.usedHistory
|
|
965
|
+
? {
|
|
966
|
+
source: distillation.source,
|
|
967
|
+
relatedFeedbackId: distillation.relatedFeedbackId,
|
|
968
|
+
evidence: distillation.evidence,
|
|
969
|
+
lessonProposal: distillation.lessonProposal,
|
|
970
|
+
}
|
|
971
|
+
: null,
|
|
972
|
+
tags,
|
|
973
|
+
skill: params.skill || null,
|
|
974
|
+
failureType: params.failureType || null,
|
|
975
|
+
rubric: rubricEvaluation
|
|
976
|
+
? {
|
|
977
|
+
rubricId: rubricEvaluation.rubricId,
|
|
978
|
+
weightedScore: rubricEvaluation.weightedScore,
|
|
979
|
+
failingCriteria: rubricEvaluation.failingCriteria,
|
|
980
|
+
failingGuardrails: rubricEvaluation.failingGuardrails,
|
|
981
|
+
judgeDisagreements: rubricEvaluation.judgeDisagreements,
|
|
982
|
+
promotionEligible: rubricEvaluation.promotionEligible,
|
|
983
|
+
}
|
|
984
|
+
: null,
|
|
985
|
+
actionType: action.type,
|
|
986
|
+
actionReason: action.reason || null,
|
|
987
|
+
conversationWindow: Array.isArray(params.conversationWindow) && params.conversationWindow.length > 0
|
|
988
|
+
? params.conversationWindow.slice(-10).map(m => ({
|
|
989
|
+
role: m.role,
|
|
990
|
+
content: (m.content || '').slice(0, 500),
|
|
991
|
+
timestamp: m.timestamp || null,
|
|
992
|
+
}))
|
|
993
|
+
: (Array.isArray(distillation.conversationWindow) && distillation.conversationWindow.length > 0
|
|
994
|
+
? distillation.conversationWindow
|
|
995
|
+
: null),
|
|
996
|
+
structuredRule: structuredRule || null,
|
|
997
|
+
...(reflection && { reflection }),
|
|
998
|
+
timestamp: now,
|
|
999
|
+
};
|
|
1000
|
+
|
|
1001
|
+
// Rich context enrichment (QUAL-02, QUAL-03) — non-blocking
|
|
1002
|
+
let feedbackEvent = enrichFeedbackContext(rawFeedbackEvent, params);
|
|
1003
|
+
const shouldDiagnose = signal === 'negative'
|
|
1004
|
+
|| (rubricEvaluation && (
|
|
1005
|
+
(rubricEvaluation.failingCriteria || []).length > 0
|
|
1006
|
+
|| (rubricEvaluation.failingGuardrails || []).length > 0
|
|
1007
|
+
))
|
|
1008
|
+
|| (typeof rawFeedbackEvent.actionReason === 'string' && /rubric gate/i.test(rawFeedbackEvent.actionReason));
|
|
1009
|
+
const diagnosis = shouldDiagnose
|
|
1010
|
+
? diagnoseFailure({
|
|
1011
|
+
step: 'feedback_capture',
|
|
1012
|
+
context,
|
|
1013
|
+
rubricEvaluation,
|
|
1014
|
+
feedbackEvent,
|
|
1015
|
+
suspect: signal === 'negative' || action.type === 'no-action',
|
|
1016
|
+
})
|
|
1017
|
+
: null;
|
|
1018
|
+
const storedDiagnosis = toStoredDiagnosis(diagnosis);
|
|
1019
|
+
if (storedDiagnosis) {
|
|
1020
|
+
feedbackEvent = {
|
|
1021
|
+
...feedbackEvent,
|
|
1022
|
+
diagnosis: storedDiagnosis,
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
const historyEntries = readJSONL(FEEDBACK_LOG_PATH).slice(-SEQUENCE_WINDOW);
|
|
1026
|
+
|
|
1027
|
+
const summary = loadSummary();
|
|
1028
|
+
summary.total += 1;
|
|
1029
|
+
summary[signal] += 1;
|
|
1030
|
+
|
|
1031
|
+
if (action.type === 'no-action') {
|
|
1032
|
+
const firewallBlocked = maybeBlockMemoryIngress({ feedbackEvent, summary, now });
|
|
1033
|
+
if (firewallBlocked) {
|
|
1034
|
+
return firewallBlocked;
|
|
1035
|
+
}
|
|
1036
|
+
const clarification = buildClarificationMessage({
|
|
1037
|
+
signal,
|
|
1038
|
+
context,
|
|
1039
|
+
whatWentWrong,
|
|
1040
|
+
whatToChange,
|
|
1041
|
+
whatWorked,
|
|
1042
|
+
});
|
|
1043
|
+
summary.rejected += 1;
|
|
1044
|
+
summary.lastUpdated = now;
|
|
1045
|
+
saveSummary(summary);
|
|
1046
|
+
appendJSONL(FEEDBACK_LOG_PATH, feedbackEvent);
|
|
1047
|
+
try { appendRejectionLedger(feedbackEvent, action.reason); } catch { /* non-critical */ }
|
|
1048
|
+
try {
|
|
1049
|
+
appendSequence(historyEntries, feedbackEvent, getFeedbackPaths(), { accepted: false });
|
|
1050
|
+
} catch { /* non-critical */ }
|
|
1051
|
+
try {
|
|
1052
|
+
const riskScorer = getRiskScorerModule();
|
|
1053
|
+
if (riskScorer) riskScorer.trainAndPersistRiskModel(FEEDBACK_DIR);
|
|
1054
|
+
} catch { /* non-critical */ }
|
|
1055
|
+
updateStatuslineWithLesson({
|
|
1056
|
+
accepted: false,
|
|
1057
|
+
signal,
|
|
1058
|
+
feedbackId: feedbackEvent.id,
|
|
1059
|
+
});
|
|
1060
|
+
return {
|
|
1061
|
+
accepted: false,
|
|
1062
|
+
signalLogged: true,
|
|
1063
|
+
status: clarification ? 'clarification_required' : 'rejected',
|
|
1064
|
+
reason: action.reason,
|
|
1065
|
+
message: clarification ? clarification.message : 'Signal logged, but reusable memory was not created.',
|
|
1066
|
+
feedbackEvent,
|
|
1067
|
+
...(clarification || {}),
|
|
1068
|
+
};
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
const prepared = prepareForStorage(action.memory);
|
|
1072
|
+
if (!prepared.ok) {
|
|
1073
|
+
const firewallBlocked = maybeBlockMemoryIngress({ feedbackEvent, summary, now });
|
|
1074
|
+
if (firewallBlocked) {
|
|
1075
|
+
return firewallBlocked;
|
|
1076
|
+
}
|
|
1077
|
+
summary.rejected += 1;
|
|
1078
|
+
summary.lastUpdated = now;
|
|
1079
|
+
saveSummary(summary);
|
|
1080
|
+
appendJSONL(FEEDBACK_LOG_PATH, {
|
|
1081
|
+
...feedbackEvent,
|
|
1082
|
+
validationIssues: prepared.issues,
|
|
1083
|
+
});
|
|
1084
|
+
try { appendRejectionLedger(feedbackEvent, `Schema validation failed: ${prepared.issues.join('; ')}`); } catch { /* non-critical */ }
|
|
1085
|
+
try {
|
|
1086
|
+
appendSequence(historyEntries, feedbackEvent, getFeedbackPaths(), { accepted: false });
|
|
1087
|
+
} catch { /* non-critical */ }
|
|
1088
|
+
try {
|
|
1089
|
+
const riskScorer = getRiskScorerModule();
|
|
1090
|
+
if (riskScorer) riskScorer.trainAndPersistRiskModel(FEEDBACK_DIR);
|
|
1091
|
+
} catch { /* non-critical */ }
|
|
1092
|
+
return {
|
|
1093
|
+
accepted: false,
|
|
1094
|
+
signalLogged: true,
|
|
1095
|
+
status: 'rejected',
|
|
1096
|
+
reason: `Schema validation failed: ${prepared.issues.join('; ')}`,
|
|
1097
|
+
message: 'Signal logged, but reusable memory was not created.',
|
|
1098
|
+
feedbackEvent,
|
|
1099
|
+
issues: prepared.issues,
|
|
1100
|
+
};
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
const memoryRecord = {
|
|
1104
|
+
id: `mem_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
|
1105
|
+
...prepared.memory,
|
|
1106
|
+
richContext: feedbackEvent.richContext || null,
|
|
1107
|
+
distillation: feedbackEvent.distillation || null,
|
|
1108
|
+
diagnosis: storedDiagnosis,
|
|
1109
|
+
structuredRule: structuredRule || null,
|
|
1110
|
+
sourceFeedbackId: feedbackEvent.id,
|
|
1111
|
+
timestamp: now,
|
|
1112
|
+
};
|
|
1113
|
+
|
|
1114
|
+
// Bayesian Belief Update (Project Bayes)
|
|
1115
|
+
try {
|
|
1116
|
+
const { updateBelief, shouldPrune } = require('./belief-update');
|
|
1117
|
+
const existingMemories = readJSONL(MEMORY_LOG_PATH);
|
|
1118
|
+
const similarMemory = existingMemories.slice().reverse().find(m =>
|
|
1119
|
+
m.tags && m.tags.some(t => memoryRecord.tags.includes(t) && !GENERIC_TAGS.has(t))
|
|
1120
|
+
);
|
|
1121
|
+
|
|
1122
|
+
if (similarMemory && similarMemory.bayesian) {
|
|
1123
|
+
const likelihood = signal === 'positive' ? 0.9 : 0.1;
|
|
1124
|
+
memoryRecord.bayesian = updateBelief(similarMemory.bayesian, likelihood);
|
|
1125
|
+
memoryRecord.revisedFromId = similarMemory.id;
|
|
1126
|
+
|
|
1127
|
+
if (shouldPrune(memoryRecord.bayesian)) {
|
|
1128
|
+
memoryRecord.pruned = true;
|
|
1129
|
+
memoryRecord.pruneReason = 'high_entropy_contradiction';
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
} catch (_err) { /* bayesian update is non-blocking */ }
|
|
1133
|
+
|
|
1134
|
+
const firewallBlocked = maybeBlockMemoryIngress({
|
|
1135
|
+
feedbackEvent,
|
|
1136
|
+
memoryRecord,
|
|
1137
|
+
summary,
|
|
1138
|
+
now,
|
|
1139
|
+
});
|
|
1140
|
+
if (firewallBlocked) {
|
|
1141
|
+
return firewallBlocked;
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
appendJSONL(FEEDBACK_LOG_PATH, feedbackEvent);
|
|
1145
|
+
|
|
1146
|
+
// Synthesis: merge similar lessons instead of creating duplicates
|
|
1147
|
+
let synthesisResult = null;
|
|
1148
|
+
try {
|
|
1149
|
+
const { findSimilarLesson, mergeIntoExisting, shouldAutoPromote, synthesizePreventionRule, appendJSONLLocal } = require('./lesson-synthesis');
|
|
1150
|
+
const similar = findSimilarLesson(MEMORY_LOG_PATH, memoryRecord);
|
|
1151
|
+
|
|
1152
|
+
if (similar) {
|
|
1153
|
+
// Merge into existing lesson
|
|
1154
|
+
const merged = mergeIntoExisting(MEMORY_LOG_PATH, similar.match, memoryRecord, feedbackEvent);
|
|
1155
|
+
synthesisResult = { action: 'merged', existingId: similar.match.id, similarity: similar.similarity, occurrences: merged.occurrences };
|
|
1156
|
+
|
|
1157
|
+
// Auto-promote if threshold reached
|
|
1158
|
+
if (shouldAutoPromote(merged)) {
|
|
1159
|
+
const rule = synthesizePreventionRule(merged);
|
|
1160
|
+
synthesisResult.autoPromoted = true;
|
|
1161
|
+
synthesisResult.preventionRule = rule;
|
|
1162
|
+
// Store the synthesized rule
|
|
1163
|
+
const rulesPath = path.join(path.dirname(MEMORY_LOG_PATH), 'synthesized-rules.jsonl');
|
|
1164
|
+
appendJSONLLocal(rulesPath, rule);
|
|
1165
|
+
}
|
|
1166
|
+
} else {
|
|
1167
|
+
// No similar lesson — check exact duplicate, then store
|
|
1168
|
+
const duplicateMemory = findDuplicateMemory(MEMORY_LOG_PATH, memoryRecord);
|
|
1169
|
+
if (!duplicateMemory) {
|
|
1170
|
+
memoryRecord.occurrences = 1;
|
|
1171
|
+
appendJSONL(MEMORY_LOG_PATH, memoryRecord);
|
|
1172
|
+
}
|
|
1173
|
+
synthesisResult = { action: duplicateMemory ? 'exact-duplicate-skipped' : 'new-lesson' };
|
|
1174
|
+
}
|
|
1175
|
+
} catch (_synthErr) {
|
|
1176
|
+
// Fallback to original behavior
|
|
1177
|
+
const duplicateMemory = findDuplicateMemory(MEMORY_LOG_PATH, memoryRecord);
|
|
1178
|
+
if (!duplicateMemory) {
|
|
1179
|
+
appendJSONL(MEMORY_LOG_PATH, memoryRecord);
|
|
1180
|
+
}
|
|
1181
|
+
synthesisResult = { action: 'fallback', error: _synthErr.message };
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// Dual-write to SQLite lesson DB — deferred to avoid blocking response
|
|
1185
|
+
let correctiveActions = [];
|
|
1186
|
+
try {
|
|
1187
|
+
const lessonDB = getLessonDB();
|
|
1188
|
+
if (lessonDB) {
|
|
1189
|
+
const { upsertLesson, inferCorrectiveActions } = require('./lesson-db');
|
|
1190
|
+
upsertLesson(lessonDB, feedbackEvent, memoryRecord);
|
|
1191
|
+
if (feedbackEvent.signal === 'negative') {
|
|
1192
|
+
correctiveActions = inferCorrectiveActions(lessonDB, feedbackEvent, 3);
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
} catch (_err) {
|
|
1196
|
+
// Lesson DB write is non-critical — never fail the capture pipeline
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
summary.accepted += 1;
|
|
1200
|
+
summary.lastUpdated = now;
|
|
1201
|
+
saveSummary(summary);
|
|
1202
|
+
|
|
1203
|
+
const _captureMs = Date.now() - _captureStart;
|
|
1204
|
+
|
|
1205
|
+
// Auto-open feedback session for follow-up capture
|
|
1206
|
+
let feedbackSession = null;
|
|
1207
|
+
try {
|
|
1208
|
+
const { openSession } = require('./feedback-session');
|
|
1209
|
+
feedbackSession = openSession(feedbackEvent.id, signal, inferredContext);
|
|
1210
|
+
} catch (_err) { /* non-critical */ }
|
|
1211
|
+
|
|
1212
|
+
// Build result immediately — all remaining side-effects are deferred
|
|
1213
|
+
const result = {
|
|
1214
|
+
accepted: true,
|
|
1215
|
+
status: 'promoted',
|
|
1216
|
+
message: 'Feedback promoted to reusable memory.',
|
|
1217
|
+
feedbackEvent,
|
|
1218
|
+
memoryRecord,
|
|
1219
|
+
_captureMs,
|
|
1220
|
+
...(correctiveActions.length > 0 && { correctiveActions }),
|
|
1221
|
+
...(reflection && { reflection }),
|
|
1222
|
+
...(feedbackSession && { feedbackSession }),
|
|
1223
|
+
...(synthesisResult && { synthesis: synthesisResult }),
|
|
1224
|
+
};
|
|
1225
|
+
|
|
1226
|
+
// Update statusline with lesson info (include proposed rule if reflection available)
|
|
1227
|
+
updateStatuslineWithLesson({
|
|
1228
|
+
accepted: true,
|
|
1229
|
+
signal,
|
|
1230
|
+
memoryId: memoryRecord.id,
|
|
1231
|
+
feedbackId: feedbackEvent.id,
|
|
1232
|
+
lesson: reflection?.proposedRule?.rule
|
|
1233
|
+
? `${inferredContext || context} | Rule: ${reflection.proposedRule.rule}`
|
|
1234
|
+
: (inferredContext || context),
|
|
1235
|
+
turnCount: Array.isArray(params.conversationWindow) ? params.conversationWindow.length : 0,
|
|
1236
|
+
});
|
|
1237
|
+
|
|
1238
|
+
// --- Synchronous side-effects (fast, needed by analyzeFeedback) ---
|
|
1239
|
+
const mlPaths = getFeedbackPaths();
|
|
1240
|
+
try {
|
|
1241
|
+
appendSequence(historyEntries, feedbackEvent, mlPaths, { accepted: true });
|
|
1242
|
+
} catch { /* Sequence tracking failure is non-critical */ }
|
|
1243
|
+
try {
|
|
1244
|
+
updateDiversityTracking(feedbackEvent, mlPaths);
|
|
1245
|
+
} catch { /* Diversity tracking failure is non-critical */ }
|
|
1246
|
+
try {
|
|
1247
|
+
const riskScorer = getRiskScorerModule();
|
|
1248
|
+
if (riskScorer) riskScorer.trainAndPersistRiskModel(FEEDBACK_DIR);
|
|
1249
|
+
} catch { /* non-critical */ }
|
|
1250
|
+
try {
|
|
1251
|
+
const toolName = feedbackEvent.toolName || feedbackEvent.tool_name || 'unknown';
|
|
1252
|
+
const toolInput = feedbackEvent.context || feedbackEvent.input || '';
|
|
1253
|
+
recordAction(toolName, toolInput);
|
|
1254
|
+
if (feedbackEvent.signal === 'negative') {
|
|
1255
|
+
attributeFeedback('negative', feedbackEvent.context || '');
|
|
1256
|
+
} else if (feedbackEvent.signal === 'positive') {
|
|
1257
|
+
attributeFeedback('positive', feedbackEvent.context || '');
|
|
1258
|
+
}
|
|
1259
|
+
} catch { /* attribution is non-blocking */ }
|
|
1260
|
+
|
|
1261
|
+
// Vector storage — track promise synchronously so waitForBackgroundSideEffects works
|
|
1262
|
+
const vectorStore = getVectorStoreModule();
|
|
1263
|
+
if (vectorStore && typeof vectorStore.upsertFeedback === 'function') {
|
|
1264
|
+
trackBackgroundSideEffect(vectorStore.upsertFeedback(feedbackEvent));
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
// Auto-promote gates on negative feedback (sync — tests depend on immediate promotion)
|
|
1268
|
+
if (feedbackEvent.signal === 'negative') {
|
|
1269
|
+
try {
|
|
1270
|
+
const autoPromote = require('./auto-promote-gates');
|
|
1271
|
+
autoPromote.promote(FEEDBACK_LOG_PATH);
|
|
1272
|
+
} catch { /* Gate promotion is non-critical */ }
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
// --- Deferred side-effects (contextFs, RLAIF — non-critical, potentially slow) ---
|
|
1276
|
+
setImmediate(() => {
|
|
1277
|
+
try {
|
|
1278
|
+
const contextFs = getContextFsModule();
|
|
1279
|
+
if (contextFs && typeof contextFs.registerFeedback === 'function') {
|
|
1280
|
+
contextFs.registerFeedback(feedbackEvent, memoryRecord);
|
|
1281
|
+
}
|
|
1282
|
+
} catch { /* Non-critical */ }
|
|
1283
|
+
|
|
1284
|
+
try {
|
|
1285
|
+
const sam = getSelfAuditModule();
|
|
1286
|
+
if (sam) sam.selfAuditAndLog(feedbackEvent, mlPaths);
|
|
1287
|
+
} catch { /* non-critical */ }
|
|
1288
|
+
|
|
1289
|
+
// Auto-create lesson for statusbar display
|
|
1290
|
+
try {
|
|
1291
|
+
const { createLesson } = require('./lesson-inference');
|
|
1292
|
+
createLesson({
|
|
1293
|
+
feedbackId: feedbackEvent.id,
|
|
1294
|
+
signal: feedbackEvent.signal,
|
|
1295
|
+
inferredLesson: memoryRecord ? memoryRecord.title : (feedbackEvent.context || '').slice(0, 200),
|
|
1296
|
+
confidence: memoryRecord ? 70 : 40,
|
|
1297
|
+
tags: feedbackEvent.tags || [],
|
|
1298
|
+
});
|
|
1299
|
+
} catch { /* non-critical — lesson creation should never block feedback */ }
|
|
1300
|
+
});
|
|
1301
|
+
|
|
1302
|
+
return result;
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
function analyzeFeedback(logPath) {
|
|
1306
|
+
const { FEEDBACK_LOG_PATH } = getFeedbackPaths();
|
|
1307
|
+
const resolvedLogPath = logPath || FEEDBACK_LOG_PATH;
|
|
1308
|
+
const feedbackDir = path.dirname(resolvedLogPath);
|
|
1309
|
+
const paths = buildFeedbackPathsFromDir(feedbackDir);
|
|
1310
|
+
const shouldUseSQLite = !logPath || path.resolve(resolvedLogPath) === path.resolve(FEEDBACK_LOG_PATH);
|
|
1311
|
+
const entries = readJSONL(resolvedLogPath, { maxLines: 0 });
|
|
1312
|
+
const diagnosticLogPath = path.join(feedbackDir, 'diagnostic-log.jsonl');
|
|
1313
|
+
const diagnosticEntries = readDiagnosticEntries(diagnosticLogPath);
|
|
1314
|
+
|
|
1315
|
+
// Prefer the JSONL mirror for full analytics fidelity. Fall back to SQLite only
|
|
1316
|
+
// when the mirror is unavailable so dashboards and proof paths keep their full shape.
|
|
1317
|
+
const db = shouldUseSQLite ? getLessonDB() : null;
|
|
1318
|
+
if (db && entries.length === 0) {
|
|
1319
|
+
try {
|
|
1320
|
+
const { getStatsFromDB } = require('./lesson-db');
|
|
1321
|
+
const sqliteStats = getStatsFromDB(db);
|
|
1322
|
+
if (sqliteStats.total > 0) return normalizeAnalysisShape(sqliteStats);
|
|
1323
|
+
} catch { /* fall through to JSONL scan */ }
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
const skills = {};
|
|
1327
|
+
const tags = {};
|
|
1328
|
+
const rubricCriteria = {};
|
|
1329
|
+
let rubricSamples = 0;
|
|
1330
|
+
let blockedPromotions = 0;
|
|
1331
|
+
|
|
1332
|
+
let totalPositive = 0;
|
|
1333
|
+
let totalNegative = 0;
|
|
1334
|
+
|
|
1335
|
+
for (const entry of entries) {
|
|
1336
|
+
if (entry.signal === 'positive') totalPositive++;
|
|
1337
|
+
if (entry.signal === 'negative') totalNegative++;
|
|
1338
|
+
|
|
1339
|
+
if (entry.skill) {
|
|
1340
|
+
if (!skills[entry.skill]) skills[entry.skill] = { positive: 0, negative: 0, total: 0 };
|
|
1341
|
+
skills[entry.skill][entry.signal] += 1;
|
|
1342
|
+
skills[entry.skill].total += 1;
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
for (const tag of entry.tags || []) {
|
|
1346
|
+
if (!tags[tag]) tags[tag] = { positive: 0, negative: 0, total: 0 };
|
|
1347
|
+
tags[tag][entry.signal] += 1;
|
|
1348
|
+
tags[tag].total += 1;
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
if (entry.actionType === 'no-action' && typeof entry.actionReason === 'string' && entry.actionReason.includes('Rubric gate')) {
|
|
1352
|
+
blockedPromotions += 1;
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
if (entry.rubric && entry.rubric.weightedScore != null) {
|
|
1356
|
+
rubricSamples += 1;
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
if (entry.rubric && Array.isArray(entry.rubric.failingCriteria)) {
|
|
1360
|
+
for (const criterion of entry.rubric.failingCriteria) {
|
|
1361
|
+
if (!rubricCriteria[criterion]) rubricCriteria[criterion] = { failures: 0 };
|
|
1362
|
+
rubricCriteria[criterion].failures += 1;
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
const total = totalPositive + totalNegative;
|
|
1368
|
+
const approvalRate = total > 0 ? Math.round((totalPositive / total) * 1000) / 1000 : 0;
|
|
1369
|
+
const recent = entries.slice(-20);
|
|
1370
|
+
const recentPos = recent.filter((e) => e.signal === 'positive').length;
|
|
1371
|
+
const recentRate = recent.length > 0 ? Math.round((recentPos / recent.length) * 1000) / 1000 : 0;
|
|
1372
|
+
|
|
1373
|
+
// Rolling windows: 7-day, 30-day, lifetime (#204)
|
|
1374
|
+
const now = Date.now();
|
|
1375
|
+
const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
|
|
1376
|
+
const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
|
|
1377
|
+
const windowStats = { '7d': { total: 0, positive: 0 }, '30d': { total: 0, positive: 0 } };
|
|
1378
|
+
for (const entry of entries) {
|
|
1379
|
+
const ts = entry.timestamp ? new Date(entry.timestamp).getTime() : 0;
|
|
1380
|
+
const age = now - ts;
|
|
1381
|
+
if (age <= SEVEN_DAYS_MS) {
|
|
1382
|
+
windowStats['7d'].total++;
|
|
1383
|
+
if (entry.signal === 'positive') windowStats['7d'].positive++;
|
|
1384
|
+
}
|
|
1385
|
+
if (age <= THIRTY_DAYS_MS) {
|
|
1386
|
+
windowStats['30d'].total++;
|
|
1387
|
+
if (entry.signal === 'positive') windowStats['30d'].positive++;
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
const rate7d = windowStats['7d'].total > 0
|
|
1391
|
+
? Math.round((windowStats['7d'].positive / windowStats['7d'].total) * 1000) / 1000 : 0;
|
|
1392
|
+
const rate30d = windowStats['30d'].total > 0
|
|
1393
|
+
? Math.round((windowStats['30d'].positive / windowStats['30d'].total) * 1000) / 1000 : 0;
|
|
1394
|
+
const TREND_THRESHOLD = 0.05;
|
|
1395
|
+
const hasTrendData = windowStats['7d'].total > 0 && windowStats['30d'].total > 0;
|
|
1396
|
+
const trend = !hasTrendData ? 'stable'
|
|
1397
|
+
: rate7d > rate30d + TREND_THRESHOLD ? 'improving'
|
|
1398
|
+
: rate7d < rate30d - TREND_THRESHOLD ? 'degrading' : 'stable';
|
|
1399
|
+
const windows = {
|
|
1400
|
+
'7d': { ...windowStats['7d'], rate: rate7d },
|
|
1401
|
+
'30d': { ...windowStats['30d'], rate: rate30d },
|
|
1402
|
+
lifetime: { total, positive: totalPositive, rate: approvalRate },
|
|
1403
|
+
};
|
|
1404
|
+
|
|
1405
|
+
const recommendations = [];
|
|
1406
|
+
|
|
1407
|
+
for (const [skill, stat] of Object.entries(skills)) {
|
|
1408
|
+
const negRate = stat.total > 0 ? stat.negative / stat.total : 0;
|
|
1409
|
+
if (stat.total >= 3 && negRate >= 0.5) {
|
|
1410
|
+
recommendations.push(`IMPROVE skill '${skill}' (${stat.negative}/${stat.total} negative)`);
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
for (const [tag, stat] of Object.entries(tags)) {
|
|
1415
|
+
const posRate = stat.total > 0 ? stat.positive / stat.total : 0;
|
|
1416
|
+
if (stat.total >= 3 && posRate >= 0.8) {
|
|
1417
|
+
recommendations.push(`REUSE pattern '${tag}' (${stat.positive}/${stat.total} positive)`);
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
if (recent.length >= 10 && recentRate < approvalRate - 0.1) {
|
|
1422
|
+
recommendations.push('DECLINING trend in last 20 signals; tighten verification before response.');
|
|
1423
|
+
}
|
|
1424
|
+
if (trend === 'degrading') {
|
|
1425
|
+
recommendations.push(`DEGRADING 7d trend (${rate7d}) vs 30d (${rate30d}); increase prevention rule injection.`);
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
let boostedRisk = null;
|
|
1429
|
+
try {
|
|
1430
|
+
const riskScorer = getRiskScorerModule();
|
|
1431
|
+
if (riskScorer) {
|
|
1432
|
+
boostedRisk = riskScorer.getRiskSummary(paths.FEEDBACK_DIR);
|
|
1433
|
+
if (boostedRisk) {
|
|
1434
|
+
boostedRisk.highRiskDomains.slice(0, 2).forEach((bucket) => {
|
|
1435
|
+
recommendations.push(`CHECK high-risk domain '${bucket.key}' (${bucket.highRisk}/${bucket.total} high-risk)`);
|
|
1436
|
+
});
|
|
1437
|
+
boostedRisk.highRiskTags.slice(0, 2).forEach((bucket) => {
|
|
1438
|
+
recommendations.push(`CHECK high-risk tag '${bucket.key}' (${bucket.highRisk}/${bucket.total} high-risk)`);
|
|
1439
|
+
});
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
} catch {
|
|
1443
|
+
boostedRisk = null;
|
|
1444
|
+
}
|
|
1445
|
+
const diagnostics = aggregateFailureDiagnostics([...entries, ...diagnosticEntries]);
|
|
1446
|
+
let delegation = null;
|
|
1447
|
+
try {
|
|
1448
|
+
const delegationRuntime = getDelegationRuntimeModule();
|
|
1449
|
+
if (delegationRuntime && typeof delegationRuntime.summarizeDelegation === 'function') {
|
|
1450
|
+
delegation = delegationRuntime.summarizeDelegation(paths.FEEDBACK_DIR);
|
|
1451
|
+
if (delegation.attemptCount >= 3 && delegation.verificationFailureRate >= 0.5) {
|
|
1452
|
+
recommendations.push(`REDUCE delegation: verification failure rate is ${Math.round(delegation.verificationFailureRate * 100)}%`);
|
|
1453
|
+
}
|
|
1454
|
+
if (delegation.avoidedDelegationCount >= 3) {
|
|
1455
|
+
recommendations.push(`REVIEW delegation policy: ${delegation.avoidedDelegationCount} handoff starts were blocked before execution`);
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
} catch {
|
|
1459
|
+
delegation = null;
|
|
1460
|
+
}
|
|
1461
|
+
diagnostics.categories.slice(0, 2).forEach((bucket) => {
|
|
1462
|
+
recommendations.push(`DIAGNOSE '${bucket.key}' failures (${bucket.count})`);
|
|
1463
|
+
});
|
|
1464
|
+
|
|
1465
|
+
return normalizeAnalysisShape({
|
|
1466
|
+
total,
|
|
1467
|
+
totalPositive,
|
|
1468
|
+
totalNegative,
|
|
1469
|
+
approvalRate,
|
|
1470
|
+
recentRate,
|
|
1471
|
+
windows,
|
|
1472
|
+
trend,
|
|
1473
|
+
skills,
|
|
1474
|
+
tags,
|
|
1475
|
+
rubric: {
|
|
1476
|
+
samples: rubricSamples,
|
|
1477
|
+
blockedPromotions,
|
|
1478
|
+
failingCriteria: rubricCriteria,
|
|
1479
|
+
},
|
|
1480
|
+
diagnostics,
|
|
1481
|
+
delegation,
|
|
1482
|
+
boostedRisk,
|
|
1483
|
+
recommendations,
|
|
1484
|
+
});
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
function buildPreventionRules(minOccurrences = 2, options = {}) {
|
|
1488
|
+
const resolvedMinOccurrences = Number.isFinite(minOccurrences)
|
|
1489
|
+
? minOccurrences
|
|
1490
|
+
: getEffectiveSetting('prevention_min_occurrences', 2);
|
|
1491
|
+
const { MEMORY_LOG_PATH, DIAGNOSTIC_LOG_PATH } = getFeedbackPaths();
|
|
1492
|
+
const memories = readJSONL(MEMORY_LOG_PATH).filter((m) => m.category === 'error');
|
|
1493
|
+
const diagnosticEntries = readDiagnosticEntries(DIAGNOSTIC_LOG_PATH);
|
|
1494
|
+
if (memories.length === 0) {
|
|
1495
|
+
if (diagnosticEntries.length === 0) {
|
|
1496
|
+
return '# Prevention Rules\n\nNo mistake memories recorded yet.';
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
// Time-weighted decay: recent mistakes count more (#202)
|
|
1501
|
+
const decayHalfLifeDays = options.decayHalfLifeDays || 7;
|
|
1502
|
+
const lambda = Math.LN2 / decayHalfLifeDays;
|
|
1503
|
+
const now = Date.now();
|
|
1504
|
+
|
|
1505
|
+
function decayWeight(memory) {
|
|
1506
|
+
const ts = memory.timestamp ? new Date(memory.timestamp).getTime() : now;
|
|
1507
|
+
const daysSince = (now - ts) / (24 * 60 * 60 * 1000);
|
|
1508
|
+
return Math.exp(-lambda * daysSince);
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
const buckets = {};
|
|
1512
|
+
const rubricBuckets = {};
|
|
1513
|
+
const diagnosisBuckets = {};
|
|
1514
|
+
const repeatedViolationBuckets = {};
|
|
1515
|
+
for (const m of memories) {
|
|
1516
|
+
const key = (m.richContext && m.richContext.domain && m.richContext.domain !== 'unknown')
|
|
1517
|
+
? m.richContext.domain
|
|
1518
|
+
: (m.tags || []).find((t) => !['feedback', 'negative', 'positive'].includes(t)) || 'general';
|
|
1519
|
+
if (!buckets[key]) buckets[key] = { items: [], weightedCount: 0 };
|
|
1520
|
+
const w = decayWeight(m);
|
|
1521
|
+
const occ = m.occurrences || 1;
|
|
1522
|
+
buckets[key].items.push(m);
|
|
1523
|
+
buckets[key].weightedCount += w * occ;
|
|
1524
|
+
|
|
1525
|
+
const failed = m.rubricSummary && Array.isArray(m.rubricSummary.failingCriteria)
|
|
1526
|
+
? m.rubricSummary.failingCriteria
|
|
1527
|
+
: [];
|
|
1528
|
+
failed.forEach((criterion) => {
|
|
1529
|
+
if (!rubricBuckets[criterion]) rubricBuckets[criterion] = [];
|
|
1530
|
+
for (let i = 0; i < occ; i++) rubricBuckets[criterion].push(m);
|
|
1531
|
+
});
|
|
1532
|
+
|
|
1533
|
+
if (m.diagnosis && m.diagnosis.rootCauseCategory) {
|
|
1534
|
+
if (!diagnosisBuckets[m.diagnosis.rootCauseCategory]) diagnosisBuckets[m.diagnosis.rootCauseCategory] = [];
|
|
1535
|
+
for (let i = 0; i < occ; i++) diagnosisBuckets[m.diagnosis.rootCauseCategory].push(m);
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
(m.diagnosis && Array.isArray(m.diagnosis.violations) ? m.diagnosis.violations : []).forEach((violation) => {
|
|
1539
|
+
const vKey = violation.constraintId || violation.message;
|
|
1540
|
+
if (!vKey) return;
|
|
1541
|
+
if (!repeatedViolationBuckets[vKey]) repeatedViolationBuckets[vKey] = [];
|
|
1542
|
+
for (let i = 0; i < occ; i++) repeatedViolationBuckets[vKey].push(m);
|
|
1543
|
+
});
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
for (const entry of diagnosticEntries) {
|
|
1547
|
+
const diagnosis = entry && entry.diagnosis ? entry.diagnosis : null;
|
|
1548
|
+
if (!diagnosis || !diagnosis.rootCauseCategory) continue;
|
|
1549
|
+
if (!diagnosisBuckets[diagnosis.rootCauseCategory]) diagnosisBuckets[diagnosis.rootCauseCategory] = [];
|
|
1550
|
+
diagnosisBuckets[diagnosis.rootCauseCategory].push(entry);
|
|
1551
|
+
|
|
1552
|
+
(Array.isArray(diagnosis.violations) ? diagnosis.violations : []).forEach((violation) => {
|
|
1553
|
+
const key = violation.constraintId || violation.message;
|
|
1554
|
+
if (!key) return;
|
|
1555
|
+
if (!repeatedViolationBuckets[key]) repeatedViolationBuckets[key] = [];
|
|
1556
|
+
repeatedViolationBuckets[key].push(entry);
|
|
1557
|
+
});
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
const lines = ['# Prevention Rules', '', 'Generated from negative feedback memories (time-weighted, half-life: ' + decayHalfLifeDays + 'd).'];
|
|
1561
|
+
|
|
1562
|
+
Object.entries(buckets)
|
|
1563
|
+
.sort((a, b) => b[1].weightedCount - a[1].weightedCount)
|
|
1564
|
+
.forEach(([domain, { items, weightedCount }]) => {
|
|
1565
|
+
const effectiveOccurrences = Math.round(weightedCount);
|
|
1566
|
+
if (effectiveOccurrences < resolvedMinOccurrences) return;
|
|
1567
|
+
const latest = items[items.length - 1];
|
|
1568
|
+
const avoid = (latest.content || '').split('\n').find((l) => l.toLowerCase().startsWith('how to avoid:')) || 'How to avoid: Investigate and prevent recurrence';
|
|
1569
|
+
lines.push('');
|
|
1570
|
+
lines.push(`## ${domain}`);
|
|
1571
|
+
lines.push(`- Recurrence count: ${items.length} (weighted: ${weightedCount.toFixed(1)})`);
|
|
1572
|
+
lines.push(`- Rule: ${avoid.replace(/^How to avoid:\s*/i, '')}`);
|
|
1573
|
+
lines.push(`- Latest mistake: ${latest.title}`);
|
|
1574
|
+
});
|
|
1575
|
+
|
|
1576
|
+
const rubricEntries = Object.entries(rubricBuckets)
|
|
1577
|
+
.sort((a, b) => b[1].length - a[1].length)
|
|
1578
|
+
.filter(([, items]) => items.length >= resolvedMinOccurrences);
|
|
1579
|
+
if (rubricEntries.length > 0) {
|
|
1580
|
+
lines.push('');
|
|
1581
|
+
lines.push('## Rubric Failure Dimensions');
|
|
1582
|
+
rubricEntries.forEach(([criterion, items]) => {
|
|
1583
|
+
lines.push(`- ${criterion}: ${items.length} failures`);
|
|
1584
|
+
});
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
const diagnosisEntries = Object.entries(diagnosisBuckets)
|
|
1588
|
+
.sort((a, b) => b[1].length - a[1].length)
|
|
1589
|
+
.filter(([, items]) => items.length >= resolvedMinOccurrences);
|
|
1590
|
+
if (diagnosisEntries.length > 0) {
|
|
1591
|
+
lines.push('');
|
|
1592
|
+
lines.push('## Root Cause Categories');
|
|
1593
|
+
diagnosisEntries.forEach(([category, items]) => {
|
|
1594
|
+
lines.push(`- ${category}: ${items.length} failures`);
|
|
1595
|
+
});
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
const repeatedViolationEntries = Object.entries(repeatedViolationBuckets)
|
|
1599
|
+
.sort((a, b) => b[1].length - a[1].length)
|
|
1600
|
+
.filter(([, items]) => items.length >= resolvedMinOccurrences);
|
|
1601
|
+
if (repeatedViolationEntries.length > 0) {
|
|
1602
|
+
lines.push('');
|
|
1603
|
+
lines.push('## Repeated Failure Constraints');
|
|
1604
|
+
repeatedViolationEntries.forEach(([constraintId, items]) => {
|
|
1605
|
+
lines.push(`- ${constraintId}: ${items.length} failures`);
|
|
1606
|
+
});
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
if (lines.length === 3) {
|
|
1610
|
+
lines.push('');
|
|
1611
|
+
lines.push(`No domain has reached the threshold (${resolvedMinOccurrences}) yet.`);
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
return lines.join('\n');
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
function writePreventionRules(filePath, minOccurrences = 2) {
|
|
1618
|
+
const { PREVENTION_RULES_PATH } = getFeedbackPaths();
|
|
1619
|
+
const outPath = filePath || PREVENTION_RULES_PATH;
|
|
1620
|
+
const resolvedMinOccurrences = Number.isFinite(minOccurrences)
|
|
1621
|
+
? minOccurrences
|
|
1622
|
+
: getEffectiveSetting('prevention_min_occurrences', 2);
|
|
1623
|
+
const markdown = buildPreventionRules(resolvedMinOccurrences);
|
|
1624
|
+
ensureDir(path.dirname(outPath));
|
|
1625
|
+
fs.writeFileSync(outPath, `${markdown}\n`);
|
|
1626
|
+
|
|
1627
|
+
const contextFs = getContextFsModule();
|
|
1628
|
+
if (contextFs && typeof contextFs.registerPreventionRules === 'function') {
|
|
1629
|
+
try {
|
|
1630
|
+
contextFs.registerPreventionRules(markdown, { minOccurrences: resolvedMinOccurrences, outputPath: outPath });
|
|
1631
|
+
} catch {
|
|
1632
|
+
// Non-critical
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
return { path: outPath, markdown };
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
function feedbackSummary(recentN = 20) {
|
|
1639
|
+
const { FEEDBACK_LOG_PATH } = getFeedbackPaths();
|
|
1640
|
+
const entries = readJSONL(FEEDBACK_LOG_PATH);
|
|
1641
|
+
if (entries.length === 0) {
|
|
1642
|
+
return '## Feedback Summary\nNo feedback recorded yet.';
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
const recent = entries.slice(-recentN);
|
|
1646
|
+
const positive = recent.filter((e) => e.signal === 'positive').length;
|
|
1647
|
+
const negative = recent.filter((e) => e.signal === 'negative').length;
|
|
1648
|
+
const pct = Math.round((positive / recent.length) * 100);
|
|
1649
|
+
|
|
1650
|
+
const analysis = analyzeFeedback(FEEDBACK_LOG_PATH);
|
|
1651
|
+
|
|
1652
|
+
const lines = [
|
|
1653
|
+
`## Feedback Summary (last ${recent.length})`,
|
|
1654
|
+
`- Positive: ${positive}`,
|
|
1655
|
+
`- Negative: ${negative}`,
|
|
1656
|
+
`- Approval: ${pct}%`,
|
|
1657
|
+
`- Overall approval: ${Math.round(analysis.approvalRate * 100)}%`,
|
|
1658
|
+
];
|
|
1659
|
+
|
|
1660
|
+
if (analysis.delegation) {
|
|
1661
|
+
lines.push(`- Delegation attempts: ${analysis.delegation.attemptCount}`);
|
|
1662
|
+
lines.push(`- Delegation accepted/rejected/aborted: ${analysis.delegation.acceptedCount}/${analysis.delegation.rejectedCount}/${analysis.delegation.abortedCount}`);
|
|
1663
|
+
lines.push(`- Delegation verification failure rate: ${Math.round((analysis.delegation.verificationFailureRate || 0) * 100)}%`);
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
if (analysis.boostedRisk) {
|
|
1667
|
+
lines.push(`- Boosted risk base rate: ${Math.round((analysis.boostedRisk.baseRate || 0) * 100)}%`);
|
|
1668
|
+
lines.push(`- Boosted risk mode: ${analysis.boostedRisk.mode}`);
|
|
1669
|
+
if (analysis.boostedRisk.highRiskDomains.length > 0) {
|
|
1670
|
+
const topDomain = analysis.boostedRisk.highRiskDomains[0];
|
|
1671
|
+
lines.push(`- Highest-risk domain: ${topDomain.key} (${Math.round(topDomain.riskRate * 100)}%)`);
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
if (analysis.recommendations.length > 0) {
|
|
1676
|
+
lines.push('- Recommendations:');
|
|
1677
|
+
analysis.recommendations.slice(0, 5).forEach((r) => lines.push(` - ${r}`));
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
return lines.join('\n');
|
|
1681
|
+
}
|
|
1682
|
+
|
|
1683
|
+
function parseArgs(argv) {
|
|
1684
|
+
const args = {};
|
|
1685
|
+
argv.forEach((arg) => {
|
|
1686
|
+
if (!arg.startsWith('--')) return;
|
|
1687
|
+
const [key, ...rest] = arg.slice(2).split('=');
|
|
1688
|
+
args[key] = rest.length > 0 ? rest.join('=') : true;
|
|
1689
|
+
});
|
|
1690
|
+
return args;
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
function runCli() {
|
|
1694
|
+
const args = parseArgs(process.argv.slice(2));
|
|
1695
|
+
|
|
1696
|
+
if (args.test) {
|
|
1697
|
+
runTests();
|
|
1698
|
+
return;
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
if (args.capture) {
|
|
1702
|
+
const result = captureFeedback({
|
|
1703
|
+
signal: args.signal,
|
|
1704
|
+
context: args.context || '',
|
|
1705
|
+
whatWentWrong: args['what-went-wrong'],
|
|
1706
|
+
whatToChange: args['what-to-change'],
|
|
1707
|
+
whatWorked: args['what-worked'],
|
|
1708
|
+
rubricScores: args['rubric-scores'],
|
|
1709
|
+
guardrails: args.guardrails,
|
|
1710
|
+
tags: args.tags,
|
|
1711
|
+
skill: args.skill,
|
|
1712
|
+
});
|
|
1713
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1714
|
+
process.exit(result.accepted ? 0 : 2);
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
if (args.analyze) {
|
|
1718
|
+
console.log(JSON.stringify(analyzeFeedback(), null, 2));
|
|
1719
|
+
return;
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
if (args.summary) {
|
|
1723
|
+
console.log(feedbackSummary(Number(args.recent || 20)));
|
|
1724
|
+
return;
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
if (args.rules) {
|
|
1728
|
+
const result = writePreventionRules(args.output, Number(args.min || 2));
|
|
1729
|
+
console.log(`Wrote prevention rules to ${result.path}`);
|
|
1730
|
+
return;
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
console.log(`Usage:
|
|
1734
|
+
node scripts/feedback-loop.js --capture --signal=up --context="..." --tags="verification,fix"
|
|
1735
|
+
node scripts/feedback-loop.js --capture --signal=up --context="..." --rubric-scores='[{\"criterion\":\"correctness\",\"score\":4}]' --guardrails='{\"testsPassed\":true}'
|
|
1736
|
+
node scripts/feedback-loop.js --analyze
|
|
1737
|
+
node scripts/feedback-loop.js --summary --recent=20
|
|
1738
|
+
node scripts/feedback-loop.js --rules [--min=2] [--output=path]
|
|
1739
|
+
node scripts/feedback-loop.js --test`);
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
function runTests() {
|
|
1743
|
+
let passed = 0;
|
|
1744
|
+
let failed = 0;
|
|
1745
|
+
|
|
1746
|
+
function assert(condition, name) {
|
|
1747
|
+
if (condition) {
|
|
1748
|
+
passed++;
|
|
1749
|
+
console.log(` PASS ${name}`);
|
|
1750
|
+
} else {
|
|
1751
|
+
failed++;
|
|
1752
|
+
console.log(` FAIL ${name}`);
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
const tmpDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'rlhf-loop-test-'));
|
|
1757
|
+
const localFeedbackLog = path.join(tmpDir, 'feedback-log.jsonl');
|
|
1758
|
+
process.env.THUMBGATE_FEEDBACK_DIR = tmpDir;
|
|
1759
|
+
|
|
1760
|
+
appendJSONL(localFeedbackLog, { signal: 'positive', tags: ['testing'], skill: 'verify' });
|
|
1761
|
+
appendJSONL(localFeedbackLog, { signal: 'negative', tags: ['testing'], skill: 'verify' });
|
|
1762
|
+
appendJSONL(localFeedbackLog, { signal: 'positive', tags: ['testing'], skill: 'verify' });
|
|
1763
|
+
|
|
1764
|
+
const stats = analyzeFeedback(localFeedbackLog);
|
|
1765
|
+
assert(stats.total === 3, 'analyzeFeedback counts total events');
|
|
1766
|
+
assert(stats.totalPositive === 2, 'analyzeFeedback counts positives');
|
|
1767
|
+
assert(stats.totalNegative === 1, 'analyzeFeedback counts negatives');
|
|
1768
|
+
assert(stats.tags.testing.total === 3, 'analyzeFeedback tracks tags');
|
|
1769
|
+
|
|
1770
|
+
const good = captureFeedback({
|
|
1771
|
+
signal: 'up',
|
|
1772
|
+
context: 'Ran tests and included output',
|
|
1773
|
+
whatWorked: 'Evidence-first flow',
|
|
1774
|
+
tags: ['verification', 'testing'],
|
|
1775
|
+
skill: 'executor',
|
|
1776
|
+
});
|
|
1777
|
+
assert(good.accepted, 'captureFeedback accepts valid positive feedback');
|
|
1778
|
+
|
|
1779
|
+
const blocked = captureFeedback({
|
|
1780
|
+
signal: 'up',
|
|
1781
|
+
context: 'Looks good',
|
|
1782
|
+
whatWorked: 'Skipped proof',
|
|
1783
|
+
tags: ['verification'],
|
|
1784
|
+
rubricScores: JSON.stringify([
|
|
1785
|
+
{ criterion: 'verification_evidence', score: 5, judge: 'judge-a' },
|
|
1786
|
+
{ criterion: 'verification_evidence', score: 2, judge: 'judge-b', evidence: 'no test output present' },
|
|
1787
|
+
]),
|
|
1788
|
+
guardrails: JSON.stringify({
|
|
1789
|
+
testsPassed: false,
|
|
1790
|
+
pathSafety: true,
|
|
1791
|
+
budgetCompliant: true,
|
|
1792
|
+
}),
|
|
1793
|
+
});
|
|
1794
|
+
assert(!blocked.accepted, 'captureFeedback blocks unsafe positive promotion via rubric gate');
|
|
1795
|
+
|
|
1796
|
+
const bad = captureFeedback({ signal: 'down' });
|
|
1797
|
+
assert(!bad.accepted, 'captureFeedback rejects vague negative feedback');
|
|
1798
|
+
assert(bad.needsClarification === true, 'captureFeedback requests clarification for vague negative feedback');
|
|
1799
|
+
|
|
1800
|
+
const summary = feedbackSummary(5);
|
|
1801
|
+
assert(summary.includes('Feedback Summary'), 'feedbackSummary returns text output');
|
|
1802
|
+
|
|
1803
|
+
const rules = writePreventionRules(path.join(tmpDir, 'rules.md'), 1);
|
|
1804
|
+
assert(rules.markdown.includes('# Prevention Rules'), 'writePreventionRules writes markdown rules');
|
|
1805
|
+
const postStats = analyzeFeedback(path.join(tmpDir, 'feedback-log.jsonl'));
|
|
1806
|
+
assert(postStats.rubric.blockedPromotions >= 1, 'analyzeFeedback tracks blocked rubric promotions');
|
|
1807
|
+
|
|
1808
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
1809
|
+
delete process.env.THUMBGATE_FEEDBACK_DIR;
|
|
1810
|
+
console.log(`\nResults: ${passed} passed, ${failed} failed\n`);
|
|
1811
|
+
process.exit(failed > 0 ? 1 : 0);
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
/**
|
|
1815
|
+
* Compact the memory JSONL log — remove exact-content duplicates, keep the most recent.
|
|
1816
|
+
* @returns {{ before: number, after: number, removed: number }}
|
|
1817
|
+
*/
|
|
1818
|
+
function compactMemories() {
|
|
1819
|
+
const { MEMORY_LOG_PATH } = getFeedbackPaths();
|
|
1820
|
+
const all = readJSONL(MEMORY_LOG_PATH, { maxLines: 0 });
|
|
1821
|
+
const seen = new Map();
|
|
1822
|
+
|
|
1823
|
+
// Walk newest-first so we keep the latest version of each memory
|
|
1824
|
+
for (let i = all.length - 1; i >= 0; i--) {
|
|
1825
|
+
const m = all[i];
|
|
1826
|
+
const key = (m.content || '').trim().toLowerCase();
|
|
1827
|
+
if (!key) {
|
|
1828
|
+
// Keep records with no content (edge case)
|
|
1829
|
+
seen.set(`__empty_${i}`, m);
|
|
1830
|
+
continue;
|
|
1831
|
+
}
|
|
1832
|
+
if (!seen.has(key)) {
|
|
1833
|
+
seen.set(key, m);
|
|
1834
|
+
}
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
const deduped = [...seen.values()].reverse();
|
|
1838
|
+
ensureDir(path.dirname(MEMORY_LOG_PATH));
|
|
1839
|
+
fs.writeFileSync(MEMORY_LOG_PATH, deduped.map((r) => JSON.stringify(r)).join('\n') + (deduped.length ? '\n' : ''));
|
|
1840
|
+
|
|
1841
|
+
return {
|
|
1842
|
+
before: all.length,
|
|
1843
|
+
after: deduped.length,
|
|
1844
|
+
removed: all.length - deduped.length,
|
|
1845
|
+
};
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
module.exports = {
|
|
1849
|
+
captureFeedback,
|
|
1850
|
+
compactMemories,
|
|
1851
|
+
analyzeFeedback,
|
|
1852
|
+
buildPreventionRules,
|
|
1853
|
+
writePreventionRules,
|
|
1854
|
+
feedbackSummary,
|
|
1855
|
+
listEnforcementMatrix,
|
|
1856
|
+
readJSONL,
|
|
1857
|
+
appendDiagnosticRecord,
|
|
1858
|
+
readDiagnosticEntries,
|
|
1859
|
+
getFeedbackPaths,
|
|
1860
|
+
inferDomain,
|
|
1861
|
+
inferOutcome,
|
|
1862
|
+
enrichFeedbackContext,
|
|
1863
|
+
inferLessonFromConversation,
|
|
1864
|
+
updateStatuslineWithLesson,
|
|
1865
|
+
waitForBackgroundSideEffects,
|
|
1866
|
+
getPendingBackgroundSideEffectCount,
|
|
1867
|
+
getFeedbackPaths,
|
|
1868
|
+
get FEEDBACK_LOG_PATH() {
|
|
1869
|
+
return getFeedbackPaths().FEEDBACK_LOG_PATH;
|
|
1870
|
+
},
|
|
1871
|
+
get DIAGNOSTIC_LOG_PATH() {
|
|
1872
|
+
return getFeedbackPaths().DIAGNOSTIC_LOG_PATH;
|
|
1873
|
+
},
|
|
1874
|
+
get MEMORY_LOG_PATH() {
|
|
1875
|
+
return getFeedbackPaths().MEMORY_LOG_PATH;
|
|
1876
|
+
},
|
|
1877
|
+
get SUMMARY_PATH() {
|
|
1878
|
+
return getFeedbackPaths().SUMMARY_PATH;
|
|
1879
|
+
},
|
|
1880
|
+
get PREVENTION_RULES_PATH() {
|
|
1881
|
+
return getFeedbackPaths().PREVENTION_RULES_PATH;
|
|
1882
|
+
},
|
|
1883
|
+
};
|
|
1884
|
+
|
|
1885
|
+
if (require.main === module) {
|
|
1886
|
+
runCli();
|
|
1887
|
+
}
|