scene-capability-engine 3.6.65 → 3.6.67

package/lib/semantic/replay-evaluator.js

@@ -0,0 +1,382 @@
+const { getSceStateStore } = require('../state/sce-state-store');
+
+const STOPWORDS = new Set([
+  'a', 'an', 'and', 'are', 'as', 'at', 'be', 'been', 'by', 'for', 'from', 'had',
+  'has', 'have', 'in', 'into', 'is', 'it', 'of', 'on', 'or', 'that', 'the', 'their',
+  'then', 'this', 'to', 'was', 'were', 'with', 'after', 'before', 'will', 'would',
+  'using', 'used', 'use', 'when', 'what', 'why', 'how', 'next', 'time', 'previous',
+  'attempt', 'issue', 'problem', 'reply', 'assistant', 'user', 'general', 'observed',
+  'debug', 'repair', 'planning', 'analysis', 'verification'
+]);
+
+const ACTION_TERMS = [
+  'add', 'analyze', 'bisect', 'check', 'compare', 'debug', 'document', 'fix', 'inspect',
+  'isolate', 'log', 'patch', 'replay', 'reproduce', 'scope', 'test', 'trace', 'validate',
+  'verify', 'review', 'narrow', '定位', '排查', '日志', '修复', '验证', '回放', '复现', '二分'
+];
+
+const SUCCESS_CLAIMS = /(done|fixed|resolved|completed|already implemented|already solved|已修复|已完成|已经解决|搞定)/i;
+
+function normalizeString(value) {
+  if (typeof value !== 'string') {
+    return '';
+  }
+  return value.trim();
+}
+
+function normalizeInteger(value, fallback = 0) {
+  const parsed = Number.parseInt(`${value}`, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    return fallback;
+  }
+  return parsed;
+}
+
+function normalizeNumber(value, fallback = 0) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    return fallback;
+  }
+  return parsed;
+}
+
+function normalizeStringArray(value, fallback = []) {
+  if (!Array.isArray(value)) {
+    return [...fallback];
+  }
+  return value
+    .map((item) => normalizeString(item))
+    .filter(Boolean);
+}
+
+function tokenizeText(value) {
+  const normalized = normalizeString(value).toLowerCase();
+  if (!normalized) {
+    return [];
+  }
+  const matches = normalized.match(/[a-z0-9\u4e00-\u9fff]+/g) || [];
+  return matches.filter((token) => token.length > 1 && !STOPWORDS.has(token));
+}
+
+function summarizeObservation(observation = {}) {
+  return {
+    observation_id: normalizeString(observation.observation_id),
+    source_runtime: normalizeString(observation.source_runtime) || null,
+    event_kind: normalizeString(observation.event_kind) || null,
+    intent: normalizeString(observation.intent) || null,
+    outcome: normalizeString(observation.outcome) || null,
+    quality_signal: normalizeString(observation.quality_signal) || null,
+    summary: normalizeString(observation.summary) || null,
+    task_ref: normalizeString(observation.task_ref) || null,
+    session_id: normalizeString(observation.session_id) || null,
+    spec_id: normalizeString(observation.spec_id) || null,
+    scene_id: normalizeString(observation.scene_id) || null
+  };
+}
+
+function buildSemanticReplayDataset(observations = [], options = {}) {
+  const expectedTermLimit = normalizeInteger(options.expectedTermLimit, 8);
+  const safeObservations = Array.isArray(observations)
+    ? observations.filter((item) => item && normalizeString(item.observation_id))
+    : [];
+  const tokenCounts = new Map();
+  const outcomeProfile = {};
+  const intentProfile = {};
+  const channelIds = new Set();
+  const sessionIds = new Set();
+  const sceneIds = new Set();
+  const specIds = new Set();
+  const sourceRuntimes = new Set();
+
+  for (const observation of safeObservations) {
+    channelIds.add(normalizeString(observation.channel_id));
+    sessionIds.add(normalizeString(observation.session_id));
+    sceneIds.add(normalizeString(observation.scene_id));
+    specIds.add(normalizeString(observation.spec_id));
+    sourceRuntimes.add(normalizeString(observation.source_runtime));
+
+    const outcome = normalizeString(observation.outcome) || 'unknown';
+    outcomeProfile[outcome] = (outcomeProfile[outcome] || 0) + 1;
+
+    const intent = normalizeString(observation.intent) || 'unknown';
+    intentProfile[intent] = (intentProfile[intent] || 0) + 1;
+
+    const basis = [
+      observation.summary,
+      observation.intent,
+      observation.event_kind,
+      observation.outcome,
+      observation.quality_signal,
+      observation.raw_payload && typeof observation.raw_payload === 'object'
+        ? observation.raw_payload.text
+        : '',
+      observation.normalized_payload && typeof observation.normalized_payload === 'object'
+        ? observation.normalized_payload.text
+        : ''
+    ].join(' ');
+
+    for (const token of tokenizeText(basis)) {
+      tokenCounts.set(token, (tokenCounts.get(token) || 0) + 1);
+    }
+  }
+
+  const expectedTerms = [...tokenCounts.entries()]
+    .sort((left, right) => {
+      if (right[1] !== left[1]) {
+        return right[1] - left[1];
+      }
+      return left[0].localeCompare(right[0]);
+    })
+    .slice(0, expectedTermLimit)
+    .map(([token]) => token);
+
+  return {
+    total_observations: safeObservations.length,
+    observation_ids: safeObservations.map((item) => normalizeString(item.observation_id)).filter(Boolean),
+    context: {
+      project_id: normalizeString(safeObservations[0] && safeObservations[0].project_id) || null,
+      channel_ids: [...channelIds].filter(Boolean),
+      session_ids: [...sessionIds].filter(Boolean),
+      scene_ids: [...sceneIds].filter(Boolean),
+      spec_ids: [...specIds].filter(Boolean),
+      source_runtimes: [...sourceRuntimes].filter(Boolean)
+    },
+    expected_terms: expectedTerms,
+    outcome_profile: outcomeProfile,
+    intent_profile: intentProfile,
+    observations: safeObservations.map((item) => summarizeObservation(item))
+  };
+}
+
+function scoreCandidateText(candidateText, replayDataset = {}, options = {}) {
+  const threshold = normalizeNumber(options.threshold, 65);
+  const expectedTerms = normalizeStringArray(replayDataset.expected_terms, []);
+  const candidateTokens = tokenizeText(candidateText);
+  const uniqueCandidateTokens = [...new Set(candidateTokens)];
+  const matchedTerms = expectedTerms.filter((term) => uniqueCandidateTokens.includes(term));
+  const actionabilityHits = ACTION_TERMS.filter((term) => candidateText.toLowerCase().includes(term.toLowerCase()));
+
+  const failedCount = normalizeNumber(replayDataset.outcome_profile && replayDataset.outcome_profile.failed, 0);
+  const blockedCount = normalizeNumber(replayDataset.outcome_profile && replayDataset.outcome_profile.blocked, 0);
+  const unstableCount = failedCount + blockedCount;
+
+  const coverageScore = expectedTerms.length > 0
+    ? (matchedTerms.length / expectedTerms.length) * 70
+    : 20;
+  const actionabilityScore = Math.min(actionabilityHits.length, 3) * (20 / 3);
+  const contextScore = replayDataset.total_observations > 0 ? 10 : 0;
+
+  const contradictionHits = [];
+  let contradictionPenalty = 0;
+  if (unstableCount > 0 && SUCCESS_CLAIMS.test(candidateText) && actionabilityHits.length < 2) {
+    contradictionHits.push('candidate-claims-success-without-remediation-steps');
+    contradictionPenalty += 20;
+  }
+  if (expectedTerms.length > 0 && matchedTerms.length === 0) {
+    contradictionHits.push('candidate-misses-replay-terms');
+    contradictionPenalty += 10;
+  }
+
+  const score = Math.max(
+    0,
+    Math.min(100, coverageScore + actionabilityScore + contextScore - contradictionPenalty)
+  );
+  const regressionReasons = [];
+  if (matchedTerms.length < Math.min(2, expectedTerms.length)) {
+    regressionReasons.push('low-replay-term-coverage');
+  }
+  if (actionabilityHits.length === 0) {
+    regressionReasons.push('missing-actionable-remediation');
+  }
+  if (contradictionHits.length > 0) {
+    regressionReasons.push(...contradictionHits);
+  }
+  if (score < threshold) {
+    regressionReasons.push('score-below-threshold');
+  }
+
+  return {
+    score: Number(score.toFixed(2)),
+    threshold: Number(threshold.toFixed(2)),
+    verdict: score >= threshold ? 'passed' : 'failed',
+    matched_terms: matchedTerms,
+    missing_terms: expectedTerms.filter((term) => !matchedTerms.includes(term)),
+    actionability_hits: actionabilityHits,
+    contradiction_hits: contradictionHits,
+    regression_reasons: [...new Set(regressionReasons)]
+  };
+}
+
+async function collectSemanticReplayInputs(options = {}, dependencies = {}) {
+  const projectPath = dependencies.projectPath || process.cwd();
+  const store = dependencies.stateStore || getSceStateStore(projectPath, {
+    fileSystem: dependencies.fileSystem,
+    env: dependencies.env,
+    sqliteModule: dependencies.sqliteModule,
+    noCache: dependencies.noCache === true
+  });
+
+  const limit = normalizeInteger(options.limit, 10);
+  const replayScanLimit = Math.max(limit * 10, 50);
+  const projectId = normalizeString(options.project_id || options.projectId);
+  const channelId = normalizeString(options.channel_id || options.channelId);
+  const sessionId = normalizeString(options.session_id || options.sessionId);
+  const sceneId = normalizeString(options.scene_id || options.sceneId);
+  const specId = normalizeString(options.spec_id || options.specId);
+  const sourceRuntime = normalizeString(options.source_runtime || options.sourceRuntime);
+  const observationIds = new Set(normalizeStringArray(options.observation_ids || options.observationIds, []));
+
+  let observations = await store.listSemanticObservations({
+    project_id: projectId,
+    session_id: sessionId,
+    source_runtime: sourceRuntime,
+    limit: replayScanLimit
+  });
+
+  observations = Array.isArray(observations) ? observations : [];
+  observations = observations.filter((item) => {
+    if (channelId && normalizeString(item.channel_id) !== channelId) {
+      return false;
+    }
+    if (sceneId && normalizeString(item.scene_id) !== sceneId) {
+      return false;
+    }
+    if (specId && normalizeString(item.spec_id) !== specId) {
+      return false;
+    }
+    if (observationIds.size > 0 && !observationIds.has(normalizeString(item.observation_id))) {
+      return false;
+    }
+    return true;
+  }).slice(0, limit);
+
+  if (observations.length === 0) {
+    throw new Error('No semantic observations matched the replay filters');
+  }
+
+  return {
+    observations,
+    replay_dataset: buildSemanticReplayDataset(observations, options),
+    store
+  };
+}
+
+async function evaluateSemanticCandidate(options = {}, dependencies = {}) {
+  const candidateReply = normalizeString(options.candidate_reply || options.candidateReply || options.candidate_text || options.candidateText);
+  const candidatePlan = normalizeString(options.candidate_plan || options.candidatePlan);
+  const candidateText = candidateReply || candidatePlan;
+  if (!candidateText) {
+    throw new Error('candidate_reply or candidate_plan is required');
+  }
+
+  const candidateKind = candidateReply ? 'reply' : 'action-plan';
+  const projectPath = dependencies.projectPath || process.cwd();
+  const { observations, replay_dataset: replayDataset, store } = await collectSemanticReplayInputs(options, {
+    ...dependencies,
+    projectPath
+  });
+
+  const baselineRunId = normalizeString(options.baseline_evaluation_run_id || options.baselineEvaluationRunId);
+  let baselineRun = null;
+  if (baselineRunId) {
+    baselineRun = await store.getSemanticEvaluationRun(baselineRunId);
+  } else {
+    const priorRuns = await store.listSemanticEvaluationRuns({
+      project_id: options.project_id || options.projectId,
+      spec_id: options.spec_id || options.specId,
+      limit: 10
+    });
+    baselineRun = Array.isArray(priorRuns)
+      ? priorRuns.find((item) => normalizeString(item.candidate_kind) === candidateKind) || null
+      : null;
+  }
+
+  const scoring = scoreCandidateText(candidateText, replayDataset, options);
+  const scoreDelta = baselineRun && typeof baselineRun.score === 'number'
+    ? Number((scoring.score - baselineRun.score).toFixed(2))
+    : null;
+  const regressionDetected = scoring.verdict === 'failed'
+    || (typeof scoreDelta === 'number' && scoreDelta <= -10);
+  const driftLevel = regressionDetected
+    ? 'regression'
+    : (typeof scoreDelta === 'number' && scoreDelta < 0 ? 'watch' : 'stable');
+  const regressionReasons = [...scoring.regression_reasons];
+  if (typeof scoreDelta === 'number' && scoreDelta <= -10) {
+    regressionReasons.push('degraded-from-baseline');
+  }
+
+  const governance = {
+    eligible_for_errorbook: regressionDetected,
+    eligible_for_problem_evaluation: regressionDetected,
+    promotion_gate_passed: regressionDetected !== true,
+    promotion_gate: regressionDetected ? 'semantic-eval-blocked' : 'semantic-eval-pass',
+    evidence_required: true,
+    release_gate: normalizeString(options.release_gate || options.releaseGate) || null
+  };
+
+  const evidence = {
+    replay_dataset: {
+      total_observations: replayDataset.total_observations,
+      expected_terms: replayDataset.expected_terms,
+      outcome_profile: replayDataset.outcome_profile,
+      intent_profile: replayDataset.intent_profile
+    },
+    candidate_preview: candidateText.slice(0, 240),
+    matched_terms: scoring.matched_terms,
+    missing_terms: scoring.missing_terms,
+    actionability_hits: scoring.actionability_hits,
+    contradiction_hits: scoring.contradiction_hits,
+    baseline: baselineRun
+      ? {
+          evaluation_run_id: baselineRun.evaluation_run_id,
+          score: baselineRun.score,
+          score_delta: scoreDelta
+        }
+      : null
+  };
+
+  const evaluationRun = await store.appendSemanticEvaluationRun({
+    project_id: options.project_id || options.projectId || replayDataset.context.project_id,
+    channel_id: options.channel_id || options.channelId || replayDataset.context.channel_ids[0],
+    session_id: options.session_id || options.sessionId || replayDataset.context.session_ids[0],
+    scene_id: options.scene_id || options.sceneId || replayDataset.context.scene_ids[0],
+    spec_id: options.spec_id || options.specId || replayDataset.context.spec_ids[0],
+    lesson_id: options.lesson_id || options.lessonId,
+    source_runtime: options.source_runtime || options.sourceRuntime || replayDataset.context.source_runtimes[0],
+    release_gate: options.release_gate || options.releaseGate,
+    baseline_evaluation_run_id: baselineRun ? baselineRun.evaluation_run_id : null,
+    candidate_kind: candidateKind,
+    candidate_label: normalizeString(options.candidate_label || options.candidateLabel) || null,
+    candidate_payload: {
+      text: candidateText
+    },
+    observation_ids: replayDataset.observation_ids,
+    replay_summary: replayDataset,
+    score: scoring.score,
+    threshold: scoring.threshold,
+    verdict: scoring.verdict,
+    regression_detected: regressionDetected,
+    drift_level: driftLevel,
+    regression_reasons: [...new Set(regressionReasons)],
+    evidence,
+    governance
+  });
+
+  if (!evaluationRun) {
+    throw new Error('Failed to persist semantic evaluation run');
+  }
+
+  return {
+    evaluation_run: evaluationRun,
+    replay_dataset: replayDataset,
+    observations: observations.map((item) => summarizeObservation(item))
+  };
+}
+
+module.exports = {
+  tokenizeText,
+  buildSemanticReplayDataset,
+  scoreCandidateText,
+  collectSemanticReplayInputs,
+  evaluateSemanticCandidate
+};