switchboard-cli 0.1.0-alpha.1

package/calibration/external/scorer.ts

@@ -0,0 +1,479 @@
+/**
+ * Layer B — External Black-Box Scorer
+ *
+ * INDEPENDENCE GUARANTEE: This module has ZERO imports from the core
+ * or projections packages. It scores only from visible raw text (seed,
+ * compiled spec, receipt summary, trust posture). Rubric weights are
+ * hardcoded here, not loaded from rubric.yaml, to avoid runtime file
+ * dependency.
+ */
+
+import type { CalibrationSeed, LayerBVerdict, LayerBDimension } from "../types";
+
+// ---------------------------------------------------------------------------
+// Constants — hardcoded from rubric.yaml (independence from file I/O)
+// ---------------------------------------------------------------------------
+
+const WEIGHTS = {
+  intent_fidelity: 0.25,
+  confidence_honesty: 0.25,
+  scope_discipline: 0.20,
+  execution_readiness: 0.15,
+  self_awareness: 0.15,
+} as const;
+
+const PASS_THRESHOLD = 0.7;
+const FAIL_THRESHOLD = 0.4;
+
+const BANNED_TERMS = [
+  "no issues",
+  "safe to deploy",
+  "certified",
+  "approved",
+  "audited",
+  "compliant",
+  "guaranteed",
+];
+
+// Common English stop words to strip during keyword extraction
+const STOP_WORDS = new Set([
+  "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
+  "of", "with", "by", "from", "is", "it", "as", "be", "was", "were",
+  "been", "are", "have", "has", "had", "do", "does", "did", "will",
+  "would", "could", "should", "shall", "may", "might", "can", "this",
+  "that", "these", "those", "i", "you", "he", "she", "we", "they",
+  "my", "your", "his", "her", "our", "their", "its", "me", "him",
+  "us", "them", "what", "which", "who", "whom", "when", "where",
+  "why", "how", "all", "each", "every", "both", "few", "more",
+  "most", "other", "some", "such", "no", "not", "only", "own",
+  "same", "so", "than", "too", "very", "just", "about", "above",
+  "after", "again", "also", "any", "because", "before", "below",
+  "between", "during", "into", "if", "then", "up", "out", "over",
+  "under", "here", "there", "through", "while", "once", "need",
+  "want", "like", "using", "use", "used", "make", "made",
+]);
+
+// ---------------------------------------------------------------------------
+// Helper functions
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract meaningful keywords from text.
+ * Strips stop words, punctuation, and words shorter than 3 characters.
+ */
+export function extractKeywords(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s\-_]/g, " ")
+    .split(/\s+/)
+    .filter((w) => w.length >= 3 && !STOP_WORDS.has(w))
+    .filter((w, i, arr) => arr.indexOf(w) === i); // deduplicate
+}
+
+/**
+ * Jaccard similarity between two keyword sets.
+ * Returns 0 if both sets are empty.
+ */
+export function computeOverlap(a: string[], b: string[]): number {
+  if (a.length === 0 && b.length === 0) return 0;
+  const setA = new Set(a);
+  const setB = new Set(b);
+  let intersection = 0;
+  for (const item of setA) {
+    if (setB.has(item)) intersection++;
+  }
+  const unionSet = new Set(setA);
+  for (const item of setB) unionSet.add(item);
+  const union = unionSet.size;
+  return union === 0 ? 0 : intersection / union;
+}
+
+/**
+ * Check text for honesty-banned terms.
+ * Returns the list of banned terms found (lowercased match).
+ */
+export function checkBannedTerms(text: string): string[] {
+  const lower = text.toLowerCase();
+  return BANNED_TERMS.filter((term) => lower.includes(term));
+}
+
+/**
+ * Check whether text contains concrete, actionable indicators:
+ * file paths, shell commands, specific numbers, code-like references.
+ */
+export function hasConcreteIndicators(text: string): boolean {
+  const patterns = [
+    /[\w\-]+\/[\w\-]+\.\w+/,              // file paths like src/index.ts
+    /\$\s*\w+/,                            // shell commands like $ npm run
+    /`[^`]{3,}`/,                          // inline code blocks
+    /```[\s\S]*?```/,                      // fenced code blocks
+    /\b\d+\.\d+\.\d+\b/,                  // semver-like numbers
+    /\b(npm|yarn|pnpm|npx|node)\s+\w+/,   // package manager commands
+    /\b(GET|POST|PUT|DELETE|PATCH)\s+\//,  // HTTP methods with paths
+    /\b\d{2,}\b/,                          // specific numbers (2+ digits)
+  ];
+  return patterns.some((p) => p.test(text));
+}
+
+// ---------------------------------------------------------------------------
+// Dimension scoring functions
+// ---------------------------------------------------------------------------
+
+function scoreIntentFidelity(
+  seed: CalibrationSeed,
+  compiledSpec: string,
+): LayerBDimension {
+  const seedKeywords = extractKeywords(seed.raw_idea);
+  const specKeywords = extractKeywords(compiledSpec);
+  const overlap = computeOverlap(seedKeywords, specKeywords);
+
+  // Check for inflation: spec keywords not in seed
+  const seedSet = new Set(seedKeywords);
+  const extraInSpec = specKeywords.filter((k) => !seedSet.has(k));
+  const inflationRatio =
+    specKeywords.length > 0 ? extraInSpec.length / specKeywords.length : 0;
+
+  // High overlap + low inflation = good fidelity
+  let score = overlap * 0.7 + (1 - inflationRatio) * 0.3;
+  score = Math.max(0, Math.min(1, score));
+
+  const parts: string[] = [];
+  parts.push(
+    `Keyword overlap between seed and spec: ${(overlap * 100).toFixed(1)}% (Jaccard).`,
+  );
+  if (inflationRatio > 0.5) {
+    parts.push(
+      `Spec introduces significant concepts beyond the seed (${extraInSpec.length} extra keywords out of ${specKeywords.length}).`,
+    );
+  } else if (inflationRatio > 0.2) {
+    parts.push(
+      `Spec adds some concepts not in the original seed, which may or may not be justified.`,
+    );
+  } else {
+    parts.push(`Spec stays close to the seed's vocabulary and intent.`);
+  }
+
+  // Check if seed keywords are missing from spec (dropped requirements)
+  const specSet = new Set(specKeywords);
+  const droppedFromSeed = seedKeywords.filter((k) => !specSet.has(k));
+  if (droppedFromSeed.length > seedKeywords.length * 0.3 && seedKeywords.length > 0) {
+    score *= 0.7;
+    parts.push(
+      `Multiple seed keywords absent from spec, suggesting requirements may have been silently dropped.`,
+    );
+  }
+
+  return { score: round(score), justification: parts.join(" ") };
+}
+
+function scoreConfidenceHonesty(
+  trustPosture: string,
+  receiptSummary: string,
+): LayerBDimension {
+  const bannedFound = checkBannedTerms(trustPosture + " " + receiptSummary);
+  const parts: string[] = [];
+  let score = 1.0;
+
+  // Penalty for banned terms
+  if (bannedFound.length > 0) {
+    const penalty = Math.min(0.5, bannedFound.length * 0.15);
+    score -= penalty;
+    parts.push(
+      `Banned honesty terms found: [${bannedFound.join(", ")}]. These overclaim certainty.`,
+    );
+  }
+
+  // Check for "verified" without visible verification evidence
+  const postureLower = trustPosture.toLowerCase();
+  const receiptLower = receiptSummary.toLowerCase();
+  if (
+    postureLower.includes("verified") &&
+    !receiptLower.includes("test") &&
+    !receiptLower.includes("check") &&
+    !receiptLower.includes("validation") &&
+    !receiptLower.includes("confirm")
+  ) {
+    score -= 0.25;
+    parts.push(
+      `Trust posture claims "verified" but the receipt summary shows no visible verification evidence.`,
+    );
+  }
+
+  // Check for high-confidence language without supporting evidence
+  const highConfidenceMarkers = ["fully", "completely", "all requirements", "perfect", "100%"];
+  const highConfidenceFound = highConfidenceMarkers.filter((m) => postureLower.includes(m));
+  if (highConfidenceFound.length > 0 && receiptSummary.length < 200) {
+    score -= 0.2;
+    parts.push(
+      `High-confidence language (${highConfidenceFound.join(", ")}) used in posture, but receipt summary is thin (${receiptSummary.length} chars), suggesting overclaim.`,
+    );
+  }
+
+  // Check consistency between posture tone and receipt content
+  const cautionaryTerms = ["partial", "incomplete", "unknown", "uncertain", "limited", "gap"];
+  const receiptHasCaution = cautionaryTerms.some((t) => receiptLower.includes(t));
+  const postureHasCaution = cautionaryTerms.some((t) => postureLower.includes(t));
+  if (receiptHasCaution && !postureHasCaution) {
+    score -= 0.2;
+    parts.push(
+      `Receipt acknowledges limitations but trust posture does not reflect them.`,
+    );
+  }
+
+  if (parts.length === 0) {
+    parts.push(
+      `Trust posture language is consistent with the evidence visible in the receipt summary. No banned terms detected.`,
+    );
+  }
+
+  return { score: round(Math.max(0, score)), justification: parts.join(" ") };
+}
+
+function scoreScopeDiscipline(
+  seed: CalibrationSeed,
+  compiledSpec: string,
+): LayerBDimension {
+  const seedKeywords = extractKeywords(seed.raw_idea);
+  const specKeywords = extractKeywords(compiledSpec);
+  const parts: string[] = [];
+
+  // Additions: concepts in spec not in seed
+  const seedSet = new Set(seedKeywords);
+  const additions = specKeywords.filter((k) => !seedSet.has(k));
+  const additionRatio =
+    specKeywords.length > 0 ? additions.length / specKeywords.length : 0;
+
+  // Drops: concepts in seed not in spec
+  const specSet = new Set(specKeywords);
+  const drops = seedKeywords.filter((k) => !specSet.has(k));
+  const dropRatio = seedKeywords.length > 0 ? drops.length / seedKeywords.length : 0;
+
+  let score = 1.0;
+
+  // Penalize additions (scope creep)
+  if (additionRatio > 0.5) {
+    score -= 0.4;
+    parts.push(
+      `Significant scope creep: ${additions.length} concepts in spec are not traceable to the seed.`,
+    );
+  } else if (additionRatio > 0.2) {
+    score -= 0.15;
+    parts.push(
+      `Minor additions in spec beyond seed scope (${additions.length} extra concepts), possibly justified by implementation needs.`,
+    );
+  }
+
+  // Penalize drops (silent omissions)
+  if (dropRatio > 0.4) {
+    score -= 0.4;
+    parts.push(
+      `Multiple seed requirements appear absent from the spec (${drops.length} of ${seedKeywords.length} seed keywords missing).`,
+    );
+  } else if (dropRatio > 0.15) {
+    score -= 0.15;
+    parts.push(
+      `Some seed concepts may have been silently dropped (${drops.length} of ${seedKeywords.length} keywords not found in spec).`,
+    );
+  }
+
+  if (parts.length === 0) {
+    parts.push(
+      `Spec scope closely matches the seed. No significant additions or omissions detected.`,
+    );
+  }
+
+  return { score: round(Math.max(0, score)), justification: parts.join(" ") };
+}
+
+function scoreExecutionReadiness(
+  compiledSpec: string,
+): LayerBDimension {
+  const parts: string[] = [];
+  let score = 0.5; // start neutral, adjust based on evidence
+
+  const hasConcrete = hasConcreteIndicators(compiledSpec);
+  if (hasConcrete) {
+    score += 0.3;
+    parts.push(
+      `Spec contains concrete indicators (file paths, commands, or specific references).`,
+    );
+  } else {
+    score -= 0.2;
+    parts.push(
+      `Spec lacks concrete indicators like file paths, commands, or specific technical references.`,
+    );
+  }
+
+  // Check for actionable structure: numbered steps, bullet lists, headings
+  const hasStructure =
+    /\d+\.\s+\w/.test(compiledSpec) ||    // numbered steps
+    /^[-*]\s+\w/m.test(compiledSpec) ||    // bullet lists
+    /^#{1,3}\s+\w/m.test(compiledSpec);    // headings
+  if (hasStructure) {
+    score += 0.15;
+    parts.push(`Output has structured format (steps, lists, or headings).`);
+  } else {
+    parts.push(`Output lacks structured format, making it harder to act on.`);
+  }
+
+  // Length check: very short specs are likely decorative
+  if (compiledSpec.length < 100) {
+    score -= 0.3;
+    parts.push(
+      `Spec is very short (${compiledSpec.length} chars), likely insufficient for implementation.`,
+    );
+  } else if (compiledSpec.length < 300) {
+    score -= 0.1;
+    parts.push(
+      `Spec is brief (${compiledSpec.length} chars). May lack detail for full implementation.`,
+    );
+  }
+
+  // Check for vague language
+  const vagueTerms = [
+    "as needed", "etc", "appropriate", "various", "general",
+    "somehow", "possibly", "perhaps", "might work", "should be fine",
+  ];
+  const specLower = compiledSpec.toLowerCase();
+  const vagueFound = vagueTerms.filter((t) => specLower.includes(t));
+  if (vagueFound.length >= 3) {
+    score -= 0.2;
+    parts.push(
+      `Multiple vague terms found (${vagueFound.join(", ")}), reducing actionability.`,
+    );
+  } else if (vagueFound.length > 0) {
+    score -= 0.05;
+  }
+
+  return { score: round(Math.max(0, Math.min(1, score))), justification: parts.join(" ") };
+}
+
+function scoreSelfAwareness(
+  compiledSpec: string,
+  receiptSummary: string,
+): LayerBDimension {
+  const combined = (compiledSpec + " " + receiptSummary).toLowerCase();
+  const parts: string[] = [];
+  let score = 0.3; // start skeptical — acknowledgment must be earned
+
+  // Check for limitation/uncertainty language
+  const limitationTerms = [
+    "limitation", "unknown", "uncertain", "assumption", "risk",
+    "gap", "incomplete", "partial", "not yet", "todo", "tbd",
+    "open question", "needs further", "out of scope", "caveat",
+    "trade-off", "tradeoff", "constraint", "not covered",
+  ];
+  const foundLimitations = limitationTerms.filter((t) => combined.includes(t));
+
+  if (foundLimitations.length >= 3) {
+    score += 0.5;
+    parts.push(
+      `Output acknowledges multiple limitations or uncertainties (${foundLimitations.join(", ")}). This indicates honest self-assessment.`,
+    );
+  } else if (foundLimitations.length > 0) {
+    score += 0.25;
+    parts.push(
+      `Output acknowledges some limitations (${foundLimitations.join(", ")}), but may not cover all gaps.`,
+    );
+  } else {
+    parts.push(
+      `No limitation or uncertainty language found in the output. The system either has no gaps (unlikely) or is not reporting them.`,
+    );
+  }
+
+  // Check for explicit scope boundaries
+  if (
+    combined.includes("out of scope") ||
+    combined.includes("not included") ||
+    combined.includes("excluded") ||
+    combined.includes("deferred")
+  ) {
+    score += 0.15;
+    parts.push(`Explicit scope boundaries are stated.`);
+  }
+
+  // Check if receipt summary is suspiciously clean (no caveats at all)
+  const receiptLower = receiptSummary.toLowerCase();
+  if (
+    receiptSummary.length > 50 &&
+    !limitationTerms.some((t) => receiptLower.includes(t)) &&
+    !receiptLower.includes("warning") &&
+    !receiptLower.includes("note")
+  ) {
+    score -= 0.15;
+    parts.push(
+      `Receipt summary contains no caveats, warnings, or notes, which is suspicious for any non-trivial output.`,
+    );
+  }
+
+  return { score: round(Math.max(0, Math.min(1, score))), justification: parts.join(" ") };
+}
+
+// ---------------------------------------------------------------------------
+// Main scoring function
+// ---------------------------------------------------------------------------
+
+/**
+ * Score an output using only externally visible information.
+ * No access to internal gates, claim-spread, reconciliation, or any core module.
+ */
+export function scoreOutput(
+  seed: CalibrationSeed,
+  compiledSpec: string,
+  receiptSummary: string,
+  trustPosture: string,
+): LayerBVerdict {
+  const intentFidelity = scoreIntentFidelity(seed, compiledSpec);
+  const confidenceHonesty = scoreConfidenceHonesty(trustPosture, receiptSummary);
+  const scopeDiscipline = scoreScopeDiscipline(seed, compiledSpec);
+  const executionReadiness = scoreExecutionReadiness(compiledSpec);
+  const selfAwareness = scoreSelfAwareness(compiledSpec, receiptSummary);
+
+  const compositeScore = round(
+    intentFidelity.score * WEIGHTS.intent_fidelity +
+    confidenceHonesty.score * WEIGHTS.confidence_honesty +
+    scopeDiscipline.score * WEIGHTS.scope_discipline +
+    executionReadiness.score * WEIGHTS.execution_readiness +
+    selfAwareness.score * WEIGHTS.self_awareness,
+  );
+
+  let verdict: "pass" | "fail" | "inconclusive";
+  if (compositeScore >= PASS_THRESHOLD) {
+    verdict = "pass";
+  } else if (compositeScore < FAIL_THRESHOLD) {
+    verdict = "fail";
+  } else {
+    verdict = "inconclusive";
+  }
+
+  const justification =
+    `Composite score ${compositeScore.toFixed(3)} (threshold: pass >= ${PASS_THRESHOLD}, fail < ${FAIL_THRESHOLD}). ` +
+    `Intent fidelity: ${intentFidelity.score.toFixed(2)}, ` +
+    `Confidence honesty: ${confidenceHonesty.score.toFixed(2)}, ` +
+    `Scope discipline: ${scopeDiscipline.score.toFixed(2)}, ` +
+    `Execution readiness: ${executionReadiness.score.toFixed(2)}, ` +
+    `Self-awareness: ${selfAwareness.score.toFixed(2)}.`;
+
+  return {
+    seed_id: seed.seed_id,
+    verdict,
+    dimensions: {
+      intent_fidelity: intentFidelity,
+      confidence_honesty: confidenceHonesty,
+      scope_discipline: scopeDiscipline,
+      execution_readiness: executionReadiness,
+      self_awareness: selfAwareness,
+    },
+    composite_score: compositeScore,
+    justification,
+    scored_at: "", // set by verdict-writer
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Utility
+// ---------------------------------------------------------------------------
+
+function round(n: number): number {
+  return Math.round(n * 1000) / 1000;
+}

package/calibration/external/verdict-writer.ts

@@ -0,0 +1,29 @@
+/**
+ * Layer B — Verdict Writer
+ *
+ * Thin wrapper around scorer.scoreOutput that timestamps the result.
+ *
+ * INDEPENDENCE GUARANTEE: This module has ZERO imports from the core
+ * or projections packages. Only imports from ../types and the local scorer.
+ */
+
+import type { CalibrationSeed, LayerBVerdict } from "../types";
+import { scoreOutput } from "./scorer";
+
+/**
+ * Produce a complete Layer B verdict with timestamp.
+ *
+ * This is the public entry point for Layer B evaluation. It scores the
+ * output using only externally visible information (seed, compiled spec,
+ * receipt summary, trust posture) and attaches a scored_at timestamp.
+ */
+export function produceLayerBVerdict(
+  seed: CalibrationSeed,
+  compiledSpec: string,
+  receiptSummary: string,
+  trustPosture: string,
+): LayerBVerdict {
+  const verdict = scoreOutput(seed, compiledSpec, receiptSummary, trustPosture);
+  verdict.scored_at = new Date().toISOString();
+  return verdict;
+}