switchboard-cli 0.1.0-alpha.1 → 0.1.0-alpha.3

package/node_modules/@switchboard/core/src/meta-harness.ts

@@ -0,0 +1,411 @@
+/**
+ * Meta Harness — internal compile subsystem for context packet improvement.
+ *
+ * First bounded experiment: repo-grounded context bootstrap.
+ *
+ * The meta harness sits inside the existing harness compilation boundary.
+ * It may improve the dispatch context packet when eligible, and is a safe
+ * no-op when it is not. It never rewrites the locked contract, gate logic,
+ * receipt truth, or canonical advancement semantics.
+ *
+ * Integration point: called between prompt generation and harness-driven
+ * dispatch in the execute route. Returns a compilation summary that is
+ * carried on the receipt as harness provenance.
+ */
+
+import { createHash, randomBytes } from "crypto";
+import type {
+  HarnessCandidate,
+  HarnessCompilationSummary,
+  HarnessInputSnapshot,
+  HarnessOutputArtifactRef,
+  HarnessSelection,
+  LoopContract,
+  ProjectContract,
+} from "./types";
+import { harnessCompilationSummarySchema } from "./types";
+
+// ---------------------------------------------------------------------------
+// Eligibility check — is this slice appropriate for meta harness?
+// ---------------------------------------------------------------------------
+
+export interface MetaHarnessEligibilityInput {
+  repoGrounded: boolean;
+  specMarkdown: string;
+  contract: ProjectContract;
+  loop: LoopContract;
+  priorDeltaId?: string | undefined;
+  priorGateLabel?: string | undefined;
+}
+
+export interface MetaHarnessEligibilityResult {
+  eligible: boolean;
+  reason: string;
+}
+
+/**
+ * Check if this compile pass is eligible for meta harness context improvement.
+ *
+ * Rules:
+ * - Project must be repo-grounded (repo-present)
+ * - Spec must be non-trivial (>= 100 chars)
+ * - Loop contract must have a non-empty objective
+ * - This wave only handles repo_context_bootstrap experiment
+ */
+export function checkMetaHarnessEligibility(
+  input: MetaHarnessEligibilityInput,
+): MetaHarnessEligibilityResult {
+  if (!input.repoGrounded) {
+    return { eligible: false, reason: "Project is not repo-grounded." };
+  }
+
+  if (!input.specMarkdown || input.specMarkdown.length < 100) {
+    return { eligible: false, reason: "Spec is too thin for bounded context improvement." };
+  }
+
+  if (!input.loop.objective || input.loop.objective.trim().length < 10) {
+    return { eligible: false, reason: "Loop objective is too thin." };
+  }
+
+  // Confidence check: scope must have at least 1 scope_in item
+  if (input.loop.scope_in.length === 0) {
+    return { eligible: false, reason: "No scope_in defined — insufficient compile context." };
+  }
+
+  return { eligible: true, reason: "Repo-grounded slice with sufficient compile context." };
+}
+
+// ---------------------------------------------------------------------------
+// Input snapshot — hash the compile-time inputs for lineage
+// ---------------------------------------------------------------------------
+
+function hashString(s: string): string {
+  return createHash("sha256").update(s).digest("hex").slice(0, 16);
+}
+
+function buildInputSnapshot(
+  contract: ProjectContract,
+  loop: LoopContract,
+  dispatchId: string,
+  specMarkdown: string,
+  priorDeltaId?: string,
+  priorGateLabel?: string,
+): HarnessInputSnapshot {
+  const specHash = hashString(specMarkdown);
+  const snapshotContent = `${contract.project_id}|${dispatchId}|${specHash}|${loop.objective}`;
+
+  return {
+    contract_id: contract.project_id,
+    dispatch_id: dispatchId,
+    spec_hash: specHash,
+    repo_grounded: contract.repo_status === "repo-present",
+    prior_delta_id: priorDeltaId ?? "",
+    prior_gate_label: priorGateLabel ?? "",
+    loop_objective: loop.objective,
+    systems_in_play: loop.scope_in.slice(0, 10),
+    snapshot_hash: hashString(snapshotContent),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Context section generation — the actual bounded improvement
+// ---------------------------------------------------------------------------
+
+/**
+ * Generate a bounded repo-grounding context section for the dispatch prompt.
+ *
+ * This is the first experiment: compile a context preamble from the existing
+ * contract, prior canonical state, and spec that helps the executor orient
+ * faster in a repo-grounded scenario.
+ *
+ * What it adds:
+ * - Prior loop context (if prior delta exists)
+ * - Systems-in-play advisory
+ * - Build constraint emphasis
+ * - Repo-presence declaration
+ *
+ * What it does NOT add:
+ * - Anything not derivable from existing contract/charter/spec/prior state
+ * - Scope widening
+ * - Contract mutation
+ */
+function generateRepoContextSection(
+  contract: ProjectContract,
+  loop: LoopContract,
+  specMarkdown: string,
+  priorDeltaId?: string,
+  priorGateLabel?: string,
+): { sections: string[]; artifacts: HarnessOutputArtifactRef[] } {
+  const sections: string[] = [];
+  const artifacts: HarnessOutputArtifactRef[] = [];
+
+  // Section 1: Repo grounding declaration
+  sections.push(
+    "## Repo context (meta-harness compiled)\n\n" +
+    "This is a repo-grounded execution. The repository already exists at the working directory. " +
+    "Do not create a new repository. Work within the existing codebase."
+  );
+
+  // Section 2: Systems-in-play advisory (from contract + loop)
+  const systems = [
+    ...loop.scope_in.filter(s => s.length < 60),
+    ...(contract.task_clauses.systems_expected ?? []),
+  ];
+  const uniqueSystems = [...new Set(systems)];
+  if (uniqueSystems.length > 0) {
+    sections.push(
+      "## Systems in play\n\n" +
+      uniqueSystems.map(s => `- ${s}`).join("\n")
+    );
+    artifacts.push({
+      artifact_type: "scope_refinement",
+      content_hash: hashString(uniqueSystems.join("|")),
+      size_bytes: uniqueSystems.join("\n").length,
+      description: `${uniqueSystems.length} systems-in-play advisory items`,
+    });
+  }
+
+  // Section 3: Build constraints emphasis (from contract)
+  const constraints = contract.task_clauses.build_constraints ?? [];
+  if (constraints.length > 0) {
+    sections.push(
+      "## Build constraints (meta-harness emphasis)\n\n" +
+      "These constraints are from the locked contract and must not be violated:\n" +
+      constraints.map(c => `- ${c}`).join("\n")
+    );
+  }
+
+  // Section 4: Prior loop context (when prior canonical delta exists)
+  if (priorDeltaId) {
+    const gateContext = priorGateLabel
+      ? ` The prior gate outcome was: ${priorGateLabel}.`
+      : "";
+    sections.push(
+      "## Prior loop context\n\n" +
+      `This slice continues from prior canonical delta \`${priorDeltaId}\`.${gateContext} ` +
+      "Build on existing work. Do not repeat completed tasks."
+    );
+    artifacts.push({
+      artifact_type: "prior_evidence_digest",
+      content_hash: hashString(priorDeltaId),
+      size_bytes: priorDeltaId.length,
+      description: `Prior delta reference: ${priorDeltaId}`,
+    });
+  }
+
+  // Always add the repo summary artifact
+  artifacts.push({
+    artifact_type: "repo_summary",
+    content_hash: hashString(specMarkdown.slice(0, 200)),
+    size_bytes: sections.reduce((sum, s) => sum + s.length, 0),
+    description: "Repo-grounding context sections for dispatch prompt",
+  });
+
+  return { sections, artifacts };
+}
+
+// ---------------------------------------------------------------------------
+// Primary compile function — run the bounded meta harness pass
+// ---------------------------------------------------------------------------
+
+export interface MetaHarnessCompileInput {
+  contract: ProjectContract;
+  loop: LoopContract;
+  dispatchId: string;
+  specMarkdown: string;
+  priorDeltaId?: string | undefined;
+  priorGateLabel?: string | undefined;
+}
+
+export interface MetaHarnessCompileResult {
+  summary: HarnessCompilationSummary;
+  /** Context sections to prepend to the dispatch prompt. Empty if no-op. */
+  contextSections: string[];
+}
+
+/**
+ * Run the bounded meta harness compile pass.
+ *
+ * Returns a compilation summary (always) and context sections (only when eligible).
+ * When ineligible, returns a clean no-op summary with empty context sections.
+ *
+ * The caller is responsible for prepending context sections to the prompt.
+ * This function never mutates the contract, loop, or any external state.
+ */
+export function compileMetaHarness(
+  input: MetaHarnessCompileInput,
+): MetaHarnessCompileResult {
+  const compilationId = `mhc-${randomBytes(8).toString("hex")}`;
+  const now = new Date().toISOString();
+
+  // Check eligibility
+  const eligibility = checkMetaHarnessEligibility({
+    repoGrounded: input.contract.repo_status === "repo-present",
+    specMarkdown: input.specMarkdown,
+    contract: input.contract,
+    loop: input.loop,
+    priorDeltaId: input.priorDeltaId,
+    priorGateLabel: input.priorGateLabel,
+  });
+
+  // Build input snapshot for lineage
+  const inputSnapshot = buildInputSnapshot(
+    input.contract, input.loop, input.dispatchId,
+    input.specMarkdown, input.priorDeltaId, input.priorGateLabel,
+  );
+
+  if (!eligibility.eligible) {
+    // Clean no-op
+    const noOpSummary = harnessCompilationSummarySchema.parse({
+      compilation_id: compilationId,
+      project_id: input.contract.project_id,
+      dispatch_id: input.dispatchId,
+      experiment: "repo_context_bootstrap",
+      candidates: [],
+      selection: {
+        selected_candidate_id: "",
+        selection_rationale: "",
+        was_no_op: true,
+        no_op_reason: eligibility.reason,
+      },
+      compiled_at: now,
+      emitted_context_sections: [],
+      context_chars_added: 0,
+    });
+    return { summary: noOpSummary, contextSections: [] };
+  }
+
+  // Generate the context sections
+  const { sections, artifacts } = generateRepoContextSection(
+    input.contract, input.loop, input.specMarkdown,
+    input.priorDeltaId, input.priorGateLabel,
+  );
+
+  // Build the candidate
+  const candidateId = `mhcand-${randomBytes(6).toString("hex")}`;
+  const candidate: HarnessCandidate = {
+    candidate_id: candidateId,
+    experiment: "repo_context_bootstrap",
+    status: "selected",
+    rationale: `Repo-grounded slice eligible for context bootstrap. ` +
+      `${sections.length} section(s) derived from contract/loop/prior state. ` +
+      `All additions traceable to existing governed inputs.`,
+    input_snapshot: inputSnapshot,
+    output_artifacts: artifacts,
+    context_additions: sections,
+    created_at: now,
+  };
+
+  // Selection: always select the single candidate in this experiment
+  const selection: HarnessSelection = {
+    selected_candidate_id: candidateId,
+    selection_rationale: "Single candidate from repo_context_bootstrap experiment. " +
+      "All context derived from existing contract, loop, and prior canonical state.",
+    was_no_op: false,
+    no_op_reason: "",
+  };
+
+  const totalChars = sections.reduce((sum, s) => sum + s.length, 0);
+
+  const summary = harnessCompilationSummarySchema.parse({
+    compilation_id: compilationId,
+    project_id: input.contract.project_id,
+    dispatch_id: input.dispatchId,
+    experiment: "repo_context_bootstrap",
+    candidates: [candidate],
+    selection,
+    compiled_at: now,
+    emitted_context_sections: sections,
+    context_chars_added: totalChars,
+  });
+
+  return { summary, contextSections: sections };
+}
+
+// ---------------------------------------------------------------------------
+// Prompt augmentation helper — prepend meta harness sections to prompt
+// ---------------------------------------------------------------------------
+
+/**
+ * Augment a dispatch prompt with meta harness context sections.
+ * Returns the original prompt unchanged if no sections exist.
+ */
+export function augmentPromptWithHarnessContext(
+  originalPrompt: string,
+  contextSections: string[],
+): string {
+  if (contextSections.length === 0) return originalPrompt;
+
+  const harnessBlock = contextSections.join("\n\n");
+  return `${harnessBlock}\n\n---\n\n${originalPrompt}`;
+}
+
+// ---------------------------------------------------------------------------
+// Extract receipt provenance from compilation summary
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract the harness provenance record for embedding in a receipt V2.
+ */
+export function extractHarnessProvenance(
+  summary: HarnessCompilationSummary | null | undefined,
+): {
+  harness_ran: boolean;
+  compilation_id: string;
+  selected_candidate_id: string;
+  experiment: string;
+  selection_rationale: string;
+  context_chars_added: number;
+  was_no_op: boolean;
+} {
+  if (!summary) {
+    return {
+      harness_ran: false,
+      compilation_id: "",
+      selected_candidate_id: "",
+      experiment: "",
+      selection_rationale: "",
+      context_chars_added: 0,
+      was_no_op: true,
+    };
+  }
+
+  return {
+    harness_ran: true,
+    compilation_id: summary.compilation_id,
+    selected_candidate_id: summary.selection.selected_candidate_id,
+    experiment: summary.experiment,
+    selection_rationale: summary.selection.was_no_op
+      ? summary.selection.no_op_reason
+      : summary.selection.selection_rationale,
+    context_chars_added: summary.context_chars_added,
+    was_no_op: summary.selection.was_no_op,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Relink dispatch ID — normalize pre-canonical ID to canonical on both
+// summary and candidate input_snapshot.dispatch_id fields
+// ---------------------------------------------------------------------------
+
+/**
+ * Relink a compilation summary to the canonical dispatch ID.
+ * Patches both the top-level dispatch_id and every candidate's
+ * input_snapshot.dispatch_id so lineage is coherent.
+ */
+export function relinkHarnessDispatchId(
+  summary: HarnessCompilationSummary,
+  canonicalDispatchId: string,
+): HarnessCompilationSummary {
+  return {
+    ...summary,
+    dispatch_id: canonicalDispatchId,
+    candidates: summary.candidates.map(c => ({
+      ...c,
+      input_snapshot: {
+        ...c.input_snapshot,
+        dispatch_id: canonicalDispatchId,
+      },
+    })),
+  };
+}

package/node_modules/@switchboard/core/src/overflow-manifest.ts

@@ -0,0 +1,220 @@
+import { randomBytes } from "crypto";
+import type { OverflowEntry, OverflowManifest, OverflowSummary, ProofArtifact } from "./types";
+import { overflowEntrySchema, overflowManifestSchema, overflowSummarySchema } from "./types";
+import { captureProofArtifact } from "./proof-capture";
+
+// ---------------------------------------------------------------------------
+// Overflow Manifest — progressive disclosure of externalized context
+// ---------------------------------------------------------------------------
+//
+// The overflow mechanism externalizes large context artifacts into governed
+// files (inside the existing proof folder) and retains a lightweight
+// reference + summary on the live control surface.
+//
+// This keeps prompts, receipts, and live events lean while preserving full
+// lineage to the externalized content for later evaluation lanes.
+//
+// Design invariants:
+//   - The threshold is content-size based, not content-type specific
+//   - Externalized artifacts reuse the existing proof-capture pipeline
+//   - The retained summary is always honest about what was externalized
+//   - The overflow manifest is itself a proof artifact (lineage preserved)
+//   - Detection is deterministic: same content → same externalization decision
+// ---------------------------------------------------------------------------
+
+// ---------------------------------------------------------------------------
+// Thresholds — when to externalize
+// ---------------------------------------------------------------------------
+//
+// These are intentionally generous for V1. The goal is to catch genuinely
+// bloated context (e.g. 100k+ char transcripts) without triggering on
+// normal-sized artifacts.
+//
+// The default threshold (50,000 chars ≈ ~12,500 tokens) is chosen because:
+//   - Claude context windows are typically 100k-200k tokens
+//   - A single artifact at 12.5k tokens is ~6-12% of the window
+//   - That's where inline inclusion starts to hurt more than help
+// ---------------------------------------------------------------------------
+
+const DEFAULT_THRESHOLD_BYTES = 50_000;
+const DEFAULT_RETAINED_SUMMARY_CHARS = 500;
+
+// ---------------------------------------------------------------------------
+// Detection — should this content be externalized?
+// ---------------------------------------------------------------------------
+
+/**
+ * Returns true if the content exceeds the externalization threshold.
+ *
+ * The threshold is in bytes (UTF-8). Content at or below the threshold
+ * stays inline. Content above is a candidate for externalization.
+ */
+export function shouldExternalize(
+  content: string,
+  thresholdBytes: number = DEFAULT_THRESHOLD_BYTES,
+): boolean {
+  if (!content) return false;
+  return Buffer.byteLength(content, "utf8") > thresholdBytes;
+}
+
+// ---------------------------------------------------------------------------
+// Summary extraction — what to retain inline
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract a retained summary from content that's being externalized.
+ *
+ * Keeps the first N chars as a preview, plus a tail snippet for context,
+ * separated by a marker indicating truncation.
+ */
+export function extractRetainedSummary(
+  content: string,
+  maxChars: number = DEFAULT_RETAINED_SUMMARY_CHARS,
+): string {
+  if (content.length <= maxChars) return content;
+
+  // Split: 70% head, 30% tail (with marker in between)
+  const headLen = Math.floor(maxChars * 0.7);
+  const tailLen = maxChars - headLen - 30; // 30 chars for the marker
+
+  const head = content.slice(0, headLen).trimEnd();
+  const tail = tailLen > 0 ? content.slice(-tailLen).trimStart() : "";
+  const marker = `\n\n... [${content.length - headLen - tailLen} chars externalized] ...\n\n`;
+
+  return head + marker + tail;
+}
+
+// ---------------------------------------------------------------------------
+// Externalize — write overflow to proof folder, return reference
+// ---------------------------------------------------------------------------
+
+/**
+ * Externalize overflow content into the proof folder.
+ *
+ * Returns the overflow entry and the proof artifact, or null if the
+ * content doesn't exceed the threshold (no externalization needed).
+ *
+ * The content is written as an `overflow-context` proof artifact,
+ * preserving full lineage through the proof manifest.
+ */
+export async function externalizeOverflow(input: {
+  proofDir: string;
+  sourceLabel: string;
+  content: string;
+  thresholdBytes?: number;
+  retainedSummaryChars?: number;
+}): Promise<{ entry: OverflowEntry; artifact: ProofArtifact } | null> {
+  const threshold = input.thresholdBytes ?? DEFAULT_THRESHOLD_BYTES;
+  if (!shouldExternalize(input.content, threshold)) return null;
+
+  const originalSize = Buffer.byteLength(input.content, "utf8");
+  const retainedSummary = extractRetainedSummary(
+    input.content,
+    input.retainedSummaryChars ?? DEFAULT_RETAINED_SUMMARY_CHARS,
+  );
+  const retainedBytes = Buffer.byteLength(retainedSummary, "utf8");
+
+  // Sanitize source label for use in filenames
+  const safeName = input.sourceLabel.replace(/[^a-z0-9_-]/gi, "-").toLowerCase();
+  const filename = `overflow-${safeName}.txt`;
+
+  // Write to proof folder via the existing proof-capture pipeline
+  const artifact = await captureProofArtifact({
+    proofDir: input.proofDir,
+    kind: "overflow-context",
+    provenance: "captured",
+    filename,
+    content: input.content,
+    description: `Overflow: ${input.sourceLabel} externalized (${originalSize} bytes > ${threshold} byte threshold)`,
+  });
+
+  const entry = overflowEntrySchema.parse({
+    overflow_id: `ovfl-${randomBytes(6).toString("hex")}`,
+    source_label: input.sourceLabel,
+    reason: `Content size ${originalSize} bytes exceeds ${threshold} byte threshold.`,
+    artifact_filename: filename,
+    artifact_id: artifact.artifact_id,
+    original_size_bytes: originalSize,
+    retained_summary: retainedSummary,
+    retained_bytes: retainedBytes,
+    externalized_at: new Date().toISOString(),
+  });
+
+  return { entry, artifact };
+}
+
+// ---------------------------------------------------------------------------
+// Manifest builder — assemble from collected entries
+// ---------------------------------------------------------------------------
+
+export function buildOverflowManifest(input: {
+  dispatchId: string;
+  entries: OverflowEntry[];
+}): OverflowManifest {
+  const totalBytes = input.entries.reduce(
+    (sum, e) => sum + e.original_size_bytes,
+    0,
+  );
+
+  return overflowManifestSchema.parse({
+    manifest_id: `ovfl-mfst-${randomBytes(6).toString("hex")}`,
+    dispatch_id: input.dispatchId,
+    entries: input.entries,
+    total_externalized_bytes: totalBytes,
+    total_entries: input.entries.length,
+    created_at: new Date().toISOString(),
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Persistence — write overflow manifest as a proof artifact
+// ---------------------------------------------------------------------------
+
+export async function captureOverflowManifest(input: {
+  proofDir: string;
+  manifest: OverflowManifest;
+}): Promise<ProofArtifact> {
+  const YAML = await import("yaml");
+  return captureProofArtifact({
+    proofDir: input.proofDir,
+    kind: "overflow-context",
+    provenance: "derived",
+    filename: "overflow-manifest.yaml",
+    content: YAML.stringify(input.manifest),
+    description: `Overflow manifest: ${input.manifest.total_entries} entries, ${input.manifest.total_externalized_bytes} bytes externalized`,
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Summary — lightweight fields for embedding on receipts / live events
+// ---------------------------------------------------------------------------
+
+/**
+ * Derive a lightweight overflow summary suitable for embedding on
+ * audited receipts and live events.
+ */
+export function deriveOverflowSummary(
+  manifest: OverflowManifest | null,
+): OverflowSummary {
+  if (!manifest || manifest.entries.length === 0) {
+    return overflowSummarySchema.parse({
+      overflow_entry_count: 0,
+      overflow_total_bytes: 0,
+      overflow_sources: [],
+      overflow_available: false,
+    });
+  }
+
+  return overflowSummarySchema.parse({
+    overflow_entry_count: manifest.total_entries,
+    overflow_total_bytes: manifest.total_externalized_bytes,
+    overflow_sources: manifest.entries.map(e => e.source_label),
+    overflow_available: true,
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Re-exports of thresholds for testing / configuration
+// ---------------------------------------------------------------------------
+
+export { DEFAULT_THRESHOLD_BYTES, DEFAULT_RETAINED_SUMMARY_CHARS };