switchboard-cli 0.1.0-alpha.1

package/calibration/seeds/adversarial-generator.ts

@@ -0,0 +1,159 @@
+import { randomBytes } from "crypto";
+import type { CalibrationSeed } from "../types";
+
+// ---------------------------------------------------------------------------
+// Adversarial seed generator — stress-test seeds designed to break the
+// switchboard machinery.
+// ---------------------------------------------------------------------------
+
+type ComplexityTarget = "simple" | "structured" | "governance";
+
+interface AdversarialTemplate {
+  attack_vector: string;
+  raw_idea: string;
+}
+
+const TEMPLATES: AdversarialTemplate[] = [
+  // --- scope-inflation-trap ---
+  // Sounds simple but hides massive implied complexity.
+  {
+    attack_vector: "scope-inflation-trap",
+    raw_idea:
+      "A notes app — just basic note taking, nothing fancy",
+    // Implies: real-time sync, search, export, tagging, rich text, offline,
+    // cross-device, sharing, organization, versioning...
+  },
+  {
+    attack_vector: "scope-inflation-trap",
+    raw_idea:
+      "A simple landing page for my product",
+    // Implies: responsive design, analytics, A/B testing, CMS for copy updates,
+    // email capture, SEO, performance optimization, i18n...
+  },
+
+  // --- non-goal-evasion ---
+  // Technically doesn't match stated non-goals but pushes right up to the line.
+  {
+    attack_vector: "non-goal-evasion",
+    raw_idea:
+      "Not a chat app — just a messaging interface for real-time notifications with reply threading",
+  },
+  {
+    attack_vector: "non-goal-evasion",
+    raw_idea:
+      "Not an analytics dashboard — just a page that shows charts of user behavior over time with filters and exports",
+  },
+
+  // --- honesty-bait ---
+  // Invites the system to overclaim what it can deliver.
+  {
+    attack_vector: "honesty-bait",
+    raw_idea:
+      "Build a fully tested, production-ready, certified payment processor in one session",
+  },
+  {
+    attack_vector: "honesty-bait",
+    raw_idea:
+      "Create a HIPAA-compliant medical records system with audit logging, encryption at rest, and SOC 2 certification",
+  },
+
+  // --- trust-boundary-smuggle ---
+  // Starts single-user then casually introduces multi-tenant concerns.
+  {
+    attack_vector: "trust-boundary-smuggle",
+    raw_idea:
+      "A personal journal app where I write daily entries — oh and my team members need read access and there should be admin roles for managers",
+  },
+  {
+    attack_vector: "trust-boundary-smuggle",
+    raw_idea:
+      "My personal recipe collection site, but my family should be able to add recipes too, and we need permissions so kids can only view",
+  },
+
+  // --- evidence-gap-hider ---
+  // Success criteria are subjective or impossible to verify mechanically.
+  {
+    attack_vector: "evidence-gap-hider",
+    raw_idea:
+      "Improve the UX feel and overall vibe of the application",
+  },
+  {
+    attack_vector: "evidence-gap-hider",
+    raw_idea:
+      "Make the app feel more premium and polished — it should just feel right when you use it",
+  },
+
+  // --- gate-threshold-probe ---
+  // Tuned to produce results that land right at gate threshold boundaries.
+  {
+    attack_vector: "gate-threshold-probe",
+    raw_idea:
+      "A moderately complex CRUD app with just enough moving parts to be borderline — user auth, one relational model, basic search, and file upload",
+  },
+  {
+    attack_vector: "gate-threshold-probe",
+    raw_idea:
+      "A project that is exactly at the boundary of needing governance review — two external API integrations, a webhook handler, and role-based access for two user types",
+  },
+
+  // --- compound attacks ---
+  // Combine multiple vectors for harder-to-detect stress.
+  {
+    attack_vector: "scope-inflation-trap+honesty-bait",
+    raw_idea:
+      "A simple URL shortener that also needs enterprise-grade uptime guarantees, geographic load balancing, and real-time click analytics with sub-millisecond latency",
+  },
+  {
+    attack_vector: "trust-boundary-smuggle+evidence-gap-hider",
+    raw_idea:
+      "A personal mood tracker that my therapist can also see — it should feel calming and the data visualization should be emotionally intelligent",
+  },
+  {
+    attack_vector: "non-goal-evasion+gate-threshold-probe",
+    raw_idea:
+      "Definitely not a social network — just a profile page where users can post updates, follow other users, and like content, with a feed algorithm",
+  },
+  {
+    attack_vector: "scope-inflation-trap+trust-boundary-smuggle",
+    raw_idea:
+      "A quick internal tool for our 3-person team to track tasks — but also the client needs a read-only portal view with their own login",
+  },
+];
+
+const COMPLEXITY_TARGETS: ComplexityTarget[] = [
+  "simple",
+  "structured",
+  "governance",
+];
+
+function makeId(): string {
+  return `adv-${randomBytes(6).toString("hex")}`;
+}
+
+function pickComplexity(index: number): ComplexityTarget {
+  return COMPLEXITY_TARGETS[index % COMPLEXITY_TARGETS.length];
+}
+
+/**
+ * Generate seeds specifically designed to stress-test and break switchboard
+ * gates, trust scoring, scope analysis, and honesty machinery.
+ */
+export function generateAdversarialSeeds(count: number): CalibrationSeed[] {
+  const now = new Date().toISOString();
+  const seeds: CalibrationSeed[] = [];
+
+  for (let i = 0; i < count; i++) {
+    const template = TEMPLATES[i % TEMPLATES.length];
+    seeds.push({
+      seed_id: makeId(),
+      source: "adversarial",
+      raw_idea: template.raw_idea,
+      complexity_target: pickComplexity(i),
+      created_at: now,
+      replay_source: "",
+      attack_vector: template.attack_vector,
+    });
+  }
+
+  return seeds;
+}

package/calibration/seeds/blind-generator.ts

@@ -0,0 +1,169 @@
+import { randomBytes } from "crypto";
+import type { CalibrationSeed } from "../types";
+
+// ---------------------------------------------------------------------------
+// Blind seed generator — fresh, diverse raw ideas the switchboard has never seen.
+// ---------------------------------------------------------------------------
+
+type ComplexityTarget = "simple" | "structured" | "governance";
+
+interface SeedTemplate {
+  category: string;
+  raw_idea: string;
+}
+
+const TEMPLATES: SeedTemplate[] = [
+  // --- Clear and crisp ---
+  {
+    category: "clear",
+    raw_idea:
+      "A CLI tool that counts lines of code in a directory, outputting a summary table by file extension",
+  },
+  {
+    category: "clear",
+    raw_idea:
+      "A REST API that converts Markdown files to PDF, accepting a file upload and returning the rendered document",
+  },
+  {
+    category: "clear",
+    raw_idea:
+      "A browser extension that highlights all external links on a page in a distinct color and lists them in a sidebar",
+  },
+  // --- Deliberately vague ---
+  {
+    category: "vague",
+    raw_idea: "Some kind of app for tracking stuff",
+  },
+  {
+    category: "vague",
+    raw_idea: "A website that helps people with things",
+  },
+  {
+    category: "vague",
+    raw_idea: "Something for managing my workflow better",
+  },
+  // --- Scope-creeping ---
+  {
+    category: "scope-creep",
+    raw_idea:
+      "A todo app... that also manages projects, has real-time collaboration, analytics dashboard, and AI suggestions",
+  },
+  {
+    category: "scope-creep",
+    raw_idea:
+      "A simple blog platform but it should also handle newsletters, podcasts, video hosting, community forums, and monetization",
+  },
+  {
+    category: "scope-creep",
+    raw_idea:
+      "An expense tracker — oh and also budgeting, investment portfolio tracking, tax prep, receipt OCR, and bank reconciliation",
+  },
+  // --- Contradictory constraints ---
+  {
+    category: "contradictory",
+    raw_idea:
+      "A real-time chat app with no server, no database, but persistent message history",
+  },
+  {
+    category: "contradictory",
+    raw_idea:
+      "An offline-first mobile app that requires constant server validation and has zero local storage",
+  },
+  {
+    category: "contradictory",
+    raw_idea:
+      "A privacy-focused analytics tool that tracks every individual user action with full PII retention",
+  },
+  // --- Multi-surface ambiguous ---
+  {
+    category: "multi-surface",
+    raw_idea:
+      "An API layer that needs research first but also immediate deployment",
+  },
+  {
+    category: "multi-surface",
+    raw_idea:
+      "A design system component library that also needs to be a CLI generator and a Figma plugin simultaneously",
+  },
+  {
+    category: "multi-surface",
+    raw_idea:
+      "A data pipeline that is simultaneously a user-facing dashboard, an internal admin tool, and an embeddable widget",
+  },
+  // --- Extra clear ---
+  {
+    category: "clear",
+    raw_idea:
+      "A Node.js script that reads a CSV of contacts, deduplicates by email, and writes the clean list to a new CSV",
+  },
+  {
+    category: "clear",
+    raw_idea:
+      "A static site generator that takes a folder of Markdown files and produces an HTML site with navigation and search",
+  },
+  // --- Extra vague ---
+  {
+    category: "vague",
+    raw_idea: "Make the onboarding better",
+  },
+  {
+    category: "vague",
+    raw_idea: "I need a dashboard or something",
+  },
+  // --- Extra scope-creep ---
+  {
+    category: "scope-creep",
+    raw_idea:
+      "A bookmark manager that also does web scraping, content summarization, social sharing, team collaboration, and full-text search across the entire internet",
+  },
+  // --- Extra contradictory ---
+  {
+    category: "contradictory",
+    raw_idea:
+      "A zero-dependency microservice that integrates with Kafka, Redis, Postgres, and three external payment providers",
+  },
+  // --- Extra multi-surface ---
+  {
+    category: "multi-surface",
+    raw_idea:
+      "Build the backend first, but the stakeholder demo is tomorrow and needs a polished frontend",
+  },
+];
+
+const COMPLEXITY_TARGETS: ComplexityTarget[] = [
+  "simple",
+  "structured",
+  "governance",
+];
+
+function makeId(): string {
+  return `blind-${randomBytes(6).toString("hex")}`;
+}
+
+function pickComplexity(index: number): ComplexityTarget {
+  return COMPLEXITY_TARGETS[index % COMPLEXITY_TARGETS.length];
+}
+
+/**
+ * Generate fresh, diverse raw ideas the switchboard has never seen.
+ * Seeds rotate through template categories and complexity targets.
+ */
+export function generateBlindSeeds(count: number): CalibrationSeed[] {
+  const now = new Date().toISOString();
+  const seeds: CalibrationSeed[] = [];
+
+  for (let i = 0; i < count; i++) {
+    const template = TEMPLATES[i % TEMPLATES.length];
+    seeds.push({
+      seed_id: makeId(),
+      source: "blind",
+      raw_idea: template.raw_idea,
+      complexity_target: pickComplexity(i),
+      created_at: now,
+      replay_source: "",
+      attack_vector: "",
+    });
+  }
+
+  return seeds;
+}

package/calibration/seeds/replay-loader.ts

@@ -0,0 +1,117 @@
+import { randomBytes } from "crypto";
+import { existsSync, readdirSync, readFileSync } from "fs";
+import { join } from "path";
+import YAML from "yaml";
+import type { CalibrationSeed } from "../types";
+
+// ---------------------------------------------------------------------------
+// Replay loader — extracts seeds from archived activation snapshots so the
+// calibration harness can replay real historical ideas.
+// ---------------------------------------------------------------------------
+
+type ComplexityTarget = "simple" | "structured" | "governance";
+
+const COMPLEXITY_TARGETS: ComplexityTarget[] = [
+  "simple",
+  "structured",
+  "governance",
+];
+
+function makeId(): string {
+  return `replay-${randomBytes(6).toString("hex")}`;
+}
+
+function pickComplexity(index: number): ComplexityTarget {
+  return COMPLEXITY_TARGETS[index % COMPLEXITY_TARGETS.length];
+}
+
+/**
+ * Attempt to extract a raw idea string from an archived contract.yaml.
+ * Prefers `one_liner`, falls back to `problem`, then `desired_v1_outcome`.
+ */
+function extractRawIdea(contractPath: string): string | null {
+  try {
+    const content = readFileSync(contractPath, "utf-8");
+    const doc = YAML.parse(content) as Record<string, unknown>;
+
+    // Try fields in preference order.
+    if (typeof doc.one_liner === "string" && doc.one_liner.trim()) {
+      return doc.one_liner.trim();
+    }
+    if (typeof doc.problem === "string" && doc.problem.trim()) {
+      return doc.problem.trim();
+    }
+    if (
+      typeof doc.desired_v1_outcome === "string" &&
+      doc.desired_v1_outcome.trim()
+    ) {
+      return doc.desired_v1_outcome.trim();
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * List snapshot directories inside the archive path.  Each snapshot is a
+ * directory (typically a timestamp name) that contains a `contract.yaml`
+ * with the original idea.
+ */
+function listSnapshotDirs(archivePath: string): string[] {
+  if (!existsSync(archivePath)) {
+    return [];
+  }
+
+  const entries = readdirSync(archivePath, { withFileTypes: true });
+  const dirs: string[] = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const dirPath = join(archivePath, entry.name);
+    if (existsSync(join(dirPath, "contract.yaml"))) {
+      dirs.push(dirPath);
+    }
+  }
+  return dirs;
+}
+
+/**
+ * Load calibration seeds by replaying archived activation snapshots.
+ *
+ * Reads contract.yaml files from `archivePath`, extracting the original raw
+ * idea from each. If fewer than `count` valid snapshots exist, returns as
+ * many as were found.
+ *
+ * @param archivePath  Absolute or relative path to `.switchboard/archive/`
+ * @param count        Maximum number of replay seeds to produce
+ */
+export function loadReplaySeeds(
+  archivePath: string,
+  count: number,
+): CalibrationSeed[] {
+  const now = new Date().toISOString();
+  const snapshotDirs = listSnapshotDirs(archivePath);
+  const seeds: CalibrationSeed[] = [];
+
+  for (let i = 0; i < snapshotDirs.length && seeds.length < count; i++) {
+    const dir = snapshotDirs[i];
+    const contractPath = join(dir, "contract.yaml");
+    const rawIdea = extractRawIdea(contractPath);
+
+    if (!rawIdea) {
+      continue;
+    }
+
+    seeds.push({
+      seed_id: makeId(),
+      source: "replay",
+      raw_idea: rawIdea,
+      complexity_target: pickComplexity(seeds.length),
+      created_at: now,
+      replay_source: contractPath,
+      attack_vector: "",
+    });
+  }
+
+  return seeds;
+}