replay-labs 0.1.0

package/src/review.js

@@ -0,0 +1,238 @@
+import { execFile } from "node:child_process";
+
+// Rubrics are the contract between the lab UI, the LLM reviewer, and the
+// heuristic fallback. Keyed by module id, then stage. Each criterion must be
+// observable in the learner's submission.
+export const RUBRICS = {
+  "runtime-boundary": {
+    repair: {
+      title: "Repair the naive voice page so you would ship it",
+      context:
+        "Next.js App Router page. Naive version calls browser-only APIs with no boundary or guards:\n" +
+        "export default function Page() {\n" +
+        "  const recognition = new window.SpeechRecognition();\n" +
+        "  localStorage.setItem(\"goals\", \"[]\");\n" +
+        "}",
+      criteria: [
+        { id: "boundary", name: "Browser behavior isolated behind a client boundary", required: true },
+        { id: "guards", name: "Browser APIs feature-checked before use", required: true },
+        { id: "unsupported", name: "Unsupported browser gets a designed state, not a crash", required: true },
+        { id: "denied", name: "Permission denial is a handled state", required: false },
+        { id: "verify", name: "Names how the failure states get verified", required: false }
+      ],
+      passRule: "all required criteria pass, plus at least one of the optional two",
+      intentNote:
+        "Pseudo-code and comments are VALID evidence for the design criteria (unsupported, denied, verify) — " +
+        "e.g. '// denied -> mic-help screen with retry' earns the denied criterion. " +
+        "The client boundary and the capability guards must be real code, not comments. " +
+        "Judge the thinking, not React fluency."
+    },
+    transfer: {
+      title: "Transfer the judgment to geolocation + camera + localStorage",
+      context:
+        "Scenario: a future AI session adds geolocation, camera capture, and localStorage to a " +
+        "Next.js dashboard. It works in the browser during development. The learner writes a plan.",
+      criteria: [
+        { id: "boundary", name: "Isolates the browser-capability work behind a client boundary", required: true },
+        { id: "guards", name: "Feature-detects geolocation/camera before use", required: true },
+        { id: "failure-states", name: "Designs denied/unsupported states up front", required: true },
+        { id: "no-overfit", name: "Reasons about runtime ownership, not 'use client' as magic syntax", required: false },
+        { id: "verify", name: "Verification beyond 'works in dev'", required: false }
+      ],
+      passRule: "all required criteria pass, plus at least one of the optional two"
+    }
+  },
+  "secret-boundary": {
+    repair: {
+      title: "Move the Anthropic call behind a server boundary you would ship",
+      context:
+        "Naive version: a client module with NEXT_PUBLIC_ANTHROPIC_KEY calling api.anthropic.com " +
+        "directly from the browser. The learner rewrites it (API route sketch plus what the client " +
+        "now calls). Next.js App Router conventions apply.",
+      criteria: [
+        { id: "route", name: "A server-side route owns the provider call", required: true },
+        { id: "secret", name: "Secret comes from server-only env (no NEXT_PUBLIC anywhere)", required: true },
+        { id: "validation", name: "Request input is validated before use", required: true },
+        { id: "safe-errors", name: "Errors to the client never leak provider details or the key", required: false },
+        { id: "abuse", name: "Considers abuse: rate, size, or token caps", required: false }
+      ],
+      passRule: "all required criteria pass, plus at least one of the optional two",
+      intentNote:
+        "Pseudo-code and comments are VALID evidence for safe-errors and abuse — " +
+        "e.g. '// 429 after 20 req/min per IP' earns the abuse criterion. " +
+        "Route ownership, the server-only secret, and input validation must be real code. " +
+        "Judge the thinking, not framework fluency."
+    },
+    transfer: {
+      title: "Transfer the judgment to Stripe checkout + webhook",
+      context:
+        "Scenario: a future AI session adds Stripe checkout and a webhook that marks orders paid. " +
+        "Works with test keys in dev. The learner writes the plan that makes it shippable.",
+      criteria: [
+        { id: "secret", name: "Stripe secret key lives server-side only", required: true },
+        { id: "webhook", name: "Webhook is trusted via signature verification, not by URL secrecy", required: true },
+        { id: "validation", name: "Validates event/order data before marking anything paid", required: true },
+        { id: "idempotency", name: "Considers replayed/duplicate events (idempotency)", required: false },
+        { id: "verify", name: "Verification beyond 'works with test keys'", required: false }
+      ],
+      passRule: "all required criteria pass, plus at least one of the optional two"
+    }
+  }
+};
+
+export function getRubric(moduleId, stage) {
+  const mod = RUBRICS[moduleId] || RUBRICS["runtime-boundary"];
+  return mod[stage];
+}
+
+export function reviewPrompt(rubric, submission) {
+  return `You are the reviewer inside Replay, a lab that turns real AI coding sessions into
+professional judgment training. Review the learner's submission against the rubric.
+
+Be concrete and tough but fair, like a senior engineer reviewing a teammate's fix.
+Every note must reference what the learner actually wrote (or failed to write) —
+no generic praise, no hedging. A criterion passes only on evidence in the submission.
+
+TASK: ${rubric.title}
+CONTEXT:
+${rubric.context}
+
+RUBRIC (id | name | required):
+${rubric.criteria.map((c) => `${c.id} | ${c.name} | ${c.required}`).join("\n")}
+
+PASS RULE: ${rubric.passRule}
+${rubric.intentNote ? `EVIDENCE POLICY: ${rubric.intentNote}` : ""}
+
+LEARNER SUBMISSION:
+<<<
+${submission}
+>>>
+
+Respond with STRICT JSON only, no markdown fences:
+{"criteria":[{"id":"...","pass":true,"note":"one concrete sentence citing their submission"}],
+ "overall":"PASS"|"FAIL",
+ "summary":"2-3 sentences: the strongest thing they did and the most important gap",
+ "misconception":"the single misunderstanding their submission reveals, or null"}`;
+}
+
+const HEURISTICS = {
+  "runtime-boundary": {
+    repair: {
+      boundary: (s) => /['"]use client['"]|dynamic\s*\(.*ssr:\s*false/.test(s),
+      guards: (s) => /typeof window|in window|window\.SpeechRecognition\s*(\?\?|\|\|)|navigator\.|webkitSpeechRecognition/.test(s),
+      unsupported: (s) => /unsupported|not supported|fallback/i.test(s),
+      denied: (s) => /denied|permission|catch|onerror|NotAllowedError/i.test(s),
+      verify: (s) => /test|verify|check\b|assert|playwright|vitest|jest/i.test(s)
+    },
+    transfer: {
+      boundary: (s) => /client (component|boundary)|['"]use client['"]|isolate/i.test(s),
+      guards: (s) => /feature.detect|capabilit|typeof|in navigator|getUserMedia|permissions\.query/i.test(s),
+      "failure-states": (s) => /denied|unsupported|fallback|error state|graceful/i.test(s),
+      "no-overfit": (s) => /runtime|server|render|boundary|ownership/i.test(s),
+      verify: (s) => /test|verify|device|browser matrix|e2e|manual check/i.test(s)
+    }
+  },
+  "secret-boundary": {
+    repair: {
+      route: (s) => /app\/api|route\.(ts|js)|NextResponse|export (async )?function (POST|GET)/.test(s),
+      secret: (s) => !/NEXT_PUBLIC/.test(s) && /process\.env\./.test(s),
+      validation: (s) => /validat|typeof|\.length|schema|zod|400|invalid|trim\(/i.test(s),
+      "safe-errors": (s) => /try|catch|status\(5|generic|console\.error|safe/i.test(s),
+      abuse: (s) => /rate|limit|429|max_tokens|cap|size/i.test(s)
+    },
+    transfer: {
+      secret: (s) => /server|env|secret key.*(server|env)|never.*(client|browser)/i.test(s),
+      webhook: (s) => /signature|constructEvent|stripe-signature|verif/i.test(s),
+      validation: (s) => /validat|check|amount|schema|before marking/i.test(s),
+      idempotency: (s) => /idempoten|replay|duplicate|already processed/i.test(s),
+      verify: (s) => /test|verify|stripe cli|webhook.*local|e2e/i.test(s)
+    }
+  }
+};
+
+export function heuristicReview(moduleId, stage, submission) {
+  const rubric = getRubric(moduleId, stage);
+  const checks = (HEURISTICS[moduleId] || HEURISTICS["runtime-boundary"])[stage] || {};
+  const criteria = rubric.criteria.map((c) => {
+    const pass = Boolean(checks[c.id] && checks[c.id](submission));
+    return {
+      id: c.id,
+      pass,
+      note: pass
+        ? `Detected evidence for "${c.name}" (pattern match — real review needs the claude CLI on the server).`
+        : `No evidence found for "${c.name}".`
+    };
+  });
+  return {
+    criteria,
+    overall: computeOverall(rubric, criteria),
+    summary: "Heuristic review only: pattern-matching, not understanding. Run with the claude CLI available for a real reviewer.",
+    misconception: null,
+    reviewer: "heuristic"
+  };
+}
+
+export function computeOverall(rubric, criteria) {
+  const byId = Object.fromEntries(criteria.map((c) => [c.id, c.pass]));
+  const requiredOk = rubric.criteria.filter((c) => c.required).every((c) => byId[c.id]);
+  const optionalOk = rubric.criteria.filter((c) => !c.required).some((c) => byId[c.id]);
+  return requiredOk && optionalOk ? "PASS" : "FAIL";
+}
+
+export function reviewWithClaude(rubric, submission, { timeoutMs = 90000 } = {}) {
+  return new Promise((resolvePromise) => {
+    const child = execFile(
+      "claude",
+      ["-p", "--model", "sonnet"],
+      { timeout: timeoutMs, maxBuffer: 1024 * 1024 },
+      (error, stdout) => {
+        if (error) return resolvePromise(null);
+        const raw = stdout.trim();
+        const start = raw.indexOf("{");
+        const end = raw.lastIndexOf("}");
+        if (start === -1 || end === -1) return resolvePromise(null);
+        try {
+          const parsed = JSON.parse(raw.slice(start, end + 1));
+          if (!Array.isArray(parsed.criteria) || !parsed.overall) return resolvePromise(null);
+          // The model judges criteria; the pass rule stays deterministic.
+          parsed.overall = computeOverall(rubric, parsed.criteria);
+          parsed.reviewer = "claude";
+          resolvePromise(parsed);
+        } catch {
+          resolvePromise(null);
+        }
+      }
+    );
+    child.stdin.write(reviewPrompt(rubric, submission));
+    child.stdin.end();
+  });
+}
+
+export async function review(stage, submission, moduleId = "runtime-boundary", inlineRubric = null) {
+  // Generated labs carry their own rubric; hand-authored labs look it up by id.
+  const rubric = inlineRubric || getRubric(moduleId, stage);
+  if (!rubric) throw new Error(`Unknown review stage: ${moduleId}:${stage}`);
+  if (!submission || submission.trim().length < 10) {
+    return {
+      criteria: [],
+      overall: "FAIL",
+      summary: "Submission is empty. Write the repair (or plan) before asking for review.",
+      misconception: null,
+      reviewer: "validator"
+    };
+  }
+  const llm = await reviewWithClaude(rubric, submission);
+  if (llm) return llm;
+  // Heuristic fallback only knows hand-authored patterns. For generated labs with
+  // no heuristic, fail honestly rather than fake a pass.
+  if (!HEURISTICS[moduleId]) {
+    return {
+      criteria: rubric.criteria.map((c) => ({ id: c.id, pass: false, note: `Cannot judge "${c.name}" without the claude CLI — generated labs need a real reviewer.` })),
+      overall: "FAIL",
+      summary: "This is a generated lab; its review needs the claude CLI on the server. Start `replay-labs serve` where claude is available.",
+      misconception: null,
+      reviewer: "unavailable"
+    };
+  }
+  return heuristicReview(moduleId, stage, submission);
+}

package/src/server.js

@@ -0,0 +1,199 @@
+import { createServer } from "node:http";
+import { readFile } from "node:fs/promises";
+import { extname, join, normalize, resolve } from "node:path";
+import { review } from "./review.js";
+import { bestSessionFrom, discoverSessions, generateInboxHtml, loadDiscoveredSession } from "./discovery.js";
+import { buildSessionBundle, bundleSlug } from "./pipeline.js";
+
+const TYPES = {
+  ".html": "text/html; charset=utf-8",
+  ".js": "text/javascript; charset=utf-8",
+  ".css": "text/css; charset=utf-8",
+  ".json": "application/json; charset=utf-8",
+  ".md": "text/plain; charset=utf-8",
+  ".svg": "image/svg+xml",
+  ".png": "image/png"
+};
+
+export function startServer({ root = process.cwd(), port = 4177, host = "127.0.0.1", homeDir = undefined, artifactRoot = undefined } = {}) {
+  const base = resolve(root);
+  const labsBase = resolve(artifactRoot || resolve(base, "replay-built"));
+
+  const server = createServer(async (req, res) => {
+    if (req.method === "POST" && req.url === "/api/review") {
+      let body = "";
+      req.on("data", (chunk) => { body += chunk; if (body.length > 200000) req.destroy(); });
+      req.on("end", async () => {
+        try {
+          const { stage, submission, moduleId, rubric } = JSON.parse(body);
+          const inlineRubric = rubric && rubric[stage] ? rubric[stage] : null;
+          const result = await review(stage, submission, moduleId, inlineRubric);
+          res.writeHead(200, { "content-type": "application/json" });
+          res.end(JSON.stringify(result));
+        } catch (error) {
+          res.writeHead(400, { "content-type": "application/json" });
+          res.end(JSON.stringify({ error: error.message }));
+        }
+      });
+      return;
+    }
+
+    if (req.method === "GET" && (req.url === "/" || req.url === "" || req.url?.startsWith("/inbox"))) {
+      try {
+        const sessions = await discoverSessions({ limit: 80, homeDir });
+        res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
+        res.end(generateInboxHtml(sessions, { interactive: true }));
+      } catch (error) {
+        json(res, 500, { error: error.message });
+      }
+      return;
+    }
+
+    if (req.method === "GET" && req.url?.startsWith("/api/sessions")) {
+      try {
+        const sessions = await discoverSessions({ limit: 80, homeDir });
+        json(res, 200, { sessions });
+      } catch (error) {
+        json(res, 500, { error: error.message });
+      }
+      return;
+    }
+
+    if (req.method === "POST" && req.url === "/api/choose-lab") {
+      readJson(req, async (error, body) => {
+        if (error) { json(res, 400, { error: error.message }); return; }
+        try {
+          const sessions = await discoverSessions({ limit: 300, homeDir });
+          const selected = bestSessionFrom(sessions);
+          if (!selected) throw new Error("No local Claude/Codex sessions found.");
+          const result = await buildFromSession({
+            base,
+            artifactRoot: labsBase,
+            sessionPath: selected.path,
+            generate: body?.generate === true || (selected.richLabs === 0 && selected.hasConcreteEvidence),
+            title: selected.title
+          });
+          json(res, 200, { ...result, title: selected.title, reason: selected.reason, sessionPath: selected.path });
+        } catch (buildError) {
+          json(res, 500, { error: buildError.message });
+        }
+      });
+      return;
+    }
+
+    if (req.method === "POST" && req.url === "/api/build-lab") {
+      readJson(req, async (error, body) => {
+        if (error) { json(res, 400, { error: error.message }); return; }
+        try {
+          if (!body?.sessionPath) throw new Error("Missing sessionPath.");
+          const result = await buildFromSession({
+            base,
+            artifactRoot: labsBase,
+            sessionPath: body.sessionPath,
+            generate: body.generate === true
+          });
+          json(res, 200, result);
+        } catch (buildError) {
+          json(res, 500, { error: buildError.message });
+        }
+      });
+      return;
+    }
+
+    if (req.method === "GET" && req.url === "/api/health") {
+      res.writeHead(200, { "content-type": "application/json" });
+      res.end(JSON.stringify({ ok: true }));
+      return;
+    }
+
+    const urlPath = decodeURIComponent((req.url || "/").split("?")[0]);
+    const safePath = normalize(urlPath).replace(/^(\.\.[/\\])+/, "");
+    let filePath = join(base, safePath);
+    if (safePath === "/replay-built" || safePath.startsWith("/replay-built/")) {
+      filePath = join(labsBase, safePath.replace(/^\/replay-built\/?/, ""));
+    }
+    if (!filePath.startsWith(base)) {
+      if (!filePath.startsWith(labsBase)) {
+        res.writeHead(403); res.end("forbidden"); return;
+      }
+    }
+    const candidates = [filePath];
+    for (const candidate of candidates) {
+      try {
+        const content = await readFile(candidate);
+        res.writeHead(200, { "content-type": TYPES[extname(candidate)] || "application/octet-stream" });
+        res.end(content);
+        return;
+      } catch { /* try next candidate */ }
+    }
+    res.writeHead(404, { "content-type": "text/plain" });
+    res.end(`not found: ${urlPath}`);
+  });
+
+  server.listen(port, host, () => {
+    const address = server.address();
+    const actualPort = typeof address === "object" && address ? address.port : port;
+    console.log(`Replay Labs: http://${host}:${actualPort}/ (root: ${base})`);
+    console.log(`session inbox: http://${host}:${actualPort}/inbox`);
+    console.log(`local app data: ${labsBase}`);
+    console.log("review endpoint: POST /api/review — uses the claude CLI when available, heuristic fallback otherwise");
+  });
+  return server;
+}
+
+async function buildFromSession({ base, artifactRoot, sessionPath, generate, title }) {
+  const loaded = await loadDiscoveredSession(sessionPath);
+  const slug = bundleSlug(title || loaded.goal || sessionPath);
+  const outDir = resolve(artifactRoot, slug);
+  const bundle = await buildSessionBundle({
+    goal: loaded.goal,
+    diff: loaded.diff,
+    transcript: loaded.transcript,
+    diffPath: sessionPath,
+    transcriptPath: sessionPath,
+    outDir,
+    generate,
+    maxGenerated: 1
+  });
+  const richCount = bundle.labs.filter((lab) => lab.rich).length;
+  const primaryLab = bundle.labs.find((lab) => lab.rich);
+  const hasDecisionSignals = bundle.labs.length > 0;
+  const href = "/replay-built/" + normalize(bundle.indexPath.slice(artifactRoot.length)).replace(/^[/\\]/, "").replaceAll("\\", "/");
+  const baseHref = href.replace(/\/index\.html$/, "");
+  return {
+    outDir,
+    indexPath: bundle.indexPath,
+    href,
+    primaryLabHref: primaryLab ? `${baseHref}/labs/${primaryLab.module.id}.html` : null,
+    labs: bundle.labs.length,
+    richLabs: richCount,
+    generated: Boolean(generate),
+    noReadyLabs: richCount === 0,
+    noDecisionSignals: !hasDecisionSignals,
+    message: richCount > 0
+      ? "Lab ready."
+      : hasDecisionSignals
+        ? "Replay Labs found decision signals, but not enough concrete code evidence to build a practice lab."
+        : "Replay could not find enough decision evidence in this session."
+  };
+}
+
+function readJson(req, callback) {
+  let body = "";
+  req.on("data", (chunk) => {
+    body += chunk;
+    if (body.length > 500000) req.destroy();
+  });
+  req.on("end", () => {
+    try {
+      callback(null, body ? JSON.parse(body) : {});
+    } catch (error) {
+      callback(error);
+    }
+  });
+}
+
+function json(res, status, payload) {
+  res.writeHead(status, { "content-type": "application/json" });
+  res.end(JSON.stringify(payload));
+}

package/src/storage.js

@@ -0,0 +1,34 @@
+import { mkdir } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export function appDataDir({ platform = process.platform, env = process.env, homeDir = homedir() } = {}) {
+  if (env.REPLAY_HOME) return env.REPLAY_HOME;
+  if (platform === "darwin") return join(homeDir, "Library", "Application Support", "Replay Labs");
+  if (platform === "win32") return join(env.APPDATA || join(homeDir, "AppData", "Roaming"), "Replay Labs");
+  return join(env.XDG_DATA_HOME || join(homeDir, ".local", "share"), "replay-labs");
+}
+
+export function appPaths(options = {}) {
+  const root = appDataDir(options);
+  return {
+    root,
+    indexDir: join(root, "index"),
+    labsDir: join(root, "labs"),
+    cacheDir: join(root, "cache"),
+    generatedModulesDir: join(root, "cache", "generated-modules"),
+    logsDir: join(root, "logs")
+  };
+}
+
+export async function ensureAppDataDirs(options = {}) {
+  const paths = appPaths(options);
+  await Promise.all([
+    mkdir(paths.indexDir, { recursive: true }),
+    mkdir(paths.labsDir, { recursive: true }),
+    mkdir(paths.generatedModulesDir, { recursive: true }),
+    mkdir(paths.logsDir, { recursive: true })
+  ]);
+  return paths;
+}
+