selftune 0.1.0

package/cli/selftune/utils/llm-call.ts

@@ -0,0 +1,192 @@
+/**
+ * Shared LLM call utility.
+ *
+ * Provides a unified interface for calling LLMs via agent subprocess
+ * (claude/codex/opencode) or via the direct Anthropic API. Extracted from
+ * grade-session.ts so other modules can reuse the same calling logic.
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { AGENT_CANDIDATES, API_URL, MODEL } from "../constants.js";
+
+// ---------------------------------------------------------------------------
+// Agent detection
+// ---------------------------------------------------------------------------
+
+/** Detect first available agent CLI in PATH. */
+export function detectAgent(): string | null {
+  for (const agent of AGENT_CANDIDATES) {
+    if (Bun.which(agent)) return agent;
+  }
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Markdown fence stripping
+// ---------------------------------------------------------------------------
+
+/** Strip markdown code fences from LLM response text. */
+export function stripMarkdownFences(raw: string): string {
+  let text = raw.trim();
+
+  // Strip fence layers (handles nested fences by repeating)
+  let stripped = true;
+  while (stripped) {
+    stripped = false;
+    const fenceMatch = text.match(/(`{3,})/);
+    if (fenceMatch) {
+      const fence = fenceMatch[0]; // e.g. "```" or "````"
+      const fenceStart = fenceMatch.index ?? 0;
+      // Jump to the fence
+      let inner = text.slice(fenceStart);
+      // Remove opening fence line (```json or ```)
+      const newlineIdx = inner.indexOf("\n");
+      inner = newlineIdx >= 0 ? inner.slice(newlineIdx + 1) : inner.slice(fence.length);
+      // Find matching closing fence (same length of backticks on its own line)
+      const closingPattern = new RegExp(`^${fence.replace(/`/g, "\\`")}\\s*$`, "m");
+      const closingMatch = inner.match(closingPattern);
+      if (closingMatch && closingMatch.index != null) {
+        inner = inner.slice(0, closingMatch.index);
+      }
+      const result = inner.trim();
+      if (result !== text) {
+        text = result;
+        stripped = true;
+      }
+    }
+  }
+
+  // Find first { in case there's preamble text
+  const braceIdx = text.indexOf("{");
+  if (braceIdx > 0) {
+    text = text.slice(braceIdx);
+  }
+
+  return text;
+}
+
+// ---------------------------------------------------------------------------
+// Call LLM via agent subprocess
+// ---------------------------------------------------------------------------
+
+/** Call LLM via agent subprocess (claude/codex/opencode). Returns raw text. */
+export async function callViaAgent(
+  systemPrompt: string,
+  userPrompt: string,
+  agent: string,
+): Promise<string> {
+  // Write prompt to temp file to avoid shell quoting issues
+  const promptFile = join(tmpdir(), `selftune-llm-${Date.now()}.txt`);
+  writeFileSync(promptFile, `${systemPrompt}\n\n${userPrompt}`, "utf-8");
+
+  try {
+    const promptContent = readFileSync(promptFile, "utf-8");
+    let cmd: string[];
+
+    if (agent === "claude") {
+      cmd = ["claude", "-p", promptContent];
+    } else if (agent === "codex") {
+      cmd = ["codex", "exec", "--skip-git-repo-check", promptContent];
+    } else if (agent === "opencode") {
+      cmd = ["opencode", "-p", promptContent, "-f", "text", "-q"];
+    } else {
+      throw new Error(`Unknown agent: ${agent}`);
+    }
+
+    const proc = Bun.spawn(cmd, {
+      stdout: "pipe",
+      stderr: "pipe",
+      env: { ...process.env, CLAUDECODE: "" },
+    });
+
+    // 120s timeout
+    const timeout = setTimeout(() => proc.kill(), 120_000);
+    const exitCode = await proc.exited;
+    clearTimeout(timeout);
+
+    if (exitCode !== 0) {
+      const stderr = await new Response(proc.stderr).text();
+      throw new Error(
+        `Agent '${agent}' exited with code ${exitCode}.\nstderr: ${stderr.slice(0, 500)}`,
+      );
+    }
+
+    const raw = await new Response(proc.stdout).text();
+    return raw;
+  } finally {
+    try {
+      const { unlinkSync } = await import("node:fs");
+      unlinkSync(promptFile);
+    } catch {
+      // ignore cleanup errors
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Call LLM via direct Anthropic API
+// ---------------------------------------------------------------------------
+
+/** Call LLM via direct Anthropic API. Returns raw text. */
+export async function callViaApi(systemPrompt: string, userPrompt: string): Promise<string> {
+  const apiKey = process.env.ANTHROPIC_API_KEY ?? "";
+  if (!apiKey) {
+    throw new Error(
+      "ANTHROPIC_API_KEY not set. Use --use-agent to grade via your " +
+        "installed Claude Code / Codex / OpenCode subscription instead.",
+    );
+  }
+
+  const payload = {
+    model: MODEL,
+    max_tokens: 2000,
+    system: systemPrompt,
+    messages: [{ role: "user", content: userPrompt }],
+  };
+
+  const resp = await fetch(API_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "x-api-key": apiKey,
+      "anthropic-version": "2023-06-01",
+    },
+    body: JSON.stringify(payload),
+  });
+
+  if (!resp.ok) {
+    const body = await resp.text();
+    throw new Error(`API error ${resp.status}: ${body}`);
+  }
+
+  const data = await resp.json();
+  let raw = "";
+  for (const block of data.content ?? []) {
+    if (block.type === "text") raw += block.text ?? "";
+  }
+
+  return raw;
+}
+
+// ---------------------------------------------------------------------------
+// Unified dispatcher
+// ---------------------------------------------------------------------------
+
+/** Dispatch to callViaAgent or callViaApi. Returns raw text. */
+export async function callLlm(
+  systemPrompt: string,
+  userPrompt: string,
+  mode: "agent" | "api",
+  agent?: string,
+): Promise<string> {
+  if (mode === "agent") {
+    if (!agent) {
+      throw new Error("Agent must be specified when mode is 'agent'");
+    }
+    return callViaAgent(systemPrompt, userPrompt, agent);
+  }
+  return callViaApi(systemPrompt, userPrompt);
+}

package/cli/selftune/utils/logging.ts

@@ -0,0 +1,40 @@
+/**
+ * Structured JSON logging to stderr.
+ */
+
+export interface Logger {
+  info(message: string): void;
+  warn(message: string): void;
+  error(message: string, err?: unknown): void;
+}
+
+/**
+ * Create a structured JSON logger for a selftune module.
+ * Output goes to stderr as one JSON line per log call.
+ */
+export function createLogger(module: string): Logger {
+  function emit(level: string, message: string, exception?: string): void {
+    const entry: Record<string, string> = {
+      timestamp: new Date().toISOString(),
+      level,
+      module,
+      message,
+    };
+    if (exception) entry.exception = exception;
+    process.stderr.write(`${JSON.stringify(entry)}\n`);
+  }
+
+  return {
+    info(message: string) {
+      emit("INFO", message);
+    },
+    warn(message: string) {
+      emit("WARN", message);
+    },
+    error(message: string, err?: unknown) {
+      const exception =
+        err instanceof Error ? `${err.name}: ${err.message}\n${err.stack}` : undefined;
+      emit("ERROR", message, exception);
+    },
+  };
+}

package/cli/selftune/utils/schema-validator.ts

@@ -0,0 +1,47 @@
+/**
+ * JSONL schema validator for selftune log records.
+ * Validates records against REQUIRED_FIELDS from constants.
+ */
+
+import { REQUIRED_FIELDS } from "../constants.js";
+
+export type LogType = "session_telemetry" | "skill_usage" | "all_queries";
+
+export interface ValidationResult {
+  valid: boolean;
+  errors: string[];
+}
+
+/** Fields that must be strings when present. */
+const STRING_FIELDS = new Set(["timestamp", "session_id", "query", "skill_name", "source"]);
+
+/**
+ * Validate a record against the schema for the given log type.
+ * Checks field presence and basic type constraints.
+ */
+export function validateRecord(record: unknown, logType: LogType): ValidationResult {
+  const errors: string[] = [];
+
+  if (record === null || typeof record !== "object" || Array.isArray(record)) {
+    errors.push("Record must be a non-null object");
+    return { valid: false, errors };
+  }
+
+  const rec = record as Record<string, unknown>;
+  const requiredFields = REQUIRED_FIELDS[logType];
+
+  if (!requiredFields) {
+    errors.push(`Unknown log type: ${logType}`);
+    return { valid: false, errors };
+  }
+
+  for (const field of requiredFields) {
+    if (!(field in rec)) {
+      errors.push(`Missing required field: ${field}`);
+    } else if (STRING_FIELDS.has(field) && typeof rec[field] !== "string") {
+      errors.push(`Field "${field}" must be a string, got ${typeof rec[field]}`);
+    }
+  }
+
+  return { valid: errors.length === 0, errors };
+}

package/cli/selftune/utils/seeded-random.ts

@@ -0,0 +1,31 @@
+/**
+ * Deterministic PRNG using mulberry32 algorithm.
+ * Used for reproducible eval set shuffling.
+ */
+
+/**
+ * Create a seeded random number generator (mulberry32).
+ * Returns a function that produces numbers in [0, 1).
+ */
+export function mulberry32(seed: number): () => number {
+  let s = seed | 0;
+  return () => {
+    s = (s + 0x6d2b79f5) | 0;
+    let t = Math.imul(s ^ (s >>> 15), 1 | s);
+    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+/**
+ * Fisher-Yates shuffle using a seeded PRNG.
+ */
+export function seededShuffle<T>(array: T[], seed: number): T[] {
+  const result = [...array];
+  const rng = mulberry32(seed);
+  for (let i = result.length - 1; i > 0; i--) {
+    const j = Math.floor(rng() * (i + 1));
+    [result[i], result[j]] = [result[j], result[i]];
+  }
+  return result;
+}

package/cli/selftune/utils/transcript.ts

@@ -0,0 +1,260 @@
+/**
+ * Transcript parsing utilities shared by hooks and grading.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import { basename, dirname } from "node:path";
+import type { TranscriptMetrics } from "../types.js";
+
+/**
+ * Parse a Claude Code transcript JSONL and extract process metrics.
+ *
+ * Handles two observed transcript variants:
+ *   Variant A (newer): {"type": "user", "message": {"role": "user", "content": [...]}}
+ *   Variant B (older): {"role": "user", "content": "..."}
+ */
+export function parseTranscript(transcriptPath: string): TranscriptMetrics {
+  if (!existsSync(transcriptPath)) return emptyMetrics();
+
+  const content = readFileSync(transcriptPath, "utf-8");
+  const lines = content.split("\n");
+  const totalChars = lines.reduce((sum, l) => sum + l.length, 0);
+
+  const toolCalls: Record<string, number> = {};
+  const bashCommands: string[] = [];
+  const skillsTriggered: string[] = [];
+  let errors = 0;
+  let assistantTurns = 0;
+  let lastUserQuery = "";
+
+  for (const raw of lines) {
+    const line = raw.trim();
+    if (!line) continue;
+
+    let entry: Record<string, unknown>;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+
+    // Normalise: unwrap nested message if present
+    const msg = (entry.message as Record<string, unknown>) ?? entry;
+    const role = (msg.role as string) ?? (entry.role as string) ?? "";
+    const content = msg.content ?? entry.content ?? "";
+
+    // Track last user query
+    if (role === "user") {
+      if (typeof content === "string" && content.trim()) {
+        lastUserQuery = content.trim();
+      } else if (Array.isArray(content)) {
+        const texts = content
+          .filter(
+            (p): p is Record<string, unknown> =>
+              typeof p === "object" && p !== null && (p as Record<string, unknown>).type === "text",
+          )
+          .map((p) => (p.text as string) ?? "")
+          .filter(Boolean);
+        const text = texts.join(" ").trim();
+        if (text) lastUserQuery = text;
+      }
+    }
+
+    // Count assistant turns and parse tool use
+    if (role === "assistant") {
+      assistantTurns++;
+      const contentBlocks = Array.isArray(content) ? content : [];
+      for (const block of contentBlocks) {
+        if (typeof block !== "object" || block === null) continue;
+        const b = block as Record<string, unknown>;
+        if (b.type === "tool_use") {
+          const toolName = (b.name as string) ?? "Unknown";
+          toolCalls[toolName] = (toolCalls[toolName] ?? 0) + 1;
+          const inp = (b.input as Record<string, unknown>) ?? {};
+
+          // Track SKILL.md reads
+          const filePath = (inp.file_path as string) ?? "";
+          if (basename(filePath).toUpperCase() === "SKILL.MD") {
+            const skillName = basename(dirname(filePath));
+            if (!skillsTriggered.includes(skillName)) {
+              skillsTriggered.push(skillName);
+            }
+          }
+
+          // Track bash commands
+          if (toolName === "Bash") {
+            const cmd = ((inp.command as string) ?? "").trim();
+            if (cmd) bashCommands.push(cmd);
+          }
+        }
+      }
+    }
+
+    // Count tool errors from result entries
+    const entryType = entry.type as string;
+    if (entryType === "tool_result" && entry.is_error) {
+      errors++;
+    }
+    // Also check inside user content (tool_result blocks)
+    if (role === "user" && Array.isArray(content)) {
+      for (const block of content) {
+        if (
+          typeof block === "object" &&
+          block !== null &&
+          (block as Record<string, unknown>).type === "tool_result" &&
+          (block as Record<string, unknown>).is_error
+        ) {
+          errors++;
+        }
+      }
+    }
+  }
+
+  return {
+    tool_calls: toolCalls,
+    total_tool_calls: Object.values(toolCalls).reduce((a, b) => a + b, 0),
+    bash_commands: bashCommands,
+    skills_triggered: skillsTriggered,
+    assistant_turns: assistantTurns,
+    errors_encountered: errors,
+    transcript_chars: totalChars,
+    last_user_query: lastUserQuery,
+  };
+}
+
+/**
+ * Walk the transcript JSONL backwards to find the most recent user message.
+ */
+export function getLastUserMessage(transcriptPath: string): string | null {
+  if (!existsSync(transcriptPath)) return null;
+
+  try {
+    const content = readFileSync(transcriptPath, "utf-8");
+    const lines = content.trim().split("\n");
+
+    for (let i = lines.length - 1; i >= 0; i--) {
+      let entry: Record<string, unknown>;
+      try {
+        entry = JSON.parse(lines[i]);
+      } catch {
+        continue;
+      }
+
+      // Format 1: top-level role field
+      if (entry.role === "user") {
+        const text = extractUserText(entry.content);
+        if (text) return text;
+      }
+
+      // Format 2: nested message object
+      const msg = entry.message as Record<string, unknown> | undefined;
+      if (msg && typeof msg === "object" && msg.role === "user") {
+        const text = extractUserText(msg.content);
+        if (text) return text;
+      }
+    }
+  } catch {
+    // silent
+  }
+
+  return null;
+}
+
+/**
+ * Parse a transcript into a human-readable excerpt for the grader.
+ */
+export function readExcerpt(transcriptPath: string, maxChars = 8000): string {
+  if (!existsSync(transcriptPath)) return "(transcript not found)";
+
+  const content = readFileSync(transcriptPath, "utf-8");
+  const lines = content.trim().split("\n");
+  const readable: string[] = [];
+
+  for (const raw of lines) {
+    const line = raw.trim();
+    if (!line) continue;
+
+    let entry: Record<string, unknown>;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+
+    const msg = (entry.message as Record<string, unknown>) ?? entry;
+    const role = (msg.role as string) ?? (entry.role as string) ?? "";
+    const entryContent = msg.content ?? entry.content ?? "";
+
+    if (role === "user") {
+      if (typeof entryContent === "string") {
+        readable.push(`[USER] ${entryContent.slice(0, 200)}`);
+      } else if (Array.isArray(entryContent)) {
+        const texts = entryContent
+          .filter(
+            (p): p is Record<string, unknown> =>
+              typeof p === "object" && p !== null && (p as Record<string, unknown>).type === "text",
+          )
+          .map((p) => (p.text as string) ?? "")
+          .filter(Boolean);
+        const text = texts.join(" ").trim().slice(0, 200);
+        if (text) readable.push(`[USER] ${text}`);
+      }
+    } else if (role === "assistant") {
+      if (Array.isArray(entryContent)) {
+        for (const block of entryContent) {
+          if (typeof block !== "object" || block === null) continue;
+          const b = block as Record<string, unknown>;
+          if (b.type === "text") {
+            readable.push(`[ASSISTANT] ${((b.text as string) ?? "").slice(0, 200)}`);
+          } else if (b.type === "tool_use") {
+            const name = (b.name as string) ?? "?";
+            const inp = (b.input as Record<string, unknown>) ?? {};
+            const detail =
+              (inp.file_path as string) ??
+              (inp.command as string) ??
+              (inp.query as string) ??
+              JSON.stringify(inp).slice(0, 100);
+            readable.push(`[TOOL:${name}] ${detail}`);
+          }
+        }
+      }
+    }
+  }
+
+  const full = readable.join("\n");
+  if (full.length <= maxChars) return full;
+  const head = Math.floor(maxChars * 0.6);
+  const tail = maxChars - head;
+  return `${full.slice(0, head)}\n\n... [truncated] ...\n\n${full.slice(-tail)}`;
+}
+
+function emptyMetrics(): TranscriptMetrics {
+  return {
+    tool_calls: {},
+    total_tool_calls: 0,
+    bash_commands: [],
+    skills_triggered: [],
+    assistant_turns: 0,
+    errors_encountered: 0,
+    transcript_chars: 0,
+    last_user_query: "",
+  };
+}
+
+function extractUserText(content: unknown): string | null {
+  if (typeof content === "string" && content.trim()) {
+    return content.trim();
+  }
+  if (Array.isArray(content)) {
+    const texts = content
+      .filter(
+        (p): p is Record<string, unknown> =>
+          typeof p === "object" && p !== null && (p as Record<string, unknown>).type === "text",
+      )
+      .map((p) => (p.text as string) ?? "")
+      .filter(Boolean);
+    const combined = texts.join(" ").trim();
+    if (combined) return combined;
+  }
+  return null;
+}

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "selftune",
+  "version": "0.1.0",
+  "description": "Skill observability and continuous improvement CLI for agent platforms",
+  "type": "module",
+  "author": "Daniel Petro",
+  "license": "MIT",
+  "bin": {
+    "selftune": "bin/selftune.cjs"
+  },
+  "files": [
+    "bin/",
+    "cli/selftune/",
+    "skill/",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "lint": "bunx biome check .",
+    "lint:fix": "bunx biome check --write .",
+    "lint:arch": "bun run lint-architecture.ts",
+    "test": "bun test",
+    "check": "bun run lint && bun run lint:arch && bun test"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^1.9.4",
+    "@types/bun": "^1.1.0"
+  }
+}