selftune 0.2.16 → 0.2.18

package/cli/selftune/evolution/validate-proposal.ts

@@ -7,7 +7,7 @@
  */
 
 import type { EvalEntry, EvolutionProposal, InvocationTypeScores } from "../types.js";
-import { callLlm } from "../utils/llm-call.js";
+import { callLlm, type EffortLevel } from "../utils/llm-call.js";
 import {
   buildBatchTriggerCheckPrompt,
   buildTriggerCheckPrompt,
@@ -52,6 +52,7 @@ export async function validateProposalSequential(
   evalSet: EvalEntry[],
   agent: string,
   modelFlag?: string,
+  effort?: EffortLevel,
 ): Promise<ValidationResult> {
   if (evalSet.length === 0) {
     return {
@@ -76,14 +77,14 @@ export async function validateProposalSequential(
   for (const entry of evalSet) {
     // Check with original description
     const beforePrompt = buildTriggerCheckPrompt(proposal.original_description, entry.query);
-    const beforeRaw = await callLlm(systemPrompt, beforePrompt, agent, modelFlag);
+    const beforeRaw = await callLlm(systemPrompt, beforePrompt, agent, modelFlag, effort);
     const beforeTriggered = parseTriggerResponse(beforeRaw);
     const beforePass =
       (entry.should_trigger && beforeTriggered) || (!entry.should_trigger && !beforeTriggered);
 
     // Check with proposed description
     const afterPrompt = buildTriggerCheckPrompt(proposal.proposed_description, entry.query);
-    const afterRaw = await callLlm(systemPrompt, afterPrompt, agent, modelFlag);
+    const afterRaw = await callLlm(systemPrompt, afterPrompt, agent, modelFlag, effort);
     const afterTriggered = parseTriggerResponse(afterRaw);
     const afterPass =
       (entry.should_trigger && afterTriggered) || (!entry.should_trigger && !afterTriggered);
@@ -208,6 +209,7 @@ export async function validateProposalBatched(
   evalSet: EvalEntry[],
   agent: string,
   modelFlag?: string,
+  effort?: EffortLevel,
 ): Promise<ValidationResult> {
   if (evalSet.length === 0) {
     return {
@@ -242,8 +244,8 @@ export async function validateProposalBatched(
     // Run VALIDATION_RUNS times in parallel and majority-vote to reduce LLM variance
     const allCalls: Promise<string>[] = [];
     for (let r = 0; r < VALIDATION_RUNS; r++) {
-      allCalls.push(callLlm(systemPrompt, beforePrompt, agent, modelFlag));
-      allCalls.push(callLlm(systemPrompt, afterPrompt, agent, modelFlag));
+      allCalls.push(callLlm(systemPrompt, beforePrompt, agent, modelFlag, effort));
+      allCalls.push(callLlm(systemPrompt, afterPrompt, agent, modelFlag, effort));
     }
     const allRaw = await Promise.all(allCalls);
 
@@ -353,6 +355,7 @@ export async function validateProposal(
   evalSet: EvalEntry[],
   agent: string,
   modelFlag?: string,
+  effort?: EffortLevel,
 ): Promise<ValidationResult> {
-  return validateProposalBatched(proposal, evalSet, agent, modelFlag);
+  return validateProposalBatched(proposal, evalSet, agent, modelFlag, effort);
 }

package/cli/selftune/grading/grade-session.ts

@@ -26,6 +26,7 @@ import type {
   GradingExpectation,
   GradingResult,
   SessionTelemetryRecord,
+  SessionType,
   SkillUsageRecord,
 } from "../types.js";
 import { CLIError, handleCLIError } from "../utils/cli-error.js";
@@ -420,6 +421,8 @@ export function buildExecutionMetrics(telemetry: SessionTelemetryRecord): Execut
     errors_encountered: telemetry.errors_encountered ?? 0,
     skills_triggered: telemetry.skills_triggered ?? [],
     transcript_chars: telemetry.transcript_chars ?? 0,
+    artifact_count: telemetry.artifact_count,
+    session_type: telemetry.session_type,
   };
 }
 
@@ -481,13 +484,30 @@ export function buildGradingPrompt(
       ? transcriptExcerpt.slice(0, MAX_TRANSCRIPT_LENGTH)
       : transcriptExcerpt;
 
+  const sessionType: SessionType = (telemetry.session_type as SessionType) ?? "mixed";
+  const SESSION_TYPE_CONTEXT: Record<SessionType, string> = {
+    dev: "This is a development session — code output and commits are expected productivity signals.",
+    research:
+      "This is a research session — information gathering and synthesis are the primary outputs, not code changes.",
+    content:
+      "This is a content/writing session — document creation is the primary output, not code commits.",
+    mixed:
+      "This is a mixed session — evaluate based on what was actually accomplished, not code-specific metrics.",
+  };
+  const sessionTypeContext = SESSION_TYPE_CONTEXT[sessionType] ?? SESSION_TYPE_CONTEXT.mixed;
+
   return `Skill: ${skillName}
 
+=== SESSION CONTEXT ===
+Session type: ${sessionType}
+${sessionTypeContext}
+
 === PROCESS TELEMETRY ===
 Skills triggered: ${JSON.stringify(telemetry.skills_triggered ?? [])}
 Assistant turns: ${telemetry.assistant_turns ?? "?"}
 Errors: ${telemetry.errors_encountered ?? "?"}
 Total tool calls: ${telemetry.total_tool_calls ?? "?"}
+Artifacts produced: ${telemetry.artifact_count ?? "?"}
 
 Tool breakdown:
 ${toolSummary}

package/cli/selftune/hooks/commit-track.ts

@@ -0,0 +1,188 @@
+#!/usr/bin/env bun
+/**
+ * PostToolUse hook: commit-track.ts
+ *
+ * Detects git commits in Bash tool output and records commit SHA, title,
+ * branch, and session ID for session-to-commit traceability.
+ *
+ * Fail-open: exits 0 on all errors. Never blocks the host agent.
+ */
+
+import { execSync } from "node:child_process";
+
+import type { PostToolUsePayload } from "../types.js";
+
+// -- Regex patterns (pre-compiled at module load) ----------------------------
+
+/** Matches git commands that produce commits. */
+const GIT_COMMIT_CMD_RE = /\bgit\s+(commit|merge|cherry-pick|revert)\b/;
+
+/**
+ * Matches the standard git commit output format: [branch SHA] title
+ * Supports optional parenthetical like (root-commit).
+ * Branch names can contain word chars, slashes, dots, hyphens, plus signs.
+ */
+const COMMIT_OUTPUT_RE = /\[([\w/.+-]+)(?:\s+\([^)]+\))?\s+([a-f0-9]{7,40})\]\s+(.+)/;
+
+// -- Pure extraction functions (exported for testability) ---------------------
+
+/** Check if a command string contains a git commit/merge/cherry-pick/revert. */
+export function containsGitCommitCommand(command: string): boolean {
+  return GIT_COMMIT_CMD_RE.test(command);
+}
+
+/** Extract commit SHA from git output. */
+export function parseCommitSha(output: string): string | undefined {
+  const match = output.match(COMMIT_OUTPUT_RE);
+  return match ? match[2] : undefined;
+}
+
+/** Extract commit title from git output. */
+export function parseCommitTitle(output: string): string | undefined {
+  const match = output.match(COMMIT_OUTPUT_RE);
+  return match ? match[3].trim() : undefined;
+}
+
+/** Extract branch name from git output. */
+export function parseBranchFromOutput(output: string): string | undefined {
+  const match = output.match(COMMIT_OUTPUT_RE);
+  return match ? match[1] : undefined;
+}
+
+/** Scrub credentials from a remote URL. Returns undefined for empty input. */
+export function scrubRemoteUrl(rawUrl: string): string | undefined {
+  if (!rawUrl) return undefined;
+  try {
+    const parsed = new URL(rawUrl);
+    parsed.username = "";
+    parsed.password = "";
+    return `${parsed.protocol}//${parsed.host}${parsed.pathname}`;
+  } catch {
+    // SSH or non-URL format — safe as-is
+    return rawUrl;
+  }
+}
+
+// -- Commit tracking record shape --------------------------------------------
+
+export interface CommitTrackRecord {
+  session_id: string;
+  commit_sha: string;
+  commit_title?: string;
+  branch?: string;
+  repo_remote?: string;
+  timestamp: string;
+}
+
+// -- Core processing logic ---------------------------------------------------
+
+/**
+ * Process a PostToolUse payload for git commit tracking.
+ * Returns the record that was written, or null if skipped.
+ * Exported for testability.
+ */
+export async function processCommitTrack(
+  payload: PostToolUsePayload,
+): Promise<CommitTrackRecord | null> {
+  // Fast-path: only care about Bash tool
+  if (payload.tool_name !== "Bash") return null;
+
+  // Fast-path: if cwd is known and not inside a git repo, skip
+  const cwd = payload.cwd ?? "";
+  if (cwd) {
+    try {
+      execSync("git rev-parse --is-inside-work-tree", {
+        cwd,
+        timeout: 1000,
+        stdio: ["ignore", "ignore", "ignore"],
+      });
+    } catch {
+      return null; // Not inside a git repo
+    }
+  }
+
+  // Fast-path: check if the command is a git commit-producing operation
+  const command = typeof payload.tool_input?.command === "string" ? payload.tool_input.command : "";
+  if (!containsGitCommitCommand(command)) return null;
+
+  // Extract stdout from tool_response
+  const response = payload.tool_response ?? {};
+  const stdout = typeof response.stdout === "string" ? response.stdout : "";
+  if (!stdout) return null;
+
+  // Parse commit SHA — if we can't find one, nothing to track
+  const commitSha = parseCommitSha(stdout);
+  if (!commitSha) return null;
+
+  const commitTitle = parseCommitTitle(stdout);
+  const outputBranch = parseBranchFromOutput(stdout);
+  const sessionId = payload.session_id;
+  if (!sessionId) return null; // No session ID — skip insert (fail-open)
+
+  // Try to get branch from git if not parsed from output
+  let branch = outputBranch;
+  if (!branch && cwd) {
+    try {
+      branch =
+        execSync("git rev-parse --abbrev-ref HEAD", {
+          cwd,
+          timeout: 3000,
+          stdio: ["ignore", "pipe", "ignore"],
+        })
+          .toString()
+          .trim() || undefined;
+    } catch {
+      /* not a git repo or git not available */
+    }
+  }
+
+  // Try to get remote URL (scrub credentials)
+  let repoRemote: string | undefined;
+  if (cwd) {
+    try {
+      const rawRemote =
+        execSync("git remote get-url origin", {
+          cwd,
+          timeout: 3000,
+          stdio: ["ignore", "pipe", "ignore"],
+        })
+          .toString()
+          .trim() || undefined;
+      if (rawRemote) {
+        repoRemote = scrubRemoteUrl(rawRemote);
+      }
+    } catch {
+      /* no remote configured */
+    }
+  }
+
+  const record: CommitTrackRecord = {
+    session_id: sessionId,
+    commit_sha: commitSha,
+    commit_title: commitTitle,
+    branch,
+    repo_remote: repoRemote,
+    timestamp: new Date().toISOString(),
+  };
+
+  // Write to SQLite (dynamic import to reduce hook startup cost)
+  try {
+    const { writeCommitTracking } = await import("../localdb/direct-write.js");
+    writeCommitTracking(record);
+  } catch {
+    /* hooks must never block */
+  }
+
+  return record;
+}
+
+// --- stdin main (only when executed directly, not when imported) ---
+if (import.meta.main) {
+  try {
+    const payload: PostToolUsePayload = JSON.parse(await Bun.stdin.text());
+    await processCommitTrack(payload);
+  } catch {
+    // silent — hooks must never block Claude
+  }
+  process.exit(0);
+}

package/cli/selftune/hooks/prompt-log.ts

@@ -226,7 +226,16 @@ export async function processPrompt(
 // --- stdin main (only when executed directly, not when imported) ---
 if (import.meta.main) {
   try {
-    const payload: PromptSubmitPayload = JSON.parse(await Bun.stdin.text());
+    const { readStdinWithPreview } = await import("./stdin-preview.js");
+    const { preview, full } = await readStdinWithPreview();
+
+    // Fast-path: prompt-log only handles UserPromptSubmit events.
+    // If the keyword is absent from the first 4 KiB we can skip JSON.parse entirely.
+    if (!preview.includes('"UserPromptSubmit"')) {
+      process.exit(0);
+    }
+
+    const payload: PromptSubmitPayload = JSON.parse(full);
     await processPrompt(payload);
   } catch {
     // silent — hooks must never block Claude

package/cli/selftune/hooks/session-stop.ts

@@ -11,7 +11,7 @@
 import { execSync } from "node:child_process";
 import { readFileSync } from "node:fs";
 
-import { CANONICAL_LOG, ORCHESTRATE_LOCK, TELEMETRY_LOG } from "../constants.js";
+import { CANONICAL_LOG, getOrchestrateLockPath, TELEMETRY_LOG } from "../constants.js";
 import {
   appendCanonicalRecords,
   buildCanonicalExecutionFact,
@@ -47,7 +47,7 @@ function hasFreshOrchestrateLock(lockPath: string): boolean {
  * Returns true if a process was spawned, false otherwise.
  */
 export async function maybeSpawnReactiveOrchestrate(
-  lockPath: string = ORCHESTRATE_LOCK,
+  lockPath: string = getOrchestrateLockPath(),
   deps: ReactiveSpawnDeps = {},
 ): Promise<boolean> {
   try {

package/cli/selftune/hooks/skill-eval.ts

@@ -359,7 +359,21 @@ async function processSkillToolUse(
 // --- stdin main (only when executed directly, not when imported) ---
 if (import.meta.main) {
   try {
-    const payload: PostToolUsePayload = JSON.parse(await Bun.stdin.text());
+    const { readStdinWithPreview } = await import("./stdin-preview.js");
+    const { preview, full } = await readStdinWithPreview();
+
+    // Fast-path: skill-eval only handles PostToolUse events.
+    if (!preview.includes('"PostToolUse"')) {
+      process.exit(0);
+    }
+
+    // Secondary fast-path: only Read and Skill tools are relevant.
+    // Most PostToolUse events are for Bash/Write/Edit — skip those entirely.
+    if (!preview.includes('"Read"') && !preview.includes('"Skill"')) {
+      process.exit(0);
+    }
+
+    const payload: PostToolUsePayload = JSON.parse(full);
     await processToolUse(payload);
   } catch {
     // silent — hooks must never block Claude

package/cli/selftune/hooks/stdin-preview.ts

@@ -0,0 +1,32 @@
+/**
+ * Shared stdin preview utility for hook fast-path optimization.
+ *
+ * Reads all of stdin once, then exposes a small preview slice so callers
+ * can do cheap `.includes()` keyword checks before paying for JSON.parse().
+ * When the keyword is absent the hook can exit in <1ms.
+ */
+
+const STDIN_PREVIEW_BYTES = 4096;
+
+/**
+ * Read stdin and return both a preview slice and the full text.
+ *
+ * Bun's `stdin.text()` consumes the entire stream in one call, so this is
+ * not a true streaming preview. The win comes from avoiding `JSON.parse()`
+ * entirely when the preview slice already proves the payload is irrelevant.
+ *
+ * @param previewBytes  Number of leading characters to expose in `preview`.
+ *                      Defaults to 4096, which comfortably covers the
+ *                      envelope fields (`hook_event_name`, `tool_name`, etc.)
+ *                      of any Claude Code hook payload.
+ */
+export async function readStdinWithPreview(previewBytes: number = STDIN_PREVIEW_BYTES): Promise<{
+  preview: string;
+  full: string;
+}> {
+  const raw = await Bun.stdin.text();
+  return {
+    preview: raw.slice(0, previewBytes),
+    full: raw,
+  };
+}

package/cli/selftune/localdb/direct-write.ts

@@ -191,14 +191,17 @@ export function writeSessionTelemetryToDb(record: SessionTelemetryRecord): boole
   return safeWrite("session-telemetry", (db) => {
     getStmt(
       db,
-      "session-telemetry",
+      "session-telemetry-v4",
       `
       INSERT INTO session_telemetry
         (session_id, timestamp, cwd, transcript_path, tool_calls_json,
          total_tool_calls, bash_commands_json, skills_triggered_json,
          skills_invoked_json, assistant_turns, errors_encountered,
-         transcript_chars, last_user_query, source, input_tokens, output_tokens)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+         transcript_chars, last_user_query, source, input_tokens, output_tokens,
+         cached_input_tokens, reasoning_output_tokens, cost_usd,
+         files_changed, lines_added, lines_removed, lines_modified,
+         artifact_count, session_type, agent_summary)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT(session_id) DO UPDATE SET
         timestamp = excluded.timestamp,
         tool_calls_json = excluded.tool_calls_json,
@@ -211,7 +214,17 @@ export function writeSessionTelemetryToDb(record: SessionTelemetryRecord): boole
         transcript_chars = excluded.transcript_chars,
         last_user_query = excluded.last_user_query,
         input_tokens = COALESCE(excluded.input_tokens, session_telemetry.input_tokens),
-        output_tokens = COALESCE(excluded.output_tokens, session_telemetry.output_tokens)
+        output_tokens = COALESCE(excluded.output_tokens, session_telemetry.output_tokens),
+        cached_input_tokens = COALESCE(excluded.cached_input_tokens, session_telemetry.cached_input_tokens),
+        reasoning_output_tokens = COALESCE(excluded.reasoning_output_tokens, session_telemetry.reasoning_output_tokens),
+        cost_usd = COALESCE(excluded.cost_usd, session_telemetry.cost_usd),
+        files_changed = COALESCE(excluded.files_changed, session_telemetry.files_changed),
+        lines_added = COALESCE(excluded.lines_added, session_telemetry.lines_added),
+        lines_removed = COALESCE(excluded.lines_removed, session_telemetry.lines_removed),
+        lines_modified = COALESCE(excluded.lines_modified, session_telemetry.lines_modified),
+        artifact_count = COALESCE(excluded.artifact_count, session_telemetry.artifact_count),
+        session_type = COALESCE(excluded.session_type, session_telemetry.session_type),
+        agent_summary = COALESCE(excluded.agent_summary, session_telemetry.agent_summary)
     `,
     ).run(
       record.session_id,
@@ -230,6 +243,16 @@ export function writeSessionTelemetryToDb(record: SessionTelemetryRecord): boole
       record.source ?? null,
       record.input_tokens ?? null,
       record.output_tokens ?? null,
+      record.cached_input_tokens ?? null,
+      record.reasoning_output_tokens ?? null,
+      record.cost_usd ?? null,
+      record.files_changed ?? null,
+      record.lines_added ?? null,
+      record.lines_removed ?? null,
+      record.lines_modified ?? null,
+      record.artifact_count ?? null,
+      record.session_type ?? null,
+      record.agent_summary ?? null,
     );
   });
 }
@@ -444,6 +467,34 @@ export function updateSignalConsumed(
   return result?.changes > 0;
 }
 
+export function writeCommitTracking(record: {
+  session_id: string;
+  commit_sha: string;
+  commit_title?: string;
+  branch?: string;
+  repo_remote?: string;
+  timestamp: string;
+}): boolean {
+  return safeWrite("commit-tracking", (db) => {
+    getStmt(
+      db,
+      "commit-tracking",
+      `
+      INSERT INTO commit_tracking
+        (session_id, commit_sha, commit_title, branch, repo_remote, timestamp)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `,
+    ).run(
+      record.session_id,
+      record.commit_sha,
+      record.commit_title ?? null,
+      record.branch ?? null,
+      record.repo_remote ?? null,
+      record.timestamp,
+    );
+  });
+}
+
 // -- Internal insert helpers (used by cached statements) ----------------------
 
 function insertSession(db: Database, s: CanonicalSessionRecord): void {
@@ -581,14 +632,17 @@ function insertSkillInvocation(
 function insertExecutionFact(db: Database, ef: CanonicalExecutionFactRecord): void {
   getStmt(
     db,
-    "execution-fact",
+    "execution-fact-v3",
     `
     INSERT INTO execution_facts
       (session_id, occurred_at, prompt_id, tool_calls_json, total_tool_calls,
        assistant_turns, errors_encountered, input_tokens, output_tokens,
+       cached_input_tokens, reasoning_output_tokens, cost_usd,
+       files_changed, lines_added, lines_removed, lines_modified,
+       artifact_count, session_type,
        duration_ms, completion_status,
        schema_version, platform, normalized_at, normalizer_version, capture_mode, raw_source_ref)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `,
   ).run(
     ef.session_id,
@@ -600,6 +654,15 @@ function insertExecutionFact(db: Database, ef: CanonicalExecutionFactRecord): vo
     ef.errors_encountered,
     ef.input_tokens ?? null,
     ef.output_tokens ?? null,
+    ef.cached_input_tokens ?? null,
+    ef.reasoning_output_tokens ?? null,
+    ef.cost_usd ?? null,
+    ef.files_changed ?? null,
+    ef.lines_added ?? null,
+    ef.lines_removed ?? null,
+    ef.lines_modified ?? null,
+    ef.artifact_count ?? null,
+    ef.session_type ?? null,
     ef.duration_ms ?? null,
     ef.completion_status ?? null,
     ef.schema_version ?? null,