selftune 0.1.2 → 0.2.0

package/cli/selftune/contribute/bundle.ts

@@ -0,0 +1,314 @@
+/**
+ * Bundle assembly for contribution export.
+ *
+ * Pure function: reads logs, filters, aggregates, and returns a ContributionBundle.
+ */
+
+import { randomUUID } from "node:crypto";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import {
+  EVOLUTION_AUDIT_LOG,
+  QUERY_LOG,
+  SELFTUNE_CONFIG_DIR,
+  SKILL_LOG,
+  TELEMETRY_LOG,
+} from "../constants.js";
+import { buildEvalSet, classifyInvocation } from "../eval/hooks-to-evals.js";
+import type {
+  ContributionBundle,
+  ContributionEvolutionSummary,
+  ContributionGradingSummary,
+  ContributionQuery,
+  ContributionSessionMetrics,
+  EvolutionAuditEntry,
+  GradingResult,
+  QueryLogRecord,
+  SessionTelemetryRecord,
+  SkillUsageRecord,
+} from "../types.js";
+import { readJsonl } from "../utils/jsonl.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function filterSince<T extends { timestamp: string }>(records: T[], since?: Date): T[] {
+  if (!since) return records;
+  return records.filter((r) => new Date(r.timestamp) >= since);
+}
+
+function getVersion(): string {
+  try {
+    const pkg = JSON.parse(readFileSync(join(import.meta.dir, "../../../package.json"), "utf-8"));
+    return pkg.version ?? "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+function getAgentType(): string {
+  try {
+    const configPath = join(SELFTUNE_CONFIG_DIR, "config.json");
+    if (!existsSync(configPath)) return "unknown";
+    const config = JSON.parse(readFileSync(configPath, "utf-8"));
+    return config.agent_type ?? "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+function avg(nums: number[]): number {
+  if (nums.length === 0) return 0;
+  return nums.reduce((a, b) => a + b, 0) / nums.length;
+}
+
+// ---------------------------------------------------------------------------
+// Grading summary
+// ---------------------------------------------------------------------------
+
+function buildGradingSummary(skillName: string): ContributionGradingSummary | null {
+  const gradingDir = join(homedir(), ".selftune", "grading");
+  if (!existsSync(gradingDir)) return null;
+
+  try {
+    const files = readdirSync(gradingDir).filter((f) => f.endsWith(".json"));
+    if (files.length === 0) return null;
+
+    let totalSessions = 0;
+    let gradedSessions = 0;
+    let passRateSum = 0;
+    let expectationCount = 0;
+
+    for (const file of files) {
+      try {
+        const data = JSON.parse(readFileSync(join(gradingDir, file), "utf-8")) as GradingResult;
+        if (data.skill_name !== skillName) continue;
+        totalSessions++;
+        if (data.summary) {
+          gradedSessions++;
+          passRateSum += data.summary.pass_rate ?? 0;
+          expectationCount += data.summary.total ?? 0;
+        }
+      } catch {
+        // skip malformed grading files
+      }
+    }
+
+    if (gradedSessions === 0) return null;
+
+    return {
+      total_sessions: totalSessions,
+      graded_sessions: gradedSessions,
+      average_pass_rate: passRateSum / gradedSessions,
+      expectation_count: expectationCount,
+    };
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Evolution summary
+// ---------------------------------------------------------------------------
+
+function buildEvolutionSummary(
+  records: EvolutionAuditEntry[],
+): ContributionEvolutionSummary | null {
+  if (records.length === 0) return null;
+
+  const proposals = new Set<string>();
+  let deployed = 0;
+  let rolledBack = 0;
+  const improvements: number[] = [];
+
+  for (const r of records) {
+    proposals.add(r.proposal_id);
+    if (r.action === "deployed") {
+      deployed++;
+      if (r.eval_snapshot?.pass_rate != null) {
+        improvements.push(r.eval_snapshot.pass_rate);
+      }
+    }
+    if (r.action === "rolled_back") {
+      rolledBack++;
+    }
+  }
+
+  return {
+    total_proposals: proposals.size,
+    deployed_proposals: deployed,
+    rolled_back_proposals: rolledBack,
+    average_improvement: improvements.length > 0 ? avg(improvements) : 0,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Session metrics
+// ---------------------------------------------------------------------------
+
+function buildSessionMetrics(records: SessionTelemetryRecord[]): ContributionSessionMetrics {
+  if (records.length === 0) {
+    return {
+      total_sessions: 0,
+      avg_assistant_turns: 0,
+      avg_tool_calls: 0,
+      avg_errors: 0,
+      top_tools: [],
+    };
+  }
+
+  const toolCounts = new Map<string, number>();
+  for (const r of records) {
+    for (const [tool, count] of Object.entries(r.tool_calls ?? {})) {
+      toolCounts.set(tool, (toolCounts.get(tool) ?? 0) + count);
+    }
+  }
+
+  const topTools = [...toolCounts.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 10)
+    .map(([tool, count]) => ({ tool, count }));
+
+  return {
+    total_sessions: records.length,
+    avg_assistant_turns: Math.round(avg(records.map((r) => r.assistant_turns ?? 0))),
+    avg_tool_calls: Math.round(avg(records.map((r) => r.total_tool_calls ?? 0))),
+    avg_errors: Number(avg(records.map((r) => r.errors_encountered ?? 0)).toFixed(2)),
+    top_tools: topTools,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Main assembly
+// ---------------------------------------------------------------------------
+
+export function assembleBundle(options: {
+  skillName: string;
+  since?: Date;
+  sanitizationLevel: "conservative" | "aggressive";
+  queryLogPath?: string;
+  skillLogPath?: string;
+  telemetryLogPath?: string;
+  evolutionAuditLogPath?: string;
+}): ContributionBundle {
+  const {
+    skillName,
+    since,
+    sanitizationLevel,
+    queryLogPath = QUERY_LOG,
+    skillLogPath = SKILL_LOG,
+    telemetryLogPath = TELEMETRY_LOG,
+    evolutionAuditLogPath = EVOLUTION_AUDIT_LOG,
+  } = options;
+
+  // Read all logs
+  const allSkillRecords = readJsonl<SkillUsageRecord>(skillLogPath);
+  const allQueryRecords = readJsonl<QueryLogRecord>(queryLogPath);
+  const allTelemetryRecords = readJsonl<SessionTelemetryRecord>(telemetryLogPath);
+  const allEvolutionRecords = readJsonl<EvolutionAuditEntry>(evolutionAuditLogPath);
+
+  // Filter by skill and since
+  const skillRecords = filterSince(
+    allSkillRecords.filter((r) => r.skill_name === skillName),
+    since,
+  );
+  const queryRecords = filterSince(allQueryRecords, since);
+  const telemetryRecords = filterSince(
+    allTelemetryRecords.filter((r) => (r.skills_triggered ?? []).includes(skillName)),
+    since,
+  );
+  // TODO: Filter evolution records by skillName once EvolutionAuditEntry gains a skill_name field.
+  // Currently includes all skills' proposals. Schema change requires human review (escalation-policy.md).
+  const evolutionRecords = filterSince(allEvolutionRecords, since);
+
+  // Build positive queries
+  const seenQueries = new Set<string>();
+  const positiveQueries: ContributionQuery[] = [];
+  const triggeredQueryTexts = new Set<string>();
+  for (const r of skillRecords) {
+    const q = (r.query ?? "").trim();
+    if (r.triggered) triggeredQueryTexts.add(q);
+    if (!q || seenQueries.has(q)) continue;
+    seenQueries.add(q);
+    positiveQueries.push({
+      query: q,
+      invocation_type: classifyInvocation(q, skillName),
+      source: r.source ?? "skill_log",
+    });
+  }
+
+  // Build unmatched queries: queries with no matching triggered skill record
+  const unmatchedQueries = queryRecords
+    .filter((r) => !triggeredQueryTexts.has((r.query ?? "").trim()))
+    .map((r) => ({ query: (r.query ?? "").trim(), timestamp: r.timestamp }))
+    .filter((r) => r.query.length > 0);
+
+  // Build pending proposals: proposals with created/validated but no terminal action
+  const terminalActions = new Set(["deployed", "rejected", "rolled_back"]);
+  const proposalActions = new Map<string, EvolutionAuditEntry[]>();
+  for (const r of evolutionRecords) {
+    const entries = proposalActions.get(r.proposal_id) ?? [];
+    entries.push(r);
+    proposalActions.set(r.proposal_id, entries);
+  }
+  const pendingProposals: Array<{
+    proposal_id: string;
+    skill_name?: string;
+    action: string;
+    timestamp: string;
+    details: string;
+  }> = [];
+  for (const [proposalId, entries] of proposalActions) {
+    const hasTerminal = entries.some((e) => terminalActions.has(e.action));
+    if (!hasTerminal) {
+      // Use the latest entry for this proposal
+      const latest = entries[entries.length - 1];
+      pendingProposals.push({
+        proposal_id: proposalId,
+        skill_name: latest.skill_name,
+        action: latest.action,
+        timestamp: latest.timestamp,
+        details: latest.details,
+      });
+    }
+  }
+
+  // Build eval entries
+  const evalEntries = buildEvalSet(skillRecords, queryRecords, skillName, 50, true, 42, true).map(
+    (e) => ({
+      query: e.query,
+      should_trigger: e.should_trigger,
+      invocation_type: e.invocation_type,
+    }),
+  );
+
+  // Build grading summary
+  const gradingSummary = buildGradingSummary(skillName);
+
+  // Build evolution summary
+  const evolutionSummary = buildEvolutionSummary(evolutionRecords);
+
+  // Build session metrics
+  const sessionMetrics = buildSessionMetrics(telemetryRecords);
+
+  const hasNewFields = unmatchedQueries.length > 0 || pendingProposals.length > 0;
+
+  return {
+    schema_version: hasNewFields ? "1.2" : "1.1",
+    skill_name: skillName,
+    contributor_id: randomUUID(),
+    created_at: new Date().toISOString(),
+    selftune_version: getVersion(),
+    agent_type: getAgentType(),
+    sanitization_level: sanitizationLevel,
+    positive_queries: positiveQueries,
+    eval_entries: evalEntries,
+    grading_summary: gradingSummary,
+    evolution_summary: evolutionSummary,
+    session_metrics: sessionMetrics,
+    ...(unmatchedQueries.length > 0 ? { unmatched_queries: unmatchedQueries } : {}),
+    ...(pendingProposals.length > 0 ? { pending_proposals: pendingProposals } : {}),
+  };
+}

package/cli/selftune/contribute/contribute.ts

@@ -0,0 +1,214 @@
+#!/usr/bin/env bun
+/**
+ * selftune contribute — opt-in export of anonymized skill observability data.
+ *
+ * Usage:
+ *   bun run cli/selftune/contribute/contribute.ts --skill selftune [--preview] [--output file.json]
+ *   bun run cli/selftune/contribute/contribute.ts --skill selftune --submit
+ */
+
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+import { CONTRIBUTIONS_DIR } from "../constants.js";
+import { assembleBundle } from "./bundle.js";
+import { sanitizeBundle } from "./sanitize.js";
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+export async function cliMain(): Promise<void> {
+  const { values } = parseArgs({
+    options: {
+      skill: { type: "string", default: "selftune" },
+      output: { type: "string" },
+      preview: { type: "boolean", default: false },
+      sanitize: { type: "string", default: "conservative" },
+      since: { type: "string" },
+      submit: { type: "boolean", default: false },
+      endpoint: { type: "string", default: "https://selftune-api.fly.dev" },
+      github: { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+
+  const skillName = values.skill ?? "selftune";
+  const sanitizationLevel = values.sanitize === "aggressive" ? "aggressive" : "conservative";
+
+  let since: Date | undefined;
+  if (values.since) {
+    since = new Date(values.since);
+    if (Number.isNaN(since.getTime())) {
+      console.error(
+        `Error: Invalid --since date: "${values.since}". Use a valid date format (e.g., 2026-01-01).`,
+      );
+      process.exit(1);
+    }
+  }
+
+  // 1. Assemble raw bundle
+  const rawBundle = assembleBundle({
+    skillName,
+    since,
+    sanitizationLevel,
+  });
+
+  // 2. Sanitize
+  const bundle = sanitizeBundle(rawBundle, sanitizationLevel, skillName);
+
+  // 3. Preview mode
+  if (values.preview) {
+    console.log(JSON.stringify(bundle, null, 2));
+    return;
+  }
+
+  // 4. Determine output path
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const defaultPath = `${CONTRIBUTIONS_DIR}/selftune-contribution-${timestamp}.json`;
+  const outputPath = values.output ?? defaultPath;
+
+  // Ensure parent directory exists
+  const dir = outputPath.substring(0, outputPath.lastIndexOf("/"));
+  if (dir && !existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+
+  // 5. Write
+  const json = JSON.stringify(bundle, null, 2);
+  writeFileSync(outputPath, json, "utf-8");
+
+  // 6. Summary
+  console.log(`Contribution bundle written to: ${outputPath}`);
+  console.log(`  Queries:       ${bundle.positive_queries.length}`);
+  console.log(`  Eval entries:  ${bundle.eval_entries.length}`);
+  console.log(`  Sessions:      ${bundle.session_metrics.total_sessions}`);
+  console.log(`  Sanitization:  ${sanitizationLevel}`);
+  if (bundle.grading_summary) {
+    console.log(
+      `  Grading:       ${bundle.grading_summary.graded_sessions} sessions, ${(bundle.grading_summary.average_pass_rate * 100).toFixed(1)}% avg pass rate`,
+    );
+  }
+  if (bundle.evolution_summary) {
+    console.log(
+      `  Evolution:     ${bundle.evolution_summary.total_proposals} proposals, ${bundle.evolution_summary.deployed_proposals} deployed`,
+    );
+  }
+
+  // 7. Submit
+  if (values.submit) {
+    if (values.github) {
+      const ok = submitToGitHub(json, outputPath);
+      if (!ok) process.exit(1);
+    } else {
+      const endpoint = values.endpoint ?? "https://selftune-api.fly.dev";
+      const ok = await submitToService(json, endpoint, skillName);
+      if (!ok) {
+        console.log("Falling back to GitHub submission...");
+        const ghOk = submitToGitHub(json, outputPath);
+        if (!ghOk) process.exit(1);
+      }
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Service submission
+// ---------------------------------------------------------------------------
+
+async function submitToService(
+  json: string,
+  endpoint: string,
+  skillName: string,
+): Promise<boolean> {
+  try {
+    const url = `${endpoint}/api/submit`;
+    const res = await fetch(url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: json,
+    });
+
+    if (!res.ok) {
+      const body = await res.text();
+      console.error(`[ERROR] Service submission failed (${res.status}): ${body}`);
+      return false;
+    }
+
+    console.log(`\nSubmitted to ${endpoint}`);
+    console.log(`  Badge: ${endpoint}/badge/${encodeURIComponent(skillName)}`);
+    console.log(`  Report: ${endpoint}/report/${encodeURIComponent(skillName)}`);
+    return true;
+  } catch (err) {
+    console.error(
+      `[ERROR] Could not reach ${endpoint}: ${err instanceof Error ? err.message : String(err)}`,
+    );
+    return false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// GitHub submission
+// ---------------------------------------------------------------------------
+
+function submitToGitHub(json: string, outputPath: string): boolean {
+  const repo = "WellDunDun/selftune";
+  const sizeKB = Buffer.byteLength(json, "utf-8") / 1024;
+
+  let body: string;
+  if (sizeKB < 50) {
+    body = `## Selftune Contribution\n\n\`\`\`json\n${json}\n\`\`\``;
+  } else {
+    // Create gist for large bundles
+    try {
+      const result = spawnSync("gh", ["gist", "create", outputPath, "--public"], {
+        encoding: "utf-8",
+      });
+      if (result.status !== 0) {
+        console.error("[ERROR] Failed to create gist. Is `gh` installed and authenticated?");
+        console.error(result.stderr || "gh gist create failed");
+        return false;
+      }
+      const gistUrl = result.stdout.trim();
+      body = `## Selftune Contribution\n\nBundle too large to inline (${sizeKB.toFixed(1)} KB).\n\nGist: ${gistUrl}`;
+    } catch (err) {
+      console.error("[ERROR] Failed to create gist. Is `gh` installed and authenticated?");
+      console.error(String(err));
+      return false;
+    }
+  }
+
+  try {
+    const result = spawnSync(
+      "gh",
+      [
+        "issue",
+        "create",
+        "--repo",
+        repo,
+        "--label",
+        "contribution",
+        "--title",
+        "selftune contribution",
+        "--body",
+        body,
+      ],
+      { encoding: "utf-8" },
+    );
+    if (result.status !== 0) {
+      console.error("[ERROR] Failed to create GitHub issue. Is `gh` installed and authenticated?");
+      console.error(result.stderr || "gh issue create failed");
+      return false;
+    }
+    console.log(`\nSubmitted: ${result.stdout.trim()}`);
+    return true;
+  } catch (err) {
+    console.error("[ERROR] Failed to create GitHub issue. Is `gh` installed and authenticated?");
+    console.error(String(err));
+    return false;
+  }
+}
+
+if (import.meta.main) {
+  await cliMain();
+}

package/cli/selftune/contribute/sanitize.ts

@@ -0,0 +1,162 @@
+/**
+ * Privacy sanitization for contribution bundles.
+ *
+ * Two levels:
+ *   conservative (default) — redacts paths, emails, secrets, IPs, project names, session IDs
+ *   aggressive — extends conservative with identifiers, quoted strings, modules, truncation
+ *
+ * All functions are pure (no side effects).
+ */
+
+import {
+  AGGRESSIVE_MAX_QUERY_LENGTH,
+  EMAIL_PATTERN,
+  FILE_PATH_PATTERN,
+  IDENTIFIER_PATTERN,
+  IP_PATTERN,
+  MODULE_PATTERN,
+  SECRET_PATTERNS,
+} from "../constants.js";
+import type { ContributionBundle } from "../types.js";
+
+// UUID v4 pattern for session ID redaction
+const UUID_PATTERN = /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi;
+
+// Quoted string patterns for aggressive mode
+const DOUBLE_QUOTED_PATTERN = /"[^"]*"/g;
+const SINGLE_QUOTED_PATTERN = /'[^']*'/g;
+
+// ---------------------------------------------------------------------------
+// Conservative sanitization
+// ---------------------------------------------------------------------------
+
+export function sanitizeConservative(text: string, projectName?: string): string {
+  if (!text) return text;
+
+  let result = text;
+
+  // Secrets first (longest/most specific patterns)
+  for (const pattern of SECRET_PATTERNS) {
+    // Clone regex to reset lastIndex
+    result = result.replace(new RegExp(pattern.source, pattern.flags), "[SECRET]");
+  }
+
+  // File paths
+  result = result.replace(new RegExp(FILE_PATH_PATTERN.source, FILE_PATH_PATTERN.flags), "[PATH]");
+
+  // Emails
+  result = result.replace(new RegExp(EMAIL_PATTERN.source, EMAIL_PATTERN.flags), "[EMAIL]");
+
+  // IPs
+  result = result.replace(new RegExp(IP_PATTERN.source, IP_PATTERN.flags), "[IP]");
+
+  // Project name
+  if (projectName) {
+    result = result.replace(new RegExp(escapeRegExp(projectName), "g"), "[PROJECT]");
+  }
+
+  // Session IDs (UUIDs)
+  result = result.replace(UUID_PATTERN, "[SESSION]");
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Aggressive sanitization
+// ---------------------------------------------------------------------------
+
+export function sanitizeAggressive(text: string, projectName?: string): string {
+  if (!text) return text;
+
+  // Start with conservative
+  let result = sanitizeConservative(text, projectName);
+
+  // Module paths (import/require/from)
+  result = result.replace(new RegExp(MODULE_PATTERN.source, MODULE_PATTERN.flags), (match) => {
+    // Preserve the keyword, replace the path
+    const keyword = match.match(/^(import|require|from)/)?.[0] ?? "";
+    // Determine what follows the keyword
+    if (match.includes("(")) {
+      return `${keyword}([MODULE])`;
+    }
+    return `${keyword} [MODULE]`;
+  });
+
+  // Quoted strings
+  result = result.replace(DOUBLE_QUOTED_PATTERN, "[STRING]");
+  result = result.replace(SINGLE_QUOTED_PATTERN, "[STRING]");
+
+  // Long identifiers (camelCase/PascalCase > 8 chars)
+  result = result.replace(
+    new RegExp(IDENTIFIER_PATTERN.source, IDENTIFIER_PATTERN.flags),
+    "[IDENTIFIER]",
+  );
+
+  // Truncate
+  if (result.length > AGGRESSIVE_MAX_QUERY_LENGTH) {
+    result = result.slice(0, AGGRESSIVE_MAX_QUERY_LENGTH);
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Dispatcher
+// ---------------------------------------------------------------------------
+
+export function sanitize(
+  text: string,
+  level: "conservative" | "aggressive",
+  projectName?: string,
+): string {
+  return level === "aggressive"
+    ? sanitizeAggressive(text, projectName)
+    : sanitizeConservative(text, projectName);
+}
+
+// ---------------------------------------------------------------------------
+// Bundle sanitization
+// ---------------------------------------------------------------------------
+
+export function sanitizeBundle(
+  bundle: ContributionBundle,
+  level: "conservative" | "aggressive",
+  projectName?: string,
+): ContributionBundle {
+  return {
+    ...bundle,
+    sanitization_level: level,
+    positive_queries: bundle.positive_queries.map((q) => ({
+      ...q,
+      query: sanitize(q.query, level, projectName),
+    })),
+    eval_entries: bundle.eval_entries.map((e) => ({
+      ...e,
+      query: sanitize(e.query, level, projectName),
+    })),
+    ...(bundle.unmatched_queries
+      ? {
+          unmatched_queries: bundle.unmatched_queries.map((q) => ({
+            ...q,
+            query: sanitize(q.query, level, projectName),
+          })),
+        }
+      : {}),
+    ...(bundle.pending_proposals
+      ? {
+          pending_proposals: bundle.pending_proposals.map((p) => ({
+            ...p,
+            details: sanitize(p.details, level, projectName),
+          })),
+        }
+      : {}),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function escapeRegExp(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}