selftune 0.2.22 → 0.2.24

package/cli/selftune/command-surface.ts

@@ -0,0 +1,397 @@
+export interface PublicCommandFlag {
+  token: string;
+  helpLabel: string;
+  description: string;
+}
+
+export interface PublicCommandSurface {
+  command: string;
+  summary: string;
+  usage: string;
+  flags: readonly PublicCommandFlag[];
+  quickReference: string;
+  extraHelpSections?: readonly string[];
+}
+
+function formatOptionLines(flags: readonly PublicCommandFlag[]): string[] {
+  const width = Math.max(...flags.map((flag) => flag.helpLabel.length), 0) + 2;
+  return flags.map((flag) => `  ${flag.helpLabel.padEnd(width)}${flag.description}`);
+}
+
+export function renderCommandHelp(surface: PublicCommandSurface): string {
+  const lines = [
+    `${surface.command} — ${surface.summary}`,
+    "",
+    "Usage:",
+    `  ${surface.usage}`,
+    "",
+    "Options:",
+    ...formatOptionLines(surface.flags),
+  ];
+
+  for (const section of surface.extraHelpSections ?? []) {
+    lines.push("", ...section.split("\n"));
+  }
+
+  return lines.join("\n");
+}
+
+export const PUBLIC_COMMAND_SURFACES = {
+  evalGenerate: {
+    command: "selftune eval generate",
+    summary: "Build eval sets from logs or SKILL.md",
+    usage: "selftune eval generate --skill <name> [options]",
+    flags: [
+      {
+        token: "--skill",
+        helpLabel: "--skill",
+        description: "Skill name (required unless --list-skills)",
+      },
+      {
+        token: "--list-skills",
+        helpLabel: "--list-skills",
+        description: "List skills with trusted-vs-raw readiness counts",
+      },
+      {
+        token: "--stats",
+        helpLabel: "--stats",
+        description: "Show aggregate telemetry stats for the skill",
+      },
+      {
+        token: "--max",
+        helpLabel: "--max",
+        description: "Maximum eval entries per side (default: 50)",
+      },
+      {
+        token: "--seed",
+        helpLabel: "--seed",
+        description: "Deterministic shuffle seed (default: 42)",
+      },
+      {
+        token: "--output",
+        helpLabel: "--output, --out",
+        description: "Output file path (default: <skill>_trigger_eval.json)",
+      },
+      {
+        token: "--no-negatives",
+        helpLabel: "--no-negatives",
+        description: "Exclude negative examples from output",
+      },
+      {
+        token: "--no-taxonomy",
+        helpLabel: "--no-taxonomy",
+        description: "Skip invocation_type classification",
+      },
+      {
+        token: "--skill-log",
+        helpLabel: "--skill-log",
+        description: "Path to skill_usage_log.jsonl",
+      },
+      {
+        token: "--query-log",
+        helpLabel: "--query-log",
+        description: "Path to all_queries_log.jsonl",
+      },
+      {
+        token: "--telemetry-log",
+        helpLabel: "--telemetry-log",
+        description: "Path to session_telemetry_log.jsonl",
+      },
+      {
+        token: "--synthetic",
+        helpLabel: "--synthetic",
+        description: "Generate evals from SKILL.md via LLM (no logs needed)",
+      },
+      {
+        token: "--auto-synthetic",
+        helpLabel: "--auto-synthetic",
+        description: "Fall back to SKILL.md cold-start evals when no trusted triggers exist",
+      },
+      {
+        token: "--blend",
+        helpLabel: "--blend",
+        description: "Blend log-based and synthetic evals into one set",
+      },
+      {
+        token: "--skill-path",
+        helpLabel: "--skill-path",
+        description: "Path to SKILL.md (required with --synthetic, used by --blend)",
+      },
+      {
+        token: "--model",
+        helpLabel: "--model",
+        description: "Override the synthetic-generation LLM model",
+      },
+      {
+        token: "--help",
+        helpLabel: "--help",
+        description: "Show this help message",
+      },
+    ],
+    quickReference:
+      "selftune eval generate      --skill <name> [--list-skills] [--stats] [--max N] [--seed N] [--output PATH] [--blend]",
+    extraHelpSections: [
+      `Recommended creator loop:
+  1. selftune eval generate --skill <name>
+  2. selftune eval unit-test --skill <name> --generate --skill-path <path>
+  3. selftune evolve --skill <name> --skill-path <path> --dry-run --validation-mode replay
+  4. selftune grade baseline --skill <name> --skill-path <path>
+
+Generated evals are also mirrored into ~/.selftune/eval-sets/<skill>.json so the dashboard and status surfaces can track readiness.`,
+    ],
+  },
+  evolve: {
+    command: "selftune evolve",
+    summary: "Evolve a skill description via failure patterns",
+    usage: "selftune evolve --skill <name> --skill-path <path> [options]",
+    flags: [
+      { token: "--skill", helpLabel: "--skill", description: "Skill name (required)" },
+      {
+        token: "--skill-path",
+        helpLabel: "--skill-path",
+        description: "Path to SKILL.md (required)",
+      },
+      {
+        token: "--eval-set",
+        helpLabel: "--eval-set",
+        description: "Path to eval set JSON (optional, builds from logs if omitted)",
+      },
+      {
+        token: "--agent",
+        helpLabel: "--agent",
+        description: "Agent CLI to use (claude, codex, opencode)",
+      },
+      {
+        token: "--dry-run",
+        helpLabel: "--dry-run",
+        description: "Validate proposal without deploying",
+      },
+      {
+        token: "--confidence",
+        helpLabel: "--confidence",
+        description: "Confidence threshold 0.0-1.0 (default: 0.6)",
+      },
+      {
+        token: "--max-iterations",
+        helpLabel: "--max-iterations",
+        description: "Max retry iterations (default: 3)",
+      },
+      {
+        token: "--pareto",
+        helpLabel: "--pareto",
+        description: "Enable Pareto multi-candidate selection",
+      },
+      {
+        token: "--candidates",
+        helpLabel: "--candidates",
+        description: "Number of candidates to generate (default: 3, max: 5)",
+      },
+      {
+        token: "--token-efficiency",
+        helpLabel: "--token-efficiency",
+        description: "Enable 5D Pareto with token efficiency scoring",
+      },
+      {
+        token: "--with-baseline",
+        helpLabel: "--with-baseline",
+        description: "Gate deployment on baseline lift > 0.05",
+      },
+      {
+        token: "--validation-mode",
+        helpLabel: "--validation-mode",
+        description: "Validation strategy: auto|replay|judge (default: auto)",
+      },
+      {
+        token: "--validation-model",
+        helpLabel: "--validation-model",
+        description: "Model for trigger-check validation calls (default: haiku)",
+      },
+      {
+        token: "--cheap-loop",
+        helpLabel: "--cheap-loop",
+        description: "Use cheap models for loop, expensive for gate (default: on)",
+      },
+      {
+        token: "--full-model",
+        helpLabel: "--full-model",
+        description: "Use same model for all stages (disables cheap-loop)",
+      },
+      {
+        token: "--gate-model",
+        helpLabel: "--gate-model",
+        description: "Model for final gate validation (default: sonnet)",
+      },
+      {
+        token: "--gate-effort",
+        helpLabel: "--gate-effort",
+        description: "Thinking effort for final gate (low|medium|high|max)",
+      },
+      {
+        token: "--adaptive-gate",
+        helpLabel: "--adaptive-gate",
+        description: "Escalate risky gate checks to opus + high effort",
+      },
+      {
+        token: "--proposal-model",
+        helpLabel: "--proposal-model",
+        description: "Model for proposal generation LLM calls",
+      },
+      {
+        token: "--sync-first",
+        helpLabel: "--sync-first",
+        description: "Refresh source-truth telemetry before building evals/failure patterns",
+      },
+      {
+        token: "--sync-force",
+        helpLabel: "--sync-force",
+        description: "Force a full rescan during --sync-first",
+      },
+      {
+        token: "--verbose",
+        helpLabel: "--verbose",
+        description: "Output full EvolveResult JSON (default: compact summary)",
+      },
+      {
+        token: "--help",
+        helpLabel: "--help",
+        description: "Show this help message",
+      },
+    ],
+    quickReference:
+      "selftune evolve          --skill <name> --skill-path <path> [--dry-run] [--validation-mode auto|replay|judge]",
+  },
+  watch: {
+    command: "selftune watch",
+    summary: "Monitor post-deploy skill health",
+    usage: "selftune watch --skill <name> --skill-path <path> [options]",
+    flags: [
+      { token: "--skill", helpLabel: "--skill", description: "Skill name (required)" },
+      {
+        token: "--skill-path",
+        helpLabel: "--skill-path",
+        description: "Path to SKILL.md (required)",
+      },
+      {
+        token: "--window",
+        helpLabel: "--window",
+        description: "Number of recent sessions to consider (default: 20)",
+      },
+      {
+        token: "--threshold",
+        helpLabel: "--threshold",
+        description: "Regression threshold below baseline (default: 0.1)",
+      },
+      {
+        token: "--auto-rollback",
+        helpLabel: "--auto-rollback",
+        description: "Automatically rollback on regression detection",
+      },
+      {
+        token: "--grade-threshold",
+        helpLabel: "--grade-threshold",
+        description: "Grade regression threshold (default: 0.15)",
+      },
+      {
+        token: "--no-grade-watch",
+        helpLabel: "--no-grade-watch",
+        description: "Disable grade-based regression watch (enabled by default)",
+      },
+      {
+        token: "--sync-first",
+        helpLabel: "--sync-first",
+        description: "Refresh source-truth telemetry before reading watch inputs",
+      },
+      {
+        token: "--sync-force",
+        helpLabel: "--sync-force",
+        description: "Force a full rescan during --sync-first",
+      },
+      {
+        token: "--help",
+        helpLabel: "--help",
+        description: "Show this help message",
+      },
+    ],
+    quickReference:
+      "selftune watch    --skill <name> --skill-path <path> [--auto-rollback] [--grade-threshold N] [--no-grade-watch]",
+  },
+  orchestrate: {
+    command: "selftune orchestrate",
+    summary: "Autonomous core loop",
+    usage: "selftune orchestrate [options]",
+    flags: [
+      {
+        token: "--dry-run",
+        helpLabel: "--dry-run",
+        description: "Preview actions without mutations",
+      },
+      {
+        token: "--review-required",
+        helpLabel: "--review-required",
+        description: "Validate candidates but require human review before deploy",
+      },
+      {
+        token: "--auto-approve",
+        helpLabel: "--auto-approve",
+        description: "Deprecated alias; autonomous mode is now the default",
+      },
+      {
+        token: "--skill",
+        helpLabel: "--skill <name>",
+        description: "Scope to a single skill",
+      },
+      {
+        token: "--max-skills",
+        helpLabel: "--max-skills <n>",
+        description: "Cap skills processed per run (default: 5)",
+      },
+      {
+        token: "--recent-window",
+        helpLabel: "--recent-window <hrs>",
+        description: "Hours to look back for watch targets (default: 48)",
+      },
+      {
+        token: "--sync-force",
+        helpLabel: "--sync-force",
+        description: "Force full rescan during sync",
+      },
+      {
+        token: "--max-auto-grade",
+        helpLabel: "--max-auto-grade <n>",
+        description: "Max ungraded skills to auto-grade per run (default: 5, 0 to disable)",
+      },
+      {
+        token: "--loop",
+        helpLabel: "--loop",
+        description: "Run in continuous loop mode (never stops)",
+      },
+      {
+        token: "--loop-interval",
+        helpLabel: "--loop-interval <s>",
+        description: "Seconds between iterations (default: 3600, min: 60)",
+      },
+      {
+        token: "--help",
+        helpLabel: "-h, --help",
+        description: "Show this help message",
+      },
+    ],
+    quickReference:
+      "selftune orchestrate [--dry-run] [--review-required] [--auto-approve] [--skill NAME] [--max-skills N] [--recent-window HOURS] [--sync-force] [--max-auto-grade N] [--loop] [--loop-interval SECS]",
+    extraHelpSections: [
+      `Safety:
+  By default, low-risk description evolution runs autonomously after
+  validation. Use --review-required to keep a human in the loop, or
+  --dry-run to preview the whole loop without mutations. Every deploy
+  still passes validation gates first.`,
+      `Examples:
+  selftune orchestrate                          # autonomous description evolution
+  selftune orchestrate --review-required        # validate but do not deploy
+  selftune orchestrate --dry-run                # preview only
+  selftune orchestrate --skill Research         # single skill
+  selftune orchestrate --max-skills 3           # limit scope
+  selftune orchestrate --loop                   # continuous loop (hourly)
+  selftune orchestrate --loop --loop-interval 600  # every 10 minutes`,
+    ],
+  },
+} satisfies Record<string, PublicCommandSurface>;

package/cli/selftune/constants.ts

@@ -13,6 +13,8 @@ const claudeHomeDir =
 const openclawHomeDir =
   process.env.SELFTUNE_OPENCLAW_DIR ??
   (resolvedHome ? join(defaultHome, ".openclaw") : join(homedir(), ".openclaw"));
+const piHomeDir =
+  process.env.SELFTUNE_PI_DIR ?? (resolvedHome ? join(defaultHome, ".pi") : join(homedir(), ".pi"));
 
 export const SELFTUNE_CONFIG_DIR =
   (process.env.SELFTUNE_CONFIG_DIR || undefined) ??
@@ -100,7 +102,7 @@ export const REQUIRED_FIELDS: Record<string, Set<string>> = {
 };
 
 /** Agent CLI candidates in detection order. */
-export const AGENT_CANDIDATES = ["claude", "codex", "opencode", "openclaw"] as const;
+export const AGENT_CANDIDATES = ["claude", "codex", "opencode", "openclaw", "pi"] as const;
 
 /** Required Claude Code hook keys in settings.json. */
 export const CLAUDE_CODE_HOOK_KEYS = [
@@ -158,6 +160,13 @@ export const OPENCLAW_AGENTS_DIR =
 /** Marker file tracking which OpenClaw sessions have been ingested. */
 export const OPENCLAW_INGEST_MARKER = join(SELFTUNE_CONFIG_DIR, "openclaw-ingest-marker.json");
 
+/** Pi sessions directory. */
+export const PI_SESSIONS_DIR =
+  process.env.SELFTUNE_PI_SESSIONS_DIR ?? join(piHomeDir, "agent", "sessions");
+
+/** Marker file tracking which Pi sessions have been ingested. */
+export const PI_INGEST_MARKER = join(SELFTUNE_CONFIG_DIR, "pi-ingest-marker.json");
+
 /** Default output directory for contribution bundles. */
 export const CONTRIBUTIONS_DIR = join(SELFTUNE_CONFIG_DIR, "contributions");
 /** Creator-directed contribution preferences (per-skill opt-in state). */

package/cli/selftune/contribute/contribute.ts

@@ -11,8 +11,11 @@ import { spawnSync } from "node:child_process";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { parseArgs } from "node:util";
 
-import { CONTRIBUTIONS_DIR } from "../constants.js";
+import { readAlphaIdentity } from "../alpha-identity.js";
+import { CONTRIBUTIONS_DIR, SELFTUNE_CONFIG_PATH } from "../constants.js";
+import { findCreatorContributionConfig } from "../contribution-config.js";
 import { handleCLIError } from "../utils/cli-error.js";
+import { getSelftuneVersion } from "../utils/selftune-meta.js";
 import { assembleBundle } from "./bundle.js";
 import { sanitizeBundle } from "./sanitize.js";
 
@@ -29,7 +32,7 @@ export async function cliMain(): Promise<void> {
       sanitize: { type: "string", default: "conservative" },
       since: { type: "string" },
       submit: { type: "boolean", default: false },
-      endpoint: { type: "string", default: "https://selftune-api.fly.dev" },
+      endpoint: { type: "string" },
       github: { type: "boolean", default: false },
       help: { type: "boolean", short: "h", default: false },
     },
@@ -37,16 +40,16 @@ export async function cliMain(): Promise<void> {
   });
 
   if (values.help) {
-    console.log(`selftune contribute — Export an anonymized community bundle
+    console.log(`selftune contribute — Export an anonymized community export bundle
 
 Usage:
   selftune contribute --skill <name> [--preview] [--sanitize conservative|aggressive]
   selftune contribute --skill <name> [--output <file>] [--submit]
 
 Purpose:
-  Build a sanitized community contribution bundle from local SQLite data.
+  Build a sanitized community export bundle from local SQLite data.
   This is separate from:
-    selftune contributions  Creator-directed sharing preferences
+    selftune contributions  Sharing preferences (creator-directed opt-in/out)
     selftune alpha upload   Personal cloud upload cycle
 
 Options:
@@ -131,7 +134,8 @@ Options:
       const ok = submitToGitHub(json, outputPath);
       if (!ok) process.exit(1);
     } else {
-      const endpoint = values.endpoint ?? "https://selftune-api.fly.dev";
+      const auth = getLocalAuthConfig();
+      const endpoint = values.endpoint ?? auth?.apiUrl ?? "https://api.selftune.dev";
       const ok = await submitToService(json, endpoint, skillName);
       if (!ok) {
         console.log("Falling back to GitHub submission...");
@@ -143,7 +147,27 @@ Options:
 }
 
 // ---------------------------------------------------------------------------
-// Service submission
+// Auth helpers
+// ---------------------------------------------------------------------------
+
+function getLocalAuthConfig(): { apiUrl: string; apiKey: string } | null {
+  try {
+    const identity = readAlphaIdentity(SELFTUNE_CONFIG_PATH);
+    if (!identity?.api_key) return null;
+    const apiUrl = identity.cloud_api_url || "https://api.selftune.dev";
+    return { apiUrl, apiKey: identity.api_key };
+  } catch {
+    return null;
+  }
+}
+
+function resolveCreatorId(skillName: string): string | null {
+  const config = findCreatorContributionConfig(skillName);
+  return config?.creator_id ?? null;
+}
+
+// ---------------------------------------------------------------------------
+// Service submission (cloud endpoint)
 // ---------------------------------------------------------------------------
 
 async function submitToService(
@@ -151,12 +175,39 @@ async function submitToService(
   endpoint: string,
   skillName: string,
 ): Promise<boolean> {
+  // Resolve creator_id from the installed selftune.contribute.json
+  const creatorId = resolveCreatorId(skillName);
+  if (!creatorId) {
+    console.error(
+      `[ERROR] No creator_id found for skill "${skillName}". ` +
+        `Ensure selftune.contribute.json exists in the skill directory with a valid creator_id.`,
+    );
+    return false;
+  }
+
+  // Resolve auth from local config
+  const auth = getLocalAuthConfig();
+
   try {
-    const url = `${endpoint}/api/submit`;
+    const url = `${endpoint}/api/v1/community/bundles`;
+    // Wrap the already-serialized bundle in the submission envelope
+    // without an unnecessary parse/stringify cycle
+    const payload = `{"creator_id":${JSON.stringify(creatorId)},"skill_name":${JSON.stringify(skillName)},"bundle":${json}}`;
+
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+      "User-Agent": `selftune/${getSelftuneVersion()}`,
+    };
+
+    if (auth?.apiKey) {
+      headers.Authorization = `Bearer ${auth.apiKey}`;
+    }
+
     const res = await fetch(url, {
       method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: json,
+      headers,
+      body: payload,
+      signal: AbortSignal.timeout(60_000),
     });
 
     if (!res.ok) {
@@ -165,9 +216,9 @@ async function submitToService(
       return false;
     }
 
-    console.log(`\nSubmitted to ${endpoint}`);
-    console.log(`  Badge: ${endpoint}/badge/${encodeURIComponent(skillName)}`);
-    console.log(`  Report: ${endpoint}/report/${encodeURIComponent(skillName)}`);
+    console.log(`\nSubmitted to ${endpoint}/api/v1/community/bundles`);
+    console.log(`  Skill: ${skillName}`);
+    console.log(`  Creator: ${creatorId}`);
     return true;
   } catch (err) {
     console.error(

package/cli/selftune/contribution-config.ts

@@ -7,21 +7,62 @@ import {
   findRepositorySkillDirs,
 } from "./utils/skill-discovery.js";
 
+/**
+ * The canonical UUID pattern for `creator_id`. This field must always be the
+ * creator's cloud user UUID (the `cloud_user_id` from alpha enrollment), e.g.
+ * "550e8400-e29b-41d4-a716-446655440000". Non-UUID values are accepted during
+ * local development but will be rejected by the relay endpoint.
+ */
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+export const SUPPORTED_CONTRIBUTION_SIGNALS = ["trigger", "grade", "miss_category"] as const;
+export type SupportedContributionSignal = (typeof SUPPORTED_CONTRIBUTION_SIGNALS)[number];
+
+/** Returns `true` when `value` looks like a valid UUID v4 (case-insensitive). */
+export function isValidCreatorUUID(value: string): boolean {
+  return UUID_RE.test(value);
+}
+
+export function isSupportedContributionSignal(value: string): value is SupportedContributionSignal {
+  return SUPPORTED_CONTRIBUTION_SIGNALS.includes(value as SupportedContributionSignal);
+}
+
+export function normalizeSupportedContributionSignals(
+  rawSignals: string[],
+): SupportedContributionSignal[] {
+  const normalized = [...new Set(rawSignals.map((signal) => signal.trim()).filter(Boolean))];
+  if (normalized.length === 0) {
+    throw new Error(
+      `At least one contribution signal is required. Supported signals: ${SUPPORTED_CONTRIBUTION_SIGNALS.join(", ")}`,
+    );
+  }
+
+  const invalid = normalized.filter((signal) => !isSupportedContributionSignal(signal));
+  if (invalid.length > 0) {
+    throw new Error(
+      `Unsupported contribution signals: ${invalid.join(", ")}. Supported signals: ${SUPPORTED_CONTRIBUTION_SIGNALS.join(", ")}`,
+    );
+  }
+
+  return normalized as SupportedContributionSignal[];
+}
+
 export interface CreatorContributionConfig {
   version: 1;
+  /** Must be the creator's cloud user UUID (`cloud_user_id`). */
   creator_id: string;
   skill_name: string;
   config_path: string;
   skill_path: string;
   contribution: {
     enabled: boolean;
-    signals: string[];
+    signals: SupportedContributionSignal[];
     message?: string;
     privacy_url?: string;
   };
 }
 
 export interface CreatorContributionConfigInput {
+  /** Must be the creator's cloud user UUID (`cloud_user_id`). */
   creator_id: string;
   skill_name: string;
   skill_path: string;
@@ -95,6 +136,13 @@ function normalizeContributionConfig(
     .filter(Boolean);
   if (signals.length === 0) return null;
 
+  if (!isValidCreatorUUID(creatorId)) {
+    process.stderr.write(
+      `[selftune] warning: creator_id "${creatorId}" is not a valid UUID. ` +
+        `Expected a cloud user UUID (e.g. "550e8400-e29b-41d4-a716-446655440000").\n`,
+    );
+  }
+
   return {
     version: 1,
     creator_id: creatorId,
@@ -103,7 +151,7 @@ function normalizeContributionConfig(
     skill_path: skillPath,
     contribution: {
       enabled: true,
-      signals: [...new Set(signals)],
+      signals: normalizeSupportedContributionSignals(signals),
       message: typeof raw.contribution.message === "string" ? raw.contribution.message : undefined,
       privacy_url:
         typeof raw.contribution.privacy_url === "string" ? raw.contribution.privacy_url : undefined,
@@ -150,6 +198,12 @@ export function resolveContributionSkillPath(
 export function writeCreatorContributionConfig(
   input: CreatorContributionConfigInput,
 ): CreatorContributionConfig {
+  if (!isValidCreatorUUID(input.creator_id)) {
+    throw new Error(
+      `creator_id must be the creator's cloud user UUID. Received "${input.creator_id}".`,
+    );
+  }
+  const signals = normalizeSupportedContributionSignals(input.signals);
   const normalized = normalizeContributionConfig(
     {
       version: 1,
@@ -157,7 +211,7 @@ export function writeCreatorContributionConfig(
       skill_name: input.skill_name,
       contribution: {
         enabled: true,
-        signals: input.signals,
+        signals,
         message: input.message,
         privacy_url: input.privacy_url,
       },