selftune 0.2.18 → 0.2.20

package/cli/selftune/creator-contributions.ts

@@ -0,0 +1,333 @@
+#!/usr/bin/env bun
+
+import { parseArgs } from "node:util";
+
+import { readAlphaIdentity } from "./alpha-identity.js";
+import { SELFTUNE_CONFIG_PATH } from "./constants.js";
+import {
+  type CreatorContributionConfig,
+  discoverCreatorContributionConfigs,
+  findCreatorContributionConfig,
+  getContributionConfigSearchRoots,
+  removeCreatorContributionConfig,
+  resolveContributionSkillPath,
+  writeCreatorContributionConfig,
+} from "./contribution-config.js";
+import { CLIError } from "./utils/cli-error.js";
+import { handleCLIError } from "./utils/cli-error.js";
+import { findInstalledSkillNames } from "./utils/skill-discovery.js";
+
+function inferCreatorId(explicitCreatorId?: string): string | null {
+  if (explicitCreatorId?.trim()) return explicitCreatorId.trim();
+  const alpha = readAlphaIdentity(SELFTUNE_CONFIG_PATH);
+  return alpha?.cloud_user_id?.trim() || null;
+}
+
+function printConfig(config: CreatorContributionConfig): void {
+  console.log(`${config.skill_name}`);
+  console.log(`  creator_id: ${config.creator_id}`);
+  console.log(`  skill_path: ${config.skill_path}`);
+  console.log(`  signals: ${config.contribution.signals.join(", ")}`);
+  if (config.contribution.message) {
+    console.log(`  message: ${config.contribution.message}`);
+  }
+  if (config.contribution.privacy_url) {
+    console.log(`  privacy_url: ${config.contribution.privacy_url}`);
+  }
+}
+
+function printStatus(skillName?: string): void {
+  const searchRoots = getContributionConfigSearchRoots();
+  const installedSkills = [...findInstalledSkillNames(searchRoots)].sort();
+  const configuredSkillNames = new Set(
+    discoverCreatorContributionConfigs(searchRoots).map((c) => c.skill_name),
+  );
+  if (skillName) {
+    const config = findCreatorContributionConfig(skillName);
+    if (!config) {
+      console.log(`No creator contribution config found for "${skillName}".`);
+      return;
+    }
+    console.log("Creator contribution config:");
+    printConfig(config);
+    return;
+  }
+
+  const configs = discoverCreatorContributionConfigs();
+  if (configs.length === 0) {
+    console.log("No creator contribution configs discovered.");
+    console.log("Use `selftune creator-contributions enable --skill <name>` to add one.");
+  } else {
+    console.log("Discovered creator contribution configs:");
+    for (const config of configs) {
+      printConfig(config);
+    }
+  }
+
+  const missingInstalled = installedSkills.filter((skill) => !configuredSkillNames.has(skill));
+  if (missingInstalled.length > 0) {
+    console.log("Installed skills without creator contribution config:");
+    for (const skill of missingInstalled) {
+      console.log(`  ${skill}`);
+    }
+  }
+}
+
+interface BulkEnableSkip {
+  skill_name: string;
+  reason: "already_configured" | "skill_path_not_found";
+}
+
+interface BulkEnableResult {
+  written: string[];
+  skipped: BulkEnableSkip[];
+}
+
+function enableCreatorContributionConfigs(options: {
+  skillName?: string;
+  all?: boolean;
+  prefix?: string;
+  explicitSkillPath?: string;
+  explicitCreatorId?: string;
+  signals: string[];
+  message?: string;
+  privacyUrl?: string;
+}): BulkEnableResult {
+  const creatorId = inferCreatorId(options.explicitCreatorId);
+  if (!creatorId) {
+    throw new CLIError(
+      "Creator ID is required.",
+      "MISSING_FLAG",
+      "Pass --creator-id <id> or enroll alpha so cloud_user_id is available.",
+    );
+  }
+
+  const searchRoots = getContributionConfigSearchRoots();
+  const targetSkills = options.all
+    ? [...findInstalledSkillNames(searchRoots)]
+        .filter((name) => !options.prefix || name.startsWith(options.prefix))
+        .sort()
+    : options.skillName
+      ? [options.skillName]
+      : [];
+
+  if (targetSkills.length === 0) {
+    throw new CLIError(
+      options.all
+        ? `No installed skills found${options.prefix ? ` with prefix "${options.prefix}"` : ""}.`
+        : "Skill name is required.",
+      options.all ? "FILE_NOT_FOUND" : "MISSING_FLAG",
+      options.all
+        ? "selftune creator-contributions status"
+        : "selftune creator-contributions enable --skill <name>",
+    );
+  }
+
+  const result: BulkEnableResult = { written: [], skipped: [] };
+  for (const skillName of targetSkills) {
+    if (findCreatorContributionConfig(skillName, searchRoots)) {
+      result.skipped.push({ skill_name: skillName, reason: "already_configured" });
+      continue;
+    }
+    const skillPath = resolveContributionSkillPath(
+      skillName,
+      options.all ? undefined : options.explicitSkillPath,
+      searchRoots,
+    );
+    if (!skillPath) {
+      result.skipped.push({ skill_name: skillName, reason: "skill_path_not_found" });
+      continue;
+    }
+
+    writeCreatorContributionConfig({
+      creator_id: creatorId,
+      skill_name: skillName,
+      skill_path: skillPath,
+      signals: options.signals,
+      message: options.message,
+      privacy_url: options.privacyUrl,
+    });
+    result.written.push(skillName);
+  }
+
+  return result;
+}
+
+export async function cliMain(): Promise<void> {
+  const sub = process.argv[2];
+  const rest = process.argv.slice(3);
+
+  if (sub === "--help" || sub === "-h") {
+    console.log(`selftune creator-contributions — Manage creator-side contribution configs
+
+Usage:
+  selftune creator-contributions
+  selftune creator-contributions status [--skill <name>]
+  selftune creator-contributions enable --skill <name> [--skill-path <path>] [--creator-id <id>] [--signals a,b,c]
+  selftune creator-contributions enable --all [--prefix <prefix>] [--creator-id <id>] [--signals a,b,c]
+  selftune creator-contributions disable --skill <name> [--skill-path <path>]
+
+Purpose:
+  Manage the local selftune.contribute.json file that a skill creator bundles
+  with a skill package. This is separate from:
+    selftune contributions  End-user sharing preferences
+    selftune contribute     Community export bundle`);
+    return;
+  }
+
+  const normalizedSub = sub ?? "status";
+
+  switch (normalizedSub) {
+    case "status": {
+      const { values } = parseArgs({
+        args: rest,
+        options: {
+          skill: { type: "string" },
+          help: { type: "boolean", short: "h", default: false },
+        },
+        strict: true,
+      });
+      if (values.help) {
+        console.log("Usage: selftune creator-contributions status [--skill <name>]");
+        return;
+      }
+      printStatus(values.skill);
+      return;
+    }
+    case "enable": {
+      const { values } = parseArgs({
+        args: rest,
+        options: {
+          skill: { type: "string" },
+          all: { type: "boolean", default: false },
+          prefix: { type: "string" },
+          "skill-path": { type: "string" },
+          "creator-id": { type: "string" },
+          signals: { type: "string", default: "trigger,grade,miss_category" },
+          message: { type: "string" },
+          "privacy-url": { type: "string" },
+          help: { type: "boolean", short: "h", default: false },
+        },
+        strict: true,
+      });
+      if (values.help) {
+        console.log(
+          "Usage: selftune creator-contributions enable (--skill <name> [--skill-path <path>] | --all [--prefix <prefix>]) [--creator-id <id>]",
+        );
+        return;
+      }
+
+      if (!values.all && !values.skill?.trim()) {
+        throw new CLIError(
+          "Pass either --skill <name> or --all.",
+          "MISSING_FLAG",
+          "selftune creator-contributions enable --skill <name>",
+        );
+      }
+
+      const signals = (values.signals ?? "trigger,grade,miss_category")
+        .split(",")
+        .map((signal) => signal.trim())
+        .filter(Boolean);
+      const outcome = enableCreatorContributionConfigs({
+        skillName: values.skill?.trim(),
+        all: values.all,
+        prefix: values.prefix?.trim(),
+        explicitSkillPath: values["skill-path"],
+        explicitCreatorId: values["creator-id"],
+        signals,
+        message: values.message,
+        privacyUrl: values["privacy-url"],
+      });
+      if (values.all) {
+        console.log(
+          `Enabled creator contribution config for ${outcome.written.length} skills${values.prefix ? ` with prefix "${values.prefix}"` : ""}.`,
+        );
+        if (outcome.written.length > 0) {
+          for (const skill of outcome.written) {
+            const config = findCreatorContributionConfig(skill);
+            if (config) printConfig(config);
+          }
+        }
+        if (outcome.skipped.length > 0) {
+          console.log(
+            `Skipped ${outcome.skipped.length} skills: ${outcome.skipped.map((entry) => entry.skill_name).join(", ")}`,
+          );
+        }
+        return;
+      }
+      const skillName = values.skill!.trim();
+      if (outcome.written.length === 0) {
+        const skip = outcome.skipped[0];
+        if (skip?.reason === "already_configured") {
+          throw new CLIError(
+            `A creator contribution config already exists for "${skillName}".`,
+            "FILE_EXISTS",
+            "Run `selftune creator-contributions status --skill <name>` to inspect it.",
+          );
+        }
+        throw new CLIError(
+          `Could not resolve SKILL.md for "${skillName}".`,
+          "FILE_NOT_FOUND",
+          "Pass --skill-path /path/to/SKILL.md",
+        );
+      }
+      const config = findCreatorContributionConfig(skillName);
+      console.log(`Enabled creator contribution config for "${skillName}".`);
+      if (config) printConfig(config);
+      return;
+    }
+    case "disable": {
+      const { values } = parseArgs({
+        args: rest,
+        options: {
+          skill: { type: "string" },
+          "skill-path": { type: "string" },
+          help: { type: "boolean", short: "h", default: false },
+        },
+        strict: true,
+      });
+      if (values.help) {
+        console.log(
+          "Usage: selftune creator-contributions disable --skill <name> [--skill-path <path>]",
+        );
+        return;
+      }
+
+      const skillName = values.skill?.trim();
+      if (!skillName) {
+        throw new CLIError(
+          "Skill name is required.",
+          "MISSING_FLAG",
+          "selftune creator-contributions disable --skill <name>",
+        );
+      }
+      const skillPath = resolveContributionSkillPath(skillName, values["skill-path"]);
+      if (!skillPath) {
+        throw new CLIError(
+          `Could not resolve SKILL.md for "${skillName}".`,
+          "FILE_NOT_FOUND",
+          "Pass --skill-path /path/to/SKILL.md",
+        );
+      }
+
+      const removed = removeCreatorContributionConfig(skillPath);
+      if (!removed) {
+        console.log(`No creator contribution config found for "${skillName}".`);
+        return;
+      }
+      console.log(`Disabled creator contribution config for "${skillName}".`);
+      return;
+    }
+    default:
+      throw new CLIError(
+        `Unknown creator-contributions subcommand: ${normalizedSub}`,
+        "UNKNOWN_COMMAND",
+        "selftune creator-contributions --help",
+      );
+  }
+}
+
+if (import.meta.main) {
+  cliMain().catch(handleCLIError);
+}

package/cli/selftune/dashboard-contract.ts

@@ -100,6 +100,10 @@ export interface EvolutionEntry {
   action: string;
   details: string;
   eval_snapshot?: EvalSnapshot | null;
+  validation_mode?: "structural_guard" | "host_replay" | "llm_judge" | null;
+  validation_agent?: string | null;
+  validation_fixture_id?: string | null;
+  validation_evidence_ref?: string | null;
 }
 
 export interface UnmatchedQuery {
@@ -134,6 +138,65 @@ export interface SkillSummary {
   unique_sessions: number;
   last_seen: string | null;
   has_evidence: boolean;
+  routing_confidence: number | null;
+  confidence_coverage: number;
+}
+
+// -- Autonomy-first overview types -------------------------------------------
+
+export type AutonomyStatusLevel = "healthy" | "watching" | "needs_review" | "blocked";
+
+export interface AutonomyStatus {
+  level: AutonomyStatusLevel;
+  summary: string;
+  last_run: string | null;
+  skills_observed: number;
+  pending_reviews: number;
+  attention_required: number;
+}
+
+export type AttentionCategory =
+  | "needs_review"
+  | "regression"
+  | "low_trust"
+  | "polluted"
+  | "blocked";
+
+export interface AttentionItem {
+  skill_name: string;
+  category: AttentionCategory;
+  severity: "critical" | "warning" | "info";
+  reason: string;
+  recommended_action: string;
+  timestamp: string;
+}
+
+export type TrustBucket = "at_risk" | "improving" | "uncertain" | "stable";
+
+export interface TrustWatchlistEntry {
+  skill_name: string;
+  bucket: TrustBucket;
+  trust_state: TrustState;
+  reason: string;
+  pass_rate: number | null;
+  checks: number;
+  last_seen: string | null;
+}
+
+export type DecisionKind =
+  | "proposal_created"
+  | "proposal_rejected"
+  | "validation_failed"
+  | "proposal_deployed"
+  | "rollback_triggered"
+  | "regression_found";
+
+export interface AutonomousDecision {
+  timestamp: string;
+  kind: DecisionKind;
+  skill_name: string;
+  proposal_id?: string;
+  summary: string;
 }
 
 export interface OverviewPayload {
@@ -158,6 +221,11 @@ export interface OverviewResponse {
   overview: OverviewPayload;
   skills: SkillSummary[];
   version?: string;
+  watched_skills: string[];
+  autonomy_status: AutonomyStatus;
+  attention_queue: AttentionItem[];
+  trust_watchlist: TrustWatchlistEntry[];
+  recent_decisions: AutonomousDecision[];
 }
 
 export interface EvidenceEntry {
@@ -188,6 +256,8 @@ export interface CanonicalInvocation {
   source?: string | null;
   skill_path?: string | null;
   skill_scope?: string | null;
+  observation_kind?: ObservationKind;
+  historical_context?: HistoricalContext | null;
 }
 
 export interface PromptSample {
@@ -266,6 +336,48 @@ export interface OrchestrateRunsResponse {
   runs: OrchestrateRunReport[];
 }
 
+// -- Performance analytics response -------------------------------------------
+
+export interface AnalyticsResponse {
+  /** Daily pass rate trend (last 90 days, bucketed by day) */
+  pass_rate_trend: Array<{
+    date: string;
+    pass_rate: number;
+    total_checks: number;
+  }>;
+
+  /** Skills ranked by pass rate with trend direction */
+  skill_rankings: Array<{
+    skill_name: string;
+    pass_rate: number;
+    total_checks: number;
+    triggered_count: number;
+  }>;
+
+  /** Daily check counts for heatmap (last 84 days / 12 weeks) */
+  daily_activity: Array<{
+    date: string;
+    checks: number;
+  }>;
+
+  /** Evolution impact — before/after pass rates for deployed evolutions */
+  evolution_impact: Array<{
+    skill_name: string;
+    proposal_id: string;
+    deployed_at: string;
+    pass_rate_before: number;
+    pass_rate_after: number;
+  }>;
+
+  /** Aggregate summary */
+  summary: {
+    total_evolutions: number;
+    avg_improvement: number;
+    total_checks_30d: number;
+    active_skills: number;
+  };
+}
+
 // -- Health endpoint response -------------------------------------------------
 
 export interface HealthResponse {
@@ -318,7 +430,103 @@ export interface CommitSummary {
   recent_commits: Array<{ sha: string; title: string; branch: string; timestamp: string }>;
 }
 
-export interface SkillReportResponse extends SkillReportPayload {
+// -- Trust-oriented types for skill report ------------------------------------
+
+export type TrustState =
+  | "low_sample"
+  | "observed"
+  | "watch"
+  | "validated"
+  | "deployed"
+  | "rolled_back";
+
+export type ObservationKind =
+  | "canonical"
+  | "repaired_trigger"
+  | "repaired_contextual_miss"
+  | "legacy_materialized";
+
+export type HistoricalContext = "previously_missed";
+
+export interface ExampleRow {
+  timestamp: string | null;
+  session_id: string;
+  query_text: string;
+  triggered: boolean;
+  confidence: number | null;
+  invocation_mode: string | null;
+  prompt_kind: string | null;
+  source: string | null;
+  platform: string | null;
+  workspace_path: string | null;
+  query_origin: "inline_query" | "matched_prompt" | "missing";
+  is_system_like: boolean;
+  observation_kind: ObservationKind;
+  historical_context?: HistoricalContext | null;
+}
+
+export interface TrustFields {
+  trust: {
+    state: TrustState;
+    summary: string;
+  };
+  coverage: {
+    checks: number;
+    sessions: number;
+    workspaces: number;
+    first_seen: string | null;
+    last_seen: string | null;
+  };
+  evidence_quality: {
+    prompt_link_rate: number;
+    inline_query_rate: number;
+    user_prompt_rate: number;
+    meta_prompt_rate: number;
+    internal_prompt_rate: number;
+    no_prompt_rate: number;
+    system_like_rate: number;
+    invocation_mode_coverage: number;
+    confidence_coverage: number;
+    source_coverage: number;
+    scope_coverage: number;
+  };
+  routing_quality: {
+    missed_triggers: number;
+    miss_rate: number;
+    avg_confidence: number | null;
+    confidence_coverage: number;
+    low_confidence_rate: number | null;
+  };
+  evolution_state: {
+    has_evidence: boolean;
+    has_pending_proposals: boolean;
+    latest_action: string | null;
+    latest_timestamp: string | null;
+    evidence_rows: number;
+    evolution_rows: number;
+  };
+  data_hygiene: {
+    naming_variants: string[];
+    source_breakdown: Array<{ source: string; count: number }>;
+    prompt_kind_breakdown: Array<{ kind: string; count: number }>;
+    observation_breakdown: Array<{ kind: ObservationKind; count: number }>;
+    raw_checks: number;
+    operational_checks: number;
+    internal_prompt_rows: number;
+    internal_prompt_rate: number;
+    legacy_rows: number;
+    legacy_rate: number;
+    repaired_rows: number;
+    repaired_rate: number;
+  };
+  examples: {
+    good: ExampleRow[];
+    missed: ExampleRow[];
+    noisy: ExampleRow[];
+  };
+}
+
+export interface SkillReportResponse extends SkillReportPayload, TrustFields {
   evolution: EvolutionEntry[];
   pending_proposals: PendingProposal[];
   token_usage: {