selftune 0.2.16 → 0.2.19

package/cli/selftune/localdb/schema.ts

@@ -239,6 +239,21 @@ CREATE TABLE IF NOT EXISTS upload_queue (
   last_error    TEXT
 )`;
 
+// -- Creator contribution staging --------------------------------------------
+
+export const CREATE_CREATOR_CONTRIBUTION_STAGING = `
+CREATE TABLE IF NOT EXISTS creator_contribution_staging (
+  id           INTEGER PRIMARY KEY AUTOINCREMENT,
+  dedupe_key   TEXT NOT NULL,
+  skill_name   TEXT NOT NULL,
+  creator_id   TEXT NOT NULL,
+  payload_json TEXT NOT NULL,
+  status       TEXT NOT NULL DEFAULT 'pending',
+  staged_at    TEXT NOT NULL,
+  updated_at   TEXT NOT NULL,
+  last_error   TEXT
+)`;
+
 // -- Canonical upload staging -------------------------------------------------
 
 export const CREATE_CANONICAL_UPLOAD_STAGING = `
@@ -260,6 +275,20 @@ CREATE TABLE IF NOT EXISTS upload_watermarks (
   updated_at       TEXT NOT NULL
 )`;
 
+// -- Commit tracking table ----------------------------------------------------
+
+export const CREATE_COMMIT_TRACKING = `
+CREATE TABLE IF NOT EXISTS commit_tracking (
+  id            INTEGER PRIMARY KEY AUTOINCREMENT,
+  session_id    TEXT NOT NULL,
+  commit_sha    TEXT NOT NULL,
+  commit_title  TEXT,
+  branch        TEXT,
+  repo_remote   TEXT,
+  timestamp     TEXT NOT NULL,
+  created_at    TEXT NOT NULL DEFAULT (datetime('now'))
+)`;
+
 // -- Metadata table -----------------------------------------------------------
 
 export const CREATE_META = `
@@ -313,10 +342,19 @@ export const CREATE_INDEXES = [
   // -- Alpha upload queue indexes ---------------------------------------------
   `CREATE INDEX IF NOT EXISTS idx_upload_queue_status ON upload_queue(status)`,
   `CREATE INDEX IF NOT EXISTS idx_upload_queue_type_status ON upload_queue(payload_type, status)`,
+  // -- Creator contribution staging indexes -----------------------------------
+  `CREATE INDEX IF NOT EXISTS idx_creator_contrib_status ON creator_contribution_staging(status)`,
+  `CREATE INDEX IF NOT EXISTS idx_creator_contrib_skill ON creator_contribution_staging(skill_name)`,
+  `CREATE UNIQUE INDEX IF NOT EXISTS idx_creator_contrib_dedup ON creator_contribution_staging(dedupe_key)`,
   // -- Canonical upload staging indexes ---------------------------------------
   `CREATE INDEX IF NOT EXISTS idx_staging_kind ON canonical_upload_staging(record_kind)`,
   `CREATE INDEX IF NOT EXISTS idx_staging_session ON canonical_upload_staging(session_id)`,
   `CREATE UNIQUE INDEX IF NOT EXISTS idx_staging_dedup ON canonical_upload_staging(record_kind, record_id)`,
+  // -- Commit tracking indexes ------------------------------------------------
+  `CREATE INDEX IF NOT EXISTS idx_commit_sha ON commit_tracking(commit_sha)`,
+  `CREATE INDEX IF NOT EXISTS idx_commit_session ON commit_tracking(session_id)`,
+  `CREATE INDEX IF NOT EXISTS idx_commit_ts ON commit_tracking(timestamp)`,
+  `CREATE UNIQUE INDEX IF NOT EXISTS idx_commit_dedup ON commit_tracking(session_id, commit_sha)`,
 ];
 
 /**
@@ -353,6 +391,31 @@ export const MIGRATIONS = [
   `ALTER TABLE execution_facts ADD COLUMN normalizer_version TEXT`,
   `ALTER TABLE execution_facts ADD COLUMN capture_mode TEXT`,
   `ALTER TABLE execution_facts ADD COLUMN raw_source_ref TEXT`,
+  // -- Win 2+3: File change metrics + token granularity + cost (execution_facts) --
+  `ALTER TABLE execution_facts ADD COLUMN files_changed INTEGER`,
+  `ALTER TABLE execution_facts ADD COLUMN lines_added INTEGER`,
+  `ALTER TABLE execution_facts ADD COLUMN lines_removed INTEGER`,
+  `ALTER TABLE execution_facts ADD COLUMN lines_modified INTEGER`,
+  `ALTER TABLE execution_facts ADD COLUMN cached_input_tokens INTEGER`,
+  `ALTER TABLE execution_facts ADD COLUMN reasoning_output_tokens INTEGER`,
+  `ALTER TABLE execution_facts ADD COLUMN cost_usd REAL`,
+  // -- Win 2+3: File change metrics + token granularity + cost (session_telemetry) --
+  `ALTER TABLE session_telemetry ADD COLUMN files_changed INTEGER`,
+  `ALTER TABLE session_telemetry ADD COLUMN lines_added INTEGER`,
+  `ALTER TABLE session_telemetry ADD COLUMN lines_removed INTEGER`,
+  `ALTER TABLE session_telemetry ADD COLUMN lines_modified INTEGER`,
+  `ALTER TABLE session_telemetry ADD COLUMN cached_input_tokens INTEGER`,
+  `ALTER TABLE session_telemetry ADD COLUMN reasoning_output_tokens INTEGER`,
+  `ALTER TABLE session_telemetry ADD COLUMN cost_usd REAL`,
+  // -- Generalized metrics: artifact count + session type --
+  `ALTER TABLE execution_facts ADD COLUMN artifact_count INTEGER`,
+  `ALTER TABLE execution_facts ADD COLUMN session_type TEXT`,
+  `ALTER TABLE session_telemetry ADD COLUMN artifact_count INTEGER`,
+  `ALTER TABLE session_telemetry ADD COLUMN session_type TEXT`,
+  // -- Session summary (heuristic, no LLM) --
+  `ALTER TABLE session_telemetry ADD COLUMN agent_summary TEXT`,
+  // -- SHA256 content hashing for upload dedup --
+  `ALTER TABLE canonical_upload_staging ADD COLUMN content_sha256 TEXT`,
 ];
 
 /** Indexes that depend on migration columns — must run AFTER MIGRATIONS. */
@@ -360,6 +423,7 @@ export const POST_MIGRATION_INDEXES = [
   `CREATE INDEX IF NOT EXISTS idx_skill_inv_query_triggered ON skill_invocations(query, triggered)`,
   `CREATE INDEX IF NOT EXISTS idx_skill_inv_scope ON skill_invocations(skill_name, skill_scope, occurred_at)`,
   `CREATE INDEX IF NOT EXISTS idx_skill_inv_dedup ON skill_invocations(session_id, skill_name, query, occurred_at, triggered)`,
+  `CREATE INDEX IF NOT EXISTS idx_staging_sha256 ON canonical_upload_staging(content_sha256)`,
 ];
 
 /** All DDL statements in creation order. */
@@ -377,8 +441,10 @@ export const ALL_DDL = [
   CREATE_GRADING_RESULTS,
   CREATE_IMPROVEMENT_SIGNALS,
   CREATE_UPLOAD_QUEUE,
+  CREATE_CREATOR_CONTRIBUTION_STAGING,
   CREATE_UPLOAD_WATERMARKS,
   CREATE_CANONICAL_UPLOAD_STAGING,
+  CREATE_COMMIT_TRACKING,
   CREATE_META,
   ...CREATE_INDEXES,
 ];

package/cli/selftune/orchestrate.ts

@@ -16,9 +16,9 @@ import { parseArgs } from "node:util";
 
 import { readAlphaIdentity } from "./alpha-identity.js";
 import type { UploadCycleSummary } from "./alpha-upload/index.js";
-import { ORCHESTRATE_LOCK, SELFTUNE_CONFIG_PATH } from "./constants.js";
+import { getOrchestrateLockPath, SELFTUNE_CONFIG_PATH } from "./constants.js";
 import type { OrchestrateRunReport, OrchestrateRunSkillAction } from "./dashboard-contract.js";
-import type { EvolveResult } from "./evolution/evolve.js";
+import type { EvolveOptions, EvolveResult } from "./evolution/evolve.js";
 import {
   buildDefaultGradingOutputPath,
   deriveExpectationsFromSkill,
@@ -74,7 +74,7 @@ interface LockInfo {
 
 const LOCK_STALE_MS = 30 * 60 * 1000; // 30 minutes
 
-export function acquireLock(lockPath: string = ORCHESTRATE_LOCK): boolean {
+export function acquireLock(lockPath: string = getOrchestrateLockPath()): boolean {
   try {
     if (existsSync(lockPath)) {
       try {
@@ -98,7 +98,7 @@ export function acquireLock(lockPath: string = ORCHESTRATE_LOCK): boolean {
   }
 }
 
-export function releaseLock(lockPath: string = ORCHESTRATE_LOCK): void {
+export function releaseLock(lockPath: string = getOrchestrateLockPath()): void {
   try {
     unlinkSync(lockPath);
   } catch {
@@ -377,6 +377,33 @@ export const MIN_CANDIDATE_EVIDENCE = 3;
 /** Default cooldown hours after a deploy before re-evolving the same skill. */
 export const DEFAULT_COOLDOWN_HOURS = 24;
 
+type AutonomousEvolveDefaults = Pick<
+  EvolveOptions,
+  | "paretoEnabled"
+  | "candidateCount"
+  | "tokenEfficiencyEnabled"
+  | "withBaseline"
+  | "validationModel"
+  | "cheapLoop"
+  | "gateModel"
+  | "adaptiveGate"
+  | "proposalModel"
+>;
+
+// Keep the autonomous loop aligned with the evolve CLI defaults so scheduled
+// runs stay cheap by default and still get a stronger gate before deploy.
+const AUTONOMOUS_EVOLVE_DEFAULTS: AutonomousEvolveDefaults = {
+  paretoEnabled: true,
+  candidateCount: 3,
+  tokenEfficiencyEnabled: false,
+  withBaseline: false,
+  validationModel: "haiku",
+  cheapLoop: true,
+  gateModel: "sonnet",
+  adaptiveGate: true,
+  proposalModel: "haiku",
+};
+
 function candidatePriority(skill: SkillStatus, signalCount = 0): number {
   const statusWeight = skill.status === "CRITICAL" ? 300 : skill.status === "WARNING" ? 200 : 100;
   const missedWeight = Math.min(skill.missedQueries, 50);
@@ -1012,6 +1039,7 @@ export async function orchestrate(
           maxIterations: 3,
           gradingResults: _readGradingResults(candidate.skill),
           syncFirst: false, // We already synced
+          ...AUTONOMOUS_EVOLVE_DEFAULTS,
         });
 
         candidate.evolveResult = evolveResult;

package/cli/selftune/recover.ts

@@ -0,0 +1,153 @@
+#!/usr/bin/env bun
+
+import { parseArgs } from "node:util";
+
+import {
+  CANONICAL_LOG,
+  EVOLUTION_AUDIT_LOG,
+  EVOLUTION_EVIDENCE_LOG,
+  ORCHESTRATE_RUN_LOG,
+  TELEMETRY_LOG,
+} from "./constants.js";
+import { getDb } from "./localdb/db.js";
+import {
+  materializeFull,
+  materializeIncremental,
+  type MaterializeOptions,
+  type MaterializeResult,
+} from "./localdb/materialize.js";
+import { CLIError, handleCLIError } from "./utils/cli-error.js";
+
+interface RecoverSummary {
+  mode: "incremental" | "full";
+  source: "legacy_jsonl_or_export_snapshot";
+  since: string | null;
+  force: boolean;
+  result: MaterializeResult;
+}
+
+function buildMaterializeOptions(values: Record<string, unknown>): MaterializeOptions {
+  return {
+    canonicalLogPath: (values["canonical-log"] as string | undefined) ?? CANONICAL_LOG,
+    telemetryLogPath: (values["telemetry-log"] as string | undefined) ?? TELEMETRY_LOG,
+    evolutionAuditPath:
+      (values["evolution-audit-log"] as string | undefined) ?? EVOLUTION_AUDIT_LOG,
+    evolutionEvidencePath:
+      (values["evolution-evidence-log"] as string | undefined) ?? EVOLUTION_EVIDENCE_LOG,
+    orchestrateRunLogPath:
+      (values["orchestrate-run-log"] as string | undefined) ?? ORCHESTRATE_RUN_LOG,
+    force: (values.force as boolean | undefined) ?? false,
+  };
+}
+
+function printHumanSummary(summary: RecoverSummary): void {
+  const rows = [
+    `mode: ${summary.mode}`,
+    "source: legacy JSONL or explicit export snapshot",
+    `sessions: ${summary.result.sessions}`,
+    `prompts: ${summary.result.prompts}`,
+    `skill invocations: ${summary.result.skillInvocations}`,
+    `execution facts: ${summary.result.executionFacts}`,
+    `session telemetry: ${summary.result.sessionTelemetry}`,
+    `legacy skill usage: ${summary.result.skillUsage}`,
+    `evolution audit: ${summary.result.evolutionAudit}`,
+    `evolution evidence: ${summary.result.evolutionEvidence}`,
+    `orchestrate runs: ${summary.result.orchestrateRuns}`,
+  ];
+  console.log(`selftune recover\n${rows.map((row) => `  ${row}`).join("\n")}`);
+}
+
+export function cliMain(): void {
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      full: { type: "boolean", default: false },
+      force: { type: "boolean", default: false },
+      since: { type: "string" },
+      json: { type: "boolean", default: false },
+      "canonical-log": { type: "string", default: CANONICAL_LOG },
+      "telemetry-log": { type: "string", default: TELEMETRY_LOG },
+      "evolution-audit-log": { type: "string", default: EVOLUTION_AUDIT_LOG },
+      "evolution-evidence-log": { type: "string", default: EVOLUTION_EVIDENCE_LOG },
+      "orchestrate-run-log": { type: "string", default: ORCHESTRATE_RUN_LOG },
+      help: { type: "boolean", short: "h", default: false },
+    },
+    strict: true,
+  });
+
+  if (values.help) {
+    console.log(`selftune recover — Recover SQLite from legacy/exported JSONL
+
+Usage:
+  selftune recover [options]
+
+Use this only for legacy backfill or explicit export-based recovery. Normal
+operation should use \`selftune sync\`, which replays native source data into
+SQLite and preserves alpha-upload compatibility.
+
+Options:
+  --full                           Rebuild SQLite tables from scratch
+  --force                          Skip preflight rebuild guard for SQLite-only rows
+  --since <date>                   Incrementally materialize records on/after date
+  --canonical-log <path>           Canonical JSONL path
+  --telemetry-log <path>           Session telemetry JSONL path
+  --evolution-audit-log <path>     Evolution audit JSONL path
+  --evolution-evidence-log <path>  Evolution evidence JSONL path
+  --orchestrate-run-log <path>     Orchestrate runs JSONL path
+  --json                           Output JSON summary
+  -h, --help                       Show this help`);
+    process.exit(0);
+  }
+
+  if (values.full && values.since) {
+    throw new CLIError(
+      "Cannot combine --full with --since.",
+      "INVALID_FLAG",
+      "Use either `selftune recover --full` or `selftune recover --since 2026-01-01`.",
+    );
+  }
+
+  let sinceIso: string | null = null;
+  if (values.since) {
+    const parsed = new Date(values.since as string);
+    if (Number.isNaN(parsed.getTime())) {
+      throw new CLIError(
+        `Invalid --since date: ${values.since}`,
+        "INVALID_FLAG",
+        "selftune recover --since 2026-01-01",
+      );
+    }
+    sinceIso = parsed.toISOString();
+  }
+
+  const db = getDb();
+  const materializeOptions = buildMaterializeOptions(values);
+  if (!values.full) materializeOptions.since = sinceIso;
+
+  const result = values.full
+    ? materializeFull(db, materializeOptions)
+    : materializeIncremental(db, materializeOptions);
+
+  const summary: RecoverSummary = {
+    mode: values.full ? "full" : "incremental",
+    source: "legacy_jsonl_or_export_snapshot",
+    since: sinceIso,
+    force: (values.force as boolean | undefined) ?? false,
+    result,
+  };
+
+  if (values.json || !process.stdout.isTTY) {
+    console.log(JSON.stringify(summary, null, 2));
+    return;
+  }
+
+  printHumanSummary(summary);
+}
+
+if (import.meta.main) {
+  try {
+    cliMain();
+  } catch (error) {
+    handleCLIError(error);
+  }
+}