selftune 0.2.6 → 0.2.9

package/cli/selftune/localdb/schema.ts

@@ -21,7 +21,10 @@ CREATE TABLE IF NOT EXISTS sessions (
   repo_remote       TEXT,
   branch            TEXT,
   schema_version    TEXT,
-  normalized_at     TEXT
+  normalized_at     TEXT,
+  normalizer_version TEXT,
+  capture_mode       TEXT,
+  raw_source_ref     TEXT
 )`;
 
 export const CREATE_PROMPTS = `
@@ -33,6 +36,12 @@ CREATE TABLE IF NOT EXISTS prompts (
   is_actionable INTEGER,
   prompt_index  INTEGER,
   prompt_text   TEXT,
+  schema_version     TEXT,
+  platform           TEXT,
+  normalized_at      TEXT,
+  normalizer_version TEXT,
+  capture_mode       TEXT,
+  raw_source_ref     TEXT,
   FOREIGN KEY (session_id) REFERENCES sessions(session_id)
 )`;
 
@@ -47,6 +56,17 @@ CREATE TABLE IF NOT EXISTS skill_invocations (
   confidence          REAL,
   tool_name           TEXT,
   matched_prompt_id   TEXT,
+  agent_type          TEXT,
+  query               TEXT,
+  skill_path          TEXT,
+  skill_scope         TEXT,
+  source              TEXT,
+  schema_version      TEXT,
+  platform            TEXT,
+  normalized_at       TEXT,
+  normalizer_version  TEXT,
+  capture_mode        TEXT,
+  raw_source_ref      TEXT,
   FOREIGN KEY (session_id) REFERENCES sessions(session_id)
 )`;
 
@@ -64,6 +84,12 @@ CREATE TABLE IF NOT EXISTS execution_facts (
   output_tokens       INTEGER,
   duration_ms         INTEGER,
   completion_status   TEXT,
+  schema_version      TEXT,
+  platform            TEXT,
+  normalized_at       TEXT,
+  normalizer_version  TEXT,
+  capture_mode        TEXT,
+  raw_source_ref      TEXT,
   FOREIGN KEY (session_id) REFERENCES sessions(session_id)
 )`;
 
@@ -151,6 +177,67 @@ CREATE TABLE IF NOT EXISTS orchestrate_runs (
   skill_actions_json TEXT NOT NULL
 )`;
 
+// -- Query log table (from all_queries_log.jsonl) ----------------------------
+
+export const CREATE_QUERIES = `
+CREATE TABLE IF NOT EXISTS queries (
+  id          INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp   TEXT NOT NULL,
+  session_id  TEXT NOT NULL,
+  query       TEXT NOT NULL,
+  source      TEXT
+)`;
+
+// -- Improvement signal table (from signal_log.jsonl) ------------------------
+
+export const CREATE_IMPROVEMENT_SIGNALS = `
+CREATE TABLE IF NOT EXISTS improvement_signals (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp       TEXT NOT NULL,
+  session_id      TEXT NOT NULL,
+  query           TEXT NOT NULL,
+  signal_type     TEXT NOT NULL,
+  mentioned_skill TEXT,
+  consumed        INTEGER NOT NULL DEFAULT 0,
+  consumed_at     TEXT,
+  consumed_by_run TEXT
+)`;
+
+// -- Alpha upload queue -------------------------------------------------------
+
+export const CREATE_UPLOAD_QUEUE = `
+CREATE TABLE IF NOT EXISTS upload_queue (
+  id            INTEGER PRIMARY KEY AUTOINCREMENT,
+  payload_type  TEXT NOT NULL,
+  payload_json  TEXT NOT NULL,
+  status        TEXT NOT NULL DEFAULT 'pending',
+  attempts      INTEGER NOT NULL DEFAULT 0,
+  created_at    TEXT NOT NULL,
+  updated_at    TEXT NOT NULL,
+  last_error    TEXT
+)`;
+
+// -- Canonical upload staging -------------------------------------------------
+
+export const CREATE_CANONICAL_UPLOAD_STAGING = `
+CREATE TABLE IF NOT EXISTS canonical_upload_staging (
+  local_seq     INTEGER PRIMARY KEY AUTOINCREMENT,
+  record_kind   TEXT NOT NULL,
+  record_id     TEXT NOT NULL,
+  record_json   TEXT NOT NULL,
+  session_id    TEXT,
+  prompt_id     TEXT,
+  normalized_at TEXT,
+  staged_at     TEXT NOT NULL
+)`;
+
+export const CREATE_UPLOAD_WATERMARKS = `
+CREATE TABLE IF NOT EXISTS upload_watermarks (
+  payload_type     TEXT PRIMARY KEY,
+  last_uploaded_id INTEGER NOT NULL,
+  updated_at       TEXT NOT NULL
+)`;
+
 // -- Metadata table -----------------------------------------------------------
 
 export const CREATE_META = `
@@ -167,6 +254,7 @@ export const CREATE_INDEXES = [
   `CREATE INDEX IF NOT EXISTS idx_prompts_occurred ON prompts(occurred_at)`,
   `CREATE INDEX IF NOT EXISTS idx_skill_inv_session ON skill_invocations(session_id)`,
   `CREATE INDEX IF NOT EXISTS idx_skill_inv_name ON skill_invocations(skill_name)`,
+  `CREATE INDEX IF NOT EXISTS idx_skill_inv_ts ON skill_invocations(occurred_at)`,
   `CREATE INDEX IF NOT EXISTS idx_exec_facts_session ON execution_facts(session_id)`,
   `CREATE INDEX IF NOT EXISTS idx_evo_evidence_proposal ON evolution_evidence(proposal_id)`,
   `CREATE INDEX IF NOT EXISTS idx_evo_evidence_skill ON evolution_evidence(skill_name)`,
@@ -186,6 +274,65 @@ export const CREATE_INDEXES = [
   `CREATE UNIQUE INDEX IF NOT EXISTS idx_evo_evidence_dedup ON evolution_evidence(proposal_id, stage, timestamp)`,
   // -- Orchestrate run indexes -----------------------------------------------
   `CREATE INDEX IF NOT EXISTS idx_orchestrate_runs_ts ON orchestrate_runs(timestamp)`,
+  // -- Query log indexes ------------------------------------------------------
+  `CREATE INDEX IF NOT EXISTS idx_queries_session ON queries(session_id)`,
+  `CREATE INDEX IF NOT EXISTS idx_queries_ts ON queries(timestamp)`,
+  `CREATE UNIQUE INDEX IF NOT EXISTS idx_queries_dedup ON queries(session_id, query, timestamp)`,
+  // -- Improvement signal indexes ---------------------------------------------
+  `CREATE INDEX IF NOT EXISTS idx_signals_session ON improvement_signals(session_id)`,
+  `CREATE INDEX IF NOT EXISTS idx_signals_consumed ON improvement_signals(consumed)`,
+  `CREATE INDEX IF NOT EXISTS idx_signals_ts ON improvement_signals(timestamp)`,
+  `CREATE UNIQUE INDEX IF NOT EXISTS idx_signals_dedup ON improvement_signals(session_id, query, signal_type, timestamp)`,
+  // -- Alpha upload queue indexes ---------------------------------------------
+  `CREATE INDEX IF NOT EXISTS idx_upload_queue_status ON upload_queue(status)`,
+  `CREATE INDEX IF NOT EXISTS idx_upload_queue_type_status ON upload_queue(payload_type, status)`,
+  // -- Canonical upload staging indexes ---------------------------------------
+  `CREATE INDEX IF NOT EXISTS idx_staging_kind ON canonical_upload_staging(record_kind)`,
+  `CREATE INDEX IF NOT EXISTS idx_staging_session ON canonical_upload_staging(session_id)`,
+  `CREATE UNIQUE INDEX IF NOT EXISTS idx_staging_dedup ON canonical_upload_staging(record_kind, record_id)`,
+];
+
+/**
+ * Schema migrations — ALTER TABLE statements for columns added after initial release.
+ * Each is safe to re-run: SQLite throws "duplicate column" which openDb() catches.
+ */
+export const MIGRATIONS = [
+  // skill_invocations consolidation (skill_usage columns merged in)
+  `ALTER TABLE skill_invocations ADD COLUMN query TEXT`,
+  `ALTER TABLE skill_invocations ADD COLUMN skill_path TEXT`,
+  `ALTER TABLE skill_invocations ADD COLUMN skill_scope TEXT`,
+  `ALTER TABLE skill_invocations ADD COLUMN source TEXT`,
+  // Track how many iteration loops each evolution run used
+  `ALTER TABLE evolution_audit ADD COLUMN iterations_used INTEGER`,
+  // Canonical contract fields for upload staging (sessions already has schema_version, platform, normalized_at)
+  `ALTER TABLE sessions ADD COLUMN normalizer_version TEXT`,
+  `ALTER TABLE sessions ADD COLUMN capture_mode TEXT`,
+  `ALTER TABLE sessions ADD COLUMN raw_source_ref TEXT`,
+  `ALTER TABLE prompts ADD COLUMN schema_version TEXT`,
+  `ALTER TABLE prompts ADD COLUMN platform TEXT`,
+  `ALTER TABLE prompts ADD COLUMN normalized_at TEXT`,
+  `ALTER TABLE prompts ADD COLUMN normalizer_version TEXT`,
+  `ALTER TABLE prompts ADD COLUMN capture_mode TEXT`,
+  `ALTER TABLE prompts ADD COLUMN raw_source_ref TEXT`,
+  `ALTER TABLE skill_invocations ADD COLUMN schema_version TEXT`,
+  `ALTER TABLE skill_invocations ADD COLUMN platform TEXT`,
+  `ALTER TABLE skill_invocations ADD COLUMN normalized_at TEXT`,
+  `ALTER TABLE skill_invocations ADD COLUMN normalizer_version TEXT`,
+  `ALTER TABLE skill_invocations ADD COLUMN capture_mode TEXT`,
+  `ALTER TABLE skill_invocations ADD COLUMN raw_source_ref TEXT`,
+  `ALTER TABLE execution_facts ADD COLUMN schema_version TEXT`,
+  `ALTER TABLE execution_facts ADD COLUMN platform TEXT`,
+  `ALTER TABLE execution_facts ADD COLUMN normalized_at TEXT`,
+  `ALTER TABLE execution_facts ADD COLUMN normalizer_version TEXT`,
+  `ALTER TABLE execution_facts ADD COLUMN capture_mode TEXT`,
+  `ALTER TABLE execution_facts ADD COLUMN raw_source_ref TEXT`,
+];
+
+/** Indexes that depend on migration columns — must run AFTER MIGRATIONS. */
+export const POST_MIGRATION_INDEXES = [
+  `CREATE INDEX IF NOT EXISTS idx_skill_inv_query_triggered ON skill_invocations(query, triggered)`,
+  `CREATE INDEX IF NOT EXISTS idx_skill_inv_scope ON skill_invocations(skill_name, skill_scope, occurred_at)`,
+  `CREATE INDEX IF NOT EXISTS idx_skill_inv_dedup ON skill_invocations(session_id, skill_name, query, occurred_at, triggered)`,
 ];
 
 /** All DDL statements in creation order. */
@@ -199,6 +346,11 @@ export const ALL_DDL = [
   CREATE_SESSION_TELEMETRY,
   CREATE_SKILL_USAGE,
   CREATE_ORCHESTRATE_RUNS,
+  CREATE_QUERIES,
+  CREATE_IMPROVEMENT_SIGNALS,
+  CREATE_UPLOAD_QUEUE,
+  CREATE_UPLOAD_WATERMARKS,
+  CREATE_CANONICAL_UPLOAD_STAGING,
   CREATE_META,
   ...CREATE_INDEXES,
 ];

package/cli/selftune/monitoring/watch.ts

@@ -11,6 +11,12 @@ import { parseArgs } from "node:util";
 import { QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "../constants.js";
 import { classifyInvocation } from "../eval/hooks-to-evals.js";
 import { getLastDeployedProposal } from "../evolution/audit.js";
+import { getDb } from "../localdb/db.js";
+import {
+  queryQueryLog,
+  querySessionTelemetry,
+  querySkillUsageRecords,
+} from "../localdb/queries.js";
 import { updateContextAfterWatch } from "../memory/writer.js";
 import type { SyncResult } from "../sync.js";
 import type {
@@ -25,7 +31,6 @@ import {
   filterActionableQueryRecords,
   filterActionableSkillUsageRecords,
 } from "../utils/query-filter.js";
-import { readEffectiveSkillUsageRecords } from "../utils/skill-log.js";
 
 // ---------------------------------------------------------------------------
 // Public interfaces
@@ -207,13 +212,27 @@ export async function watch(options: WatchOptions): Promise<WatchResult> {
     );
   }
 
-  // 1. Read log files
-  const telemetry = readJsonl<SessionTelemetryRecord>(_telemetryLogPath);
-  const skillRecords =
-    _skillLogPath === SKILL_LOG
-      ? readEffectiveSkillUsageRecords()
-      : readJsonl<SkillUsageRecord>(_skillLogPath);
-  const queryRecords = readJsonl<QueryLogRecord>(_queryLogPath);
+  // 1. Read log files from SQLite (fall back to JSONL for custom paths)
+  let telemetry: SessionTelemetryRecord[];
+  let skillRecords: SkillUsageRecord[];
+  let queryRecords: QueryLogRecord[];
+  if (
+    _telemetryLogPath === TELEMETRY_LOG &&
+    _skillLogPath === SKILL_LOG &&
+    _queryLogPath === QUERY_LOG
+  ) {
+    const db = getDb();
+    telemetry = querySessionTelemetry(db) as SessionTelemetryRecord[];
+    // SQLite queries return DESC order; computeMonitoringSnapshot expects chronological (ASC)
+    telemetry.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+    skillRecords = querySkillUsageRecords(db) as SkillUsageRecord[];
+    queryRecords = queryQueryLog(db) as QueryLogRecord[];
+  } else {
+    // Intentional JSONL fallback: custom log path overrides bypass SQLite reads
+    telemetry = readJsonl<SessionTelemetryRecord>(_telemetryLogPath);
+    skillRecords = readJsonl<SkillUsageRecord>(_skillLogPath);
+    queryRecords = readJsonl<QueryLogRecord>(_queryLogPath);
+  }
 
   // 2. Determine baseline pass rate from last deployed audit entry
   const lastDeployed = getLastDeployedProposal(skillName, _auditLogPath);

package/cli/selftune/normalization.ts

@@ -2,7 +2,7 @@
  * Canonical telemetry normalization helpers.
  *
  * This module provides shared functions that all platform adapters call
- * to produce canonical records alongside their raw JSONL output.
+ * to produce canonical records written to SQLite via writeCanonicalToDb().
  *
  * Contract rules (from telemetry-field-map.md):
  *   1. Normalization is additive — raw capture is preserved separately.
@@ -25,6 +25,7 @@ import {
 } from "node:fs";
 import { basename, dirname } from "node:path";
 import { CANONICAL_LOG, canonicalSessionStatePath } from "./constants.js";
+import { writeCanonicalBatchToDb, writeCanonicalToDb } from "./localdb/direct-write.js";
 import {
   CANONICAL_SCHEMA_VERSION,
   type CanonicalCaptureMode,
@@ -81,9 +82,49 @@ function defaultPromptSessionState(sessionId: string): CanonicalPromptSessionSta
 
 function derivePromptSessionStateFromCanonicalLog(
   sessionId: string,
-  canonicalLogPath: string = CANONICAL_LOG,
+  _canonicalLogPath: string = CANONICAL_LOG,
 ): CanonicalPromptSessionState {
   const recovered = defaultPromptSessionState(sessionId);
+
+  // Try SQLite first — canonical records now go to the local DB.
+  // Uses dynamic require + try/catch so this remains fail-safe during
+  // hook execution when the DB module may not be loadable.
+  try {
+    const { getDb } = require("./localdb/db.js") as {
+      getDb: () => import("bun:sqlite").Database;
+    };
+    const db = getDb();
+    const rows = db
+      .query(
+        "SELECT prompt_id, prompt_index, is_actionable FROM prompts WHERE session_id = ? ORDER BY prompt_index DESC LIMIT 1",
+      )
+      .all(sessionId) as Array<{
+      prompt_id: string;
+      prompt_index: number;
+      is_actionable: number;
+    }>;
+    if (rows.length > 0) {
+      const row = rows[0];
+      recovered.next_prompt_index = row.prompt_index + 1;
+      recovered.last_prompt_id = row.prompt_id;
+      // Get last actionable
+      const actionable = db
+        .query(
+          "SELECT prompt_id, prompt_index FROM prompts WHERE session_id = ? AND is_actionable = 1 ORDER BY prompt_index DESC LIMIT 1",
+        )
+        .get(sessionId) as { prompt_id: string; prompt_index: number } | null;
+      if (actionable) recovered.last_actionable_prompt_id = actionable.prompt_id;
+      return recovered;
+    }
+    if (_canonicalLogPath === CANONICAL_LOG) {
+      return recovered;
+    }
+  } catch {
+    // DB unavailable — fall through to JSONL recovery below.
+  }
+
+  // Fallback: scan canonical JSONL log (legacy path or DB unavailable).
+  const canonicalLogPath = _canonicalLogPath;
   let maxPromptIndex = -1;
   let maxActionablePromptIndex = -1;
 
@@ -346,22 +387,32 @@ export function getLatestPromptIdentity(
   };
 }
 
-export function appendCanonicalRecord(
-  record: CanonicalRecord,
-  logPath: string = CANONICAL_LOG,
-): void {
-  const dir = dirname(logPath);
-  if (!existsSync(dir)) {
-    mkdirSync(dir, { recursive: true });
+export function appendCanonicalRecord(record: CanonicalRecord, logPath?: string): void {
+  writeCanonicalToDb(record);
+  // JSONL append — best-effort backup for prompt state recovery
+  try {
+    const path = logPath ?? CANONICAL_LOG;
+    const dir = dirname(path);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+    appendFileSync(path, `${JSON.stringify(record)}\n`, "utf-8");
+  } catch {
+    /* best-effort only */
   }
-  appendFileSync(logPath, `${JSON.stringify(record)}\n`, "utf-8");
 }
 
-export function appendCanonicalRecords(
-  records: CanonicalRecord[],
-  logPath: string = CANONICAL_LOG,
-): void {
-  for (const record of records) appendCanonicalRecord(record, logPath);
+export function appendCanonicalRecords(records: CanonicalRecord[], logPath?: string): void {
+  writeCanonicalBatchToDb(records);
+  // JSONL append — best-effort backup for prompt state recovery
+  try {
+    const path = logPath ?? CANONICAL_LOG;
+    const dir = dirname(path);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+    for (const record of records) {
+      appendFileSync(path, `${JSON.stringify(record)}\n`, "utf-8");
+    }
+  } catch {
+    /* best-effort only */
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -439,14 +490,34 @@ export interface InvocationClassification {
 
 /**
  * Classify how a skill was invoked.
+ *
+ * When `hook_invocation_type` is provided (from the skill-eval hook's
+ * classifyInvocationType), it takes precedence over the legacy heuristics:
+ *   - "explicit"   → user typed /skill (slash command)            → explicit,  confidence 1.0
+ *   - "implicit"   → user named the skill, Claude invoked it      → implicit,  confidence 0.85
+ *   - "inferred"   → Claude chose skill autonomously               → inferred,  confidence 0.6
+ *   - "contextual" → SKILL.md was read (Read tool, not Skill tool) → inferred,  confidence 0.5
  */
 export function deriveInvocationMode(opts: {
   has_skill_tool_call?: boolean;
   has_skill_md_read?: boolean;
   is_text_mention_only?: boolean;
   is_repaired?: boolean;
+  hook_invocation_type?: "explicit" | "implicit" | "inferred" | "contextual";
 }): InvocationClassification {
   if (opts.is_repaired) return { invocation_mode: "repaired", confidence: 0.9 };
+
+  // Prefer hook-level classification when available
+  if (opts.hook_invocation_type === "explicit")
+    return { invocation_mode: "explicit", confidence: 1.0 };
+  if (opts.hook_invocation_type === "implicit")
+    return { invocation_mode: "implicit", confidence: 0.85 };
+  if (opts.hook_invocation_type === "inferred")
+    return { invocation_mode: "inferred", confidence: 0.6 };
+  if (opts.hook_invocation_type === "contextual")
+    return { invocation_mode: "inferred", confidence: 0.5 };
+
+  // Legacy fallback for callers that don't pass hook_invocation_type
   if (opts.has_skill_tool_call) return { invocation_mode: "explicit", confidence: 1.0 };
   if (opts.has_skill_md_read) return { invocation_mode: "implicit", confidence: 0.7 };
   if (opts.is_text_mention_only) return { invocation_mode: "inferred", confidence: 0.4 };
@@ -613,6 +684,7 @@ export interface BuildSkillInvocationInput extends CanonicalBaseInput {
   confidence: number;
   tool_name?: string;
   tool_call_id?: string;
+  agent_type?: string;
 }
 
 export function buildCanonicalSkillInvocation(
@@ -636,6 +708,7 @@ export function buildCanonicalSkillInvocation(
   if (input.skill_version_hash !== undefined) record.skill_version_hash = input.skill_version_hash;
   if (input.tool_name !== undefined) record.tool_name = input.tool_name;
   if (input.tool_call_id !== undefined) record.tool_call_id = input.tool_call_id;
+  if (input.agent_type !== undefined) record.agent_type = input.agent_type;
 
   return record;
 }