selftune 0.1.0 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/LICENSE +21 -0
- package/README.md +62 -5
- package/bin/selftune.cjs +3 -1
- package/cli/selftune/constants.ts +0 -6
- package/cli/selftune/dashboard.ts +176 -0
- package/cli/selftune/evolution/evolve.ts +31 -20
- package/cli/selftune/evolution/propose-description.ts +2 -3
- package/cli/selftune/evolution/rollback.ts +1 -1
- package/cli/selftune/evolution/validate-proposal.ts +3 -4
- package/cli/selftune/grading/grade-session.ts +18 -62
- package/cli/selftune/index.ts +21 -0
- package/cli/selftune/ingestors/codex-rollout.ts +1 -1
- package/cli/selftune/ingestors/opencode-ingest.ts +2 -2
- package/cli/selftune/init.ts +25 -36
- package/cli/selftune/last.ts +138 -0
- package/cli/selftune/observability.ts +1 -1
- package/cli/selftune/status.ts +318 -0
- package/cli/selftune/types.ts +1 -1
- package/cli/selftune/utils/llm-call.ts +8 -57
- package/dashboard/index.html +1119 -0
- package/package.json +32 -3
- package/skill/SKILL.md +19 -14
- package/skill/Workflows/Doctor.md +2 -9
- package/skill/Workflows/Evals.md +8 -17
- package/skill/Workflows/Evolve.md +5 -11
- package/skill/Workflows/Grade.md +5 -10
- package/skill/Workflows/Ingest.md +16 -34
- package/skill/Workflows/Initialize.md +32 -34
- package/skill/Workflows/Rollback.md +3 -10
- package/skill/Workflows/Watch.md +2 -9
package/cli/selftune/index.ts
CHANGED
|
@@ -13,6 +13,9 @@
|
|
|
13
13
|
* selftune rollback [options] — Rollback a skill to its pre-evolution state
|
|
14
14
|
* selftune watch [options] — Monitor post-deploy skill health
|
|
15
15
|
* selftune doctor — Run health checks
|
|
16
|
+
* selftune status — Show skill health summary
|
|
17
|
+
* selftune last — Show last session details
|
|
18
|
+
* selftune dashboard [options] — Open visual data dashboard
|
|
16
19
|
*/
|
|
17
20
|
|
|
18
21
|
const command = process.argv[2];
|
|
@@ -34,6 +37,9 @@ Commands:
|
|
|
34
37
|
rollback Rollback a skill to its pre-evolution state
|
|
35
38
|
watch Monitor post-deploy skill health
|
|
36
39
|
doctor Run health checks
|
|
40
|
+
status Show skill health summary
|
|
41
|
+
last Show last session details
|
|
42
|
+
dashboard Open visual data dashboard
|
|
37
43
|
|
|
38
44
|
Run 'selftune <command> --help' for command-specific options.`);
|
|
39
45
|
process.exit(0);
|
|
@@ -98,6 +104,21 @@ switch (command) {
|
|
|
98
104
|
process.exit(result.healthy ? 0 : 1);
|
|
99
105
|
break;
|
|
100
106
|
}
|
|
107
|
+
case "status": {
|
|
108
|
+
const { cliMain } = await import("./status.js");
|
|
109
|
+
cliMain();
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
case "last": {
|
|
113
|
+
const { cliMain } = await import("./last.js");
|
|
114
|
+
cliMain();
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
case "dashboard": {
|
|
118
|
+
const { cliMain } = await import("./dashboard.js");
|
|
119
|
+
cliMain();
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
101
122
|
default:
|
|
102
123
|
console.error(`Unknown command: ${command}\nRun 'selftune --help' for available commands.`);
|
|
103
124
|
process.exit(1);
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
* bun codex-rollout.ts --force
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
-
import { existsSync,
|
|
24
|
+
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
25
25
|
import { homedir } from "node:os";
|
|
26
26
|
import { basename, join } from "node:path";
|
|
27
27
|
import { parseArgs } from "node:util";
|
|
@@ -21,12 +21,12 @@
|
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
23
|
import { Database } from "bun:sqlite";
|
|
24
|
-
import { existsSync,
|
|
24
|
+
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
25
25
|
import { homedir } from "node:os";
|
|
26
26
|
import { basename, join } from "node:path";
|
|
27
27
|
import { parseArgs } from "node:util";
|
|
28
28
|
import { QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "../constants.js";
|
|
29
|
-
import type { QueryLogRecord,
|
|
29
|
+
import type { QueryLogRecord, SkillUsageRecord } from "../types.js";
|
|
30
30
|
import { appendJsonl, loadMarker, saveMarker } from "../utils/jsonl.js";
|
|
31
31
|
|
|
32
32
|
const XDG_DATA_HOME = process.env.XDG_DATA_HOME ?? join(homedir(), ".local", "share");
|
package/cli/selftune/init.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* the result to ~/.selftune/config.json.
|
|
8
8
|
*
|
|
9
9
|
* Usage:
|
|
10
|
-
* selftune init [--agent <type>] [--cli-path <path>] [--
|
|
10
|
+
* selftune init [--agent <type>] [--cli-path <path>] [--force]
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
@@ -40,6 +40,16 @@ const VALID_AGENT_TYPES: SelftuneConfig["agent_type"][] = [
|
|
|
40
40
|
"unknown",
|
|
41
41
|
];
|
|
42
42
|
|
|
43
|
+
const AGENT_TYPE_CLI_MAP: Record<string, string> = {
|
|
44
|
+
claude_code: "claude",
|
|
45
|
+
codex: "codex",
|
|
46
|
+
opencode: "opencode",
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
function agentTypeToCli(agentType: string): string | null {
|
|
50
|
+
return AGENT_TYPE_CLI_MAP[agentType] ?? null;
|
|
51
|
+
}
|
|
52
|
+
|
|
43
53
|
export function detectAgentType(
|
|
44
54
|
override?: string,
|
|
45
55
|
homeOverride?: string,
|
|
@@ -95,34 +105,11 @@ export function determineCliPath(override?: string): string {
|
|
|
95
105
|
/**
|
|
96
106
|
* Determine LLM mode and agent CLI based on available signals.
|
|
97
107
|
*/
|
|
98
|
-
export function determineLlmMode(
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
const detectedAgent = agentCli;
|
|
104
|
-
const validModes = ["agent", "api"] as const;
|
|
105
|
-
if (modeOverride && !validModes.includes(modeOverride as (typeof validModes)[number])) {
|
|
106
|
-
throw new Error(
|
|
107
|
-
`Invalid --llm-mode "${modeOverride}". Allowed values: ${validModes.join(", ")}`,
|
|
108
|
-
);
|
|
109
|
-
}
|
|
110
|
-
const resolvedMode = modeOverride as "agent" | "api" | undefined;
|
|
111
|
-
|
|
112
|
-
if (resolvedMode) {
|
|
113
|
-
return { llm_mode: resolvedMode, agent_cli: detectedAgent };
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
if (detectedAgent) {
|
|
117
|
-
return { llm_mode: "agent", agent_cli: detectedAgent };
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
if (hasApiKey) {
|
|
121
|
-
return { llm_mode: "api", agent_cli: null };
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
// Fallback: agent mode with null cli (will need setup)
|
|
125
|
-
return { llm_mode: "agent", agent_cli: null };
|
|
108
|
+
export function determineLlmMode(agentCli: string | null): {
|
|
109
|
+
llm_mode: "agent";
|
|
110
|
+
agent_cli: string | null;
|
|
111
|
+
} {
|
|
112
|
+
return { llm_mode: "agent", agent_cli: agentCli };
|
|
126
113
|
}
|
|
127
114
|
|
|
128
115
|
// ---------------------------------------------------------------------------
|
|
@@ -170,7 +157,6 @@ export interface InitOptions {
|
|
|
170
157
|
force: boolean;
|
|
171
158
|
agentOverride?: string;
|
|
172
159
|
cliPathOverride?: string;
|
|
173
|
-
llmModeOverride?: string;
|
|
174
160
|
homeDir?: string;
|
|
175
161
|
}
|
|
176
162
|
|
|
@@ -203,12 +189,17 @@ export function runInit(opts: InitOptions): SelftuneConfig {
|
|
|
203
189
|
// Resolve CLI path
|
|
204
190
|
const cliPath = determineCliPath(opts.cliPathOverride);
|
|
205
191
|
|
|
206
|
-
// Detect agent CLI
|
|
207
|
-
|
|
192
|
+
// Detect agent CLI — when an override is provided, fall back to mapped CLI
|
|
193
|
+
// name so init works in test/CI environments without agent binaries in PATH
|
|
194
|
+
const agentCli = detectAgent() ?? (opts.agentOverride ? agentTypeToCli(agentType) : null);
|
|
195
|
+
if (!agentCli) {
|
|
196
|
+
throw new Error(
|
|
197
|
+
"No supported agent CLI detected (claude, codex, opencode). Install one, then rerun `selftune init`.",
|
|
198
|
+
);
|
|
199
|
+
}
|
|
208
200
|
|
|
209
201
|
// Determine LLM mode
|
|
210
|
-
const
|
|
211
|
-
const { llm_mode, agent_cli } = determineLlmMode(agentCli, hasApiKey, opts.llmModeOverride);
|
|
202
|
+
const { llm_mode, agent_cli } = determineLlmMode(agentCli);
|
|
212
203
|
|
|
213
204
|
// Check hooks (Claude Code only)
|
|
214
205
|
const home = opts.homeDir ?? homedir();
|
|
@@ -240,7 +231,6 @@ export async function cliMain(): Promise<void> {
|
|
|
240
231
|
options: {
|
|
241
232
|
agent: { type: "string" },
|
|
242
233
|
"cli-path": { type: "string" },
|
|
243
|
-
"llm-mode": { type: "string" },
|
|
244
234
|
force: { type: "boolean", default: false },
|
|
245
235
|
},
|
|
246
236
|
strict: true,
|
|
@@ -271,7 +261,6 @@ export async function cliMain(): Promise<void> {
|
|
|
271
261
|
force,
|
|
272
262
|
agentOverride: values.agent,
|
|
273
263
|
cliPathOverride: values["cli-path"],
|
|
274
|
-
llmModeOverride: values["llm-mode"],
|
|
275
264
|
});
|
|
276
265
|
|
|
277
266
|
console.log(JSON.stringify(config, null, 2));
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Quick insight from the most recent session.
|
|
4
|
+
* Lightweight, no LLM calls.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "./constants.js";
|
|
8
|
+
import type { QueryLogRecord, SessionTelemetryRecord, SkillUsageRecord } from "./types.js";
|
|
9
|
+
import { readJsonl } from "./utils/jsonl.js";
|
|
10
|
+
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
// Types
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
|
|
15
|
+
export interface LastSessionInsight {
|
|
16
|
+
sessionId: string;
|
|
17
|
+
timestamp: string;
|
|
18
|
+
skillsTriggered: string[];
|
|
19
|
+
unmatchedQueries: string[];
|
|
20
|
+
errors: number;
|
|
21
|
+
toolCalls: number;
|
|
22
|
+
recommendation: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Pure logic
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Compute insight from the most recent session.
|
|
31
|
+
* Returns null when no telemetry data exists.
|
|
32
|
+
*/
|
|
33
|
+
export function computeLastInsight(
|
|
34
|
+
telemetry: SessionTelemetryRecord[],
|
|
35
|
+
skillRecords: SkillUsageRecord[],
|
|
36
|
+
queryRecords: QueryLogRecord[],
|
|
37
|
+
): LastSessionInsight | null {
|
|
38
|
+
if (telemetry.length === 0) return null;
|
|
39
|
+
|
|
40
|
+
// Find most recent telemetry record
|
|
41
|
+
const sorted = [...telemetry].sort(
|
|
42
|
+
(a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime(),
|
|
43
|
+
);
|
|
44
|
+
const latest = sorted[0];
|
|
45
|
+
const sessionId = latest.session_id;
|
|
46
|
+
|
|
47
|
+
// Skills triggered: unique skill names where triggered=true AND session matches
|
|
48
|
+
const triggeredSkillQueries = new Set<string>();
|
|
49
|
+
const skillsTriggered = [
|
|
50
|
+
...new Set(
|
|
51
|
+
skillRecords
|
|
52
|
+
.filter((r) => r.session_id === sessionId && r.triggered)
|
|
53
|
+
.map((r) => {
|
|
54
|
+
triggeredSkillQueries.add(r.query.toLowerCase().trim());
|
|
55
|
+
return r.skill_name;
|
|
56
|
+
}),
|
|
57
|
+
),
|
|
58
|
+
];
|
|
59
|
+
|
|
60
|
+
// Unmatched queries: session queries whose text does NOT appear in any triggered skill record
|
|
61
|
+
const sessionQueries = queryRecords.filter((r) => r.session_id === sessionId);
|
|
62
|
+
const unmatchedQueries = sessionQueries
|
|
63
|
+
.filter((q) => !triggeredSkillQueries.has(q.query.toLowerCase().trim()))
|
|
64
|
+
.map((q) => q.query);
|
|
65
|
+
|
|
66
|
+
const errors = latest.errors_encountered;
|
|
67
|
+
const toolCalls = latest.total_tool_calls;
|
|
68
|
+
|
|
69
|
+
// Contextual recommendation
|
|
70
|
+
let recommendation: string;
|
|
71
|
+
const unmatched = unmatchedQueries.length;
|
|
72
|
+
if (unmatched > 0) {
|
|
73
|
+
recommendation = `${unmatched} queries had no skill match. Run 'selftune evals --list-skills' to investigate.`;
|
|
74
|
+
} else if (errors > 0) {
|
|
75
|
+
recommendation = `${errors} errors encountered. Check logs for details.`;
|
|
76
|
+
} else {
|
|
77
|
+
recommendation = "All queries matched skills. System is operating well.";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return {
|
|
81
|
+
sessionId,
|
|
82
|
+
timestamp: latest.timestamp,
|
|
83
|
+
skillsTriggered,
|
|
84
|
+
unmatchedQueries,
|
|
85
|
+
errors,
|
|
86
|
+
toolCalls,
|
|
87
|
+
recommendation,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// ---------------------------------------------------------------------------
|
|
92
|
+
// Formatting
|
|
93
|
+
// ---------------------------------------------------------------------------
|
|
94
|
+
|
|
95
|
+
/** Format an insight as a human-readable plain-text summary. */
|
|
96
|
+
export function formatInsight(insight: LastSessionInsight): string {
|
|
97
|
+
const date = new Date(insight.timestamp);
|
|
98
|
+
const month = date.toLocaleString("en-US", { month: "short" });
|
|
99
|
+
const day = date.getDate();
|
|
100
|
+
const hours = String(date.getHours()).padStart(2, "0");
|
|
101
|
+
const minutes = String(date.getMinutes()).padStart(2, "0");
|
|
102
|
+
const dateStr = `${month} ${day}, ${hours}:${minutes}`;
|
|
103
|
+
|
|
104
|
+
const lines: string[] = [];
|
|
105
|
+
lines.push(`Last session: ${insight.sessionId} (${dateStr})`);
|
|
106
|
+
lines.push("");
|
|
107
|
+
lines.push(` Skills triggered: ${insight.skillsTriggered.join(", ") || "none"}`);
|
|
108
|
+
lines.push(` Unmatched queries: ${insight.unmatchedQueries.length}`);
|
|
109
|
+
for (const q of insight.unmatchedQueries) {
|
|
110
|
+
lines.push(` \u00B7 "${q}"`);
|
|
111
|
+
}
|
|
112
|
+
lines.push(` Errors: ${insight.errors}`);
|
|
113
|
+
lines.push(` Tool calls: ${insight.toolCalls}`);
|
|
114
|
+
lines.push("");
|
|
115
|
+
lines.push(` \u2192 ${insight.recommendation}`);
|
|
116
|
+
|
|
117
|
+
return lines.join("\n");
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// ---------------------------------------------------------------------------
|
|
121
|
+
// CLI entry point
|
|
122
|
+
// ---------------------------------------------------------------------------
|
|
123
|
+
|
|
124
|
+
/** CLI main: reads logs, prints insight. */
|
|
125
|
+
export function cliMain(): void {
|
|
126
|
+
const telemetry = readJsonl<SessionTelemetryRecord>(TELEMETRY_LOG);
|
|
127
|
+
const skillRecords = readJsonl<SkillUsageRecord>(SKILL_LOG);
|
|
128
|
+
const queryRecords = readJsonl<QueryLogRecord>(QUERY_LOG);
|
|
129
|
+
|
|
130
|
+
const insight = computeLastInsight(telemetry, skillRecords, queryRecords);
|
|
131
|
+
if (!insight) {
|
|
132
|
+
console.log("No session data found.");
|
|
133
|
+
process.exit(0);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
console.log(formatInsight(insight));
|
|
137
|
+
process.exit(0);
|
|
138
|
+
}
|
|
@@ -16,7 +16,7 @@ import { LOG_DIR, REQUIRED_FIELDS, SELFTUNE_CONFIG_PATH } from "./constants.js";
|
|
|
16
16
|
import type { DoctorResult, HealthCheck, HealthStatus, SelftuneConfig } from "./types.js";
|
|
17
17
|
|
|
18
18
|
const VALID_AGENT_TYPES = new Set(["claude_code", "codex", "opencode", "unknown"]);
|
|
19
|
-
const VALID_LLM_MODES = new Set(["agent"
|
|
19
|
+
const VALID_LLM_MODES = new Set(["agent"]);
|
|
20
20
|
|
|
21
21
|
const LOG_FILES: Record<string, string> = {
|
|
22
22
|
session_telemetry: join(LOG_DIR, "session_telemetry_log.jsonl"),
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* selftune status — Skill health summary CLI command.
|
|
3
|
+
*
|
|
4
|
+
* Exports:
|
|
5
|
+
* - computeStatus() (pure function, deterministic)
|
|
6
|
+
* - formatStatus() (colored terminal output)
|
|
7
|
+
* - cliMain() (reads logs, runs doctor, prints output)
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { EVOLUTION_AUDIT_LOG, QUERY_LOG, SKILL_LOG, TELEMETRY_LOG } from "./constants.js";
|
|
11
|
+
import { computeMonitoringSnapshot } from "./monitoring/watch.js";
|
|
12
|
+
import { doctor } from "./observability.js";
|
|
13
|
+
import type {
|
|
14
|
+
DoctorResult,
|
|
15
|
+
EvolutionAuditEntry,
|
|
16
|
+
MonitoringSnapshot,
|
|
17
|
+
QueryLogRecord,
|
|
18
|
+
SessionTelemetryRecord,
|
|
19
|
+
SkillUsageRecord,
|
|
20
|
+
} from "./types.js";
|
|
21
|
+
import { readJsonl } from "./utils/jsonl.js";
|
|
22
|
+
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
// Result types
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
|
|
27
|
+
export interface SkillStatus {
|
|
28
|
+
name: string;
|
|
29
|
+
passRate: number | null;
|
|
30
|
+
trend: "up" | "down" | "stable" | "unknown";
|
|
31
|
+
missedQueries: number;
|
|
32
|
+
status: "HEALTHY" | "REGRESSED" | "NO DATA";
|
|
33
|
+
snapshot: MonitoringSnapshot | null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface StatusResult {
|
|
37
|
+
skills: SkillStatus[];
|
|
38
|
+
unmatchedQueries: number;
|
|
39
|
+
pendingProposals: number;
|
|
40
|
+
lastSession: string | null;
|
|
41
|
+
system: {
|
|
42
|
+
healthy: boolean;
|
|
43
|
+
pass: number;
|
|
44
|
+
fail: number;
|
|
45
|
+
warn: number;
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// ---------------------------------------------------------------------------
|
|
50
|
+
// Constants
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
|
|
53
|
+
const DEFAULT_WINDOW_SESSIONS = 20;
|
|
54
|
+
const DEFAULT_BASELINE_PASS_RATE = 0.5;
|
|
55
|
+
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
// computeStatus — pure function
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
export function computeStatus(
|
|
61
|
+
telemetry: SessionTelemetryRecord[],
|
|
62
|
+
skillRecords: SkillUsageRecord[],
|
|
63
|
+
queryRecords: QueryLogRecord[],
|
|
64
|
+
auditEntries: EvolutionAuditEntry[],
|
|
65
|
+
doctorResult: DoctorResult,
|
|
66
|
+
): StatusResult {
|
|
67
|
+
// Derive unique skill names from skill records
|
|
68
|
+
const skillNames = [...new Set(skillRecords.map((r) => r.skill_name))];
|
|
69
|
+
|
|
70
|
+
// Build per-skill status
|
|
71
|
+
const skills: SkillStatus[] = skillNames.map((skillName) => {
|
|
72
|
+
const skillSpecificRecords = skillRecords.filter((r) => r.skill_name === skillName);
|
|
73
|
+
const triggeredRecords = skillSpecificRecords.filter((r) => r.triggered);
|
|
74
|
+
|
|
75
|
+
// Get baseline from last deployed proposal
|
|
76
|
+
const lastDeployed = getLastDeployedProposalFromEntries(auditEntries, skillName);
|
|
77
|
+
const baselinePassRate = lastDeployed?.eval_snapshot?.pass_rate ?? DEFAULT_BASELINE_PASS_RATE;
|
|
78
|
+
|
|
79
|
+
// Compute monitoring snapshot
|
|
80
|
+
const snapshot = computeMonitoringSnapshot(
|
|
81
|
+
skillName,
|
|
82
|
+
telemetry,
|
|
83
|
+
skillRecords,
|
|
84
|
+
queryRecords,
|
|
85
|
+
DEFAULT_WINDOW_SESSIONS,
|
|
86
|
+
baselinePassRate,
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
// Determine if there's any meaningful data
|
|
90
|
+
const totalQueries = queryRecords.length;
|
|
91
|
+
const hasData = triggeredRecords.length > 0 || totalQueries > 0;
|
|
92
|
+
|
|
93
|
+
// Compute pass rate (null if no data)
|
|
94
|
+
let passRate: number | null = null;
|
|
95
|
+
if (hasData && totalQueries > 0) {
|
|
96
|
+
passRate = snapshot.pass_rate;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Determine trend: compare first-half vs second-half pass rates
|
|
100
|
+
const trend = computeTrend(skillSpecificRecords);
|
|
101
|
+
|
|
102
|
+
// Count missed queries for this skill (queries where skill was checked but not triggered)
|
|
103
|
+
const missedQueries = skillSpecificRecords.filter((r) => !r.triggered).length;
|
|
104
|
+
|
|
105
|
+
// Determine status
|
|
106
|
+
let status: "HEALTHY" | "REGRESSED" | "NO DATA";
|
|
107
|
+
if (!hasData || passRate === null) {
|
|
108
|
+
status = "NO DATA";
|
|
109
|
+
} else if (snapshot.regression_detected) {
|
|
110
|
+
status = "REGRESSED";
|
|
111
|
+
} else {
|
|
112
|
+
status = "HEALTHY";
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return { name: skillName, passRate, trend, missedQueries, status, snapshot };
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
// Sort: REGRESSED first, then HEALTHY, then NO DATA
|
|
119
|
+
const statusOrder = { REGRESSED: 0, HEALTHY: 1, "NO DATA": 2 };
|
|
120
|
+
skills.sort((a, b) => statusOrder[a.status] - statusOrder[b.status]);
|
|
121
|
+
|
|
122
|
+
// Unmatched queries: queries whose text appears in zero triggered skill_usage_log entries
|
|
123
|
+
const triggeredQueryTexts = new Set(
|
|
124
|
+
skillRecords.filter((r) => r.triggered).map((r) => r.query.toLowerCase().trim()),
|
|
125
|
+
);
|
|
126
|
+
const unmatchedQueries = queryRecords.filter(
|
|
127
|
+
(q) => !triggeredQueryTexts.has(q.query.toLowerCase().trim()),
|
|
128
|
+
).length;
|
|
129
|
+
|
|
130
|
+
// Pending proposals: audit entries with action=created/validated that have
|
|
131
|
+
// no later deployed/rejected/rolled_back for same proposal_id
|
|
132
|
+
const terminalActions = new Set(["deployed", "rejected", "rolled_back"]);
|
|
133
|
+
const proposalIds = [...new Set(auditEntries.map((e) => e.proposal_id))];
|
|
134
|
+
const pendingProposals = proposalIds.filter((pid) => {
|
|
135
|
+
const entries = auditEntries.filter((e) => e.proposal_id === pid);
|
|
136
|
+
const hasTerminal = entries.some((e) => terminalActions.has(e.action));
|
|
137
|
+
const hasCreatedOrValidated = entries.some(
|
|
138
|
+
(e) => e.action === "created" || e.action === "validated",
|
|
139
|
+
);
|
|
140
|
+
return hasCreatedOrValidated && !hasTerminal;
|
|
141
|
+
}).length;
|
|
142
|
+
|
|
143
|
+
// Last session timestamp
|
|
144
|
+
let lastSession: string | null = null;
|
|
145
|
+
if (telemetry.length > 0) {
|
|
146
|
+
lastSession = telemetry.reduce(
|
|
147
|
+
(latest, t) => (t.timestamp > latest ? t.timestamp : latest),
|
|
148
|
+
telemetry[0].timestamp,
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// System health from doctor result
|
|
153
|
+
const system = {
|
|
154
|
+
healthy: doctorResult.healthy,
|
|
155
|
+
pass: doctorResult.summary.pass,
|
|
156
|
+
fail: doctorResult.summary.fail,
|
|
157
|
+
warn: doctorResult.summary.warn,
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
return { skills, unmatchedQueries, pendingProposals, lastSession, system };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// ---------------------------------------------------------------------------
|
|
164
|
+
// Trend computation
|
|
165
|
+
// ---------------------------------------------------------------------------
|
|
166
|
+
|
|
167
|
+
function computeTrend(skillRecords: SkillUsageRecord[]): "up" | "down" | "stable" | "unknown" {
|
|
168
|
+
if (skillRecords.length < 2) return "unknown";
|
|
169
|
+
|
|
170
|
+
// Sort by timestamp
|
|
171
|
+
const sorted = [...skillRecords].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
|
172
|
+
const mid = Math.floor(sorted.length / 2);
|
|
173
|
+
|
|
174
|
+
const firstHalf = sorted.slice(0, mid);
|
|
175
|
+
const secondHalf = sorted.slice(mid);
|
|
176
|
+
|
|
177
|
+
const firstRate =
|
|
178
|
+
firstHalf.length > 0 ? firstHalf.filter((r) => r.triggered).length / firstHalf.length : 0;
|
|
179
|
+
const secondRate =
|
|
180
|
+
secondHalf.length > 0 ? secondHalf.filter((r) => r.triggered).length / secondHalf.length : 0;
|
|
181
|
+
|
|
182
|
+
if (secondRate > firstRate) return "up";
|
|
183
|
+
if (secondRate < firstRate) return "down";
|
|
184
|
+
return "stable";
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// ---------------------------------------------------------------------------
|
|
188
|
+
// Helper: get last deployed proposal from in-memory audit entries
|
|
189
|
+
// ---------------------------------------------------------------------------
|
|
190
|
+
|
|
191
|
+
function getLastDeployedProposalFromEntries(
|
|
192
|
+
entries: EvolutionAuditEntry[],
|
|
193
|
+
skillName: string,
|
|
194
|
+
): EvolutionAuditEntry | null {
|
|
195
|
+
const needle = skillName.toLowerCase();
|
|
196
|
+
// Use word-boundary regex to avoid substring false positives (e.g. "api" matching "rapid-api").
|
|
197
|
+
// Note: skillName originates from internal JSONL logs, not user input, so ReDoS risk is minimal.
|
|
198
|
+
const pattern = new RegExp(`\\b${needle.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "i");
|
|
199
|
+
const deployed = entries.filter((e) => e.action === "deployed" && pattern.test(e.details ?? ""));
|
|
200
|
+
return deployed.length > 0 ? deployed[deployed.length - 1] : null;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// ---------------------------------------------------------------------------
|
|
204
|
+
// formatStatus — colored terminal output
|
|
205
|
+
// ---------------------------------------------------------------------------
|
|
206
|
+
|
|
207
|
+
const TREND_SYMBOLS: Record<string, string> = {
|
|
208
|
+
up: "\u2191",
|
|
209
|
+
down: "\u2193",
|
|
210
|
+
stable: "\u2192",
|
|
211
|
+
unknown: "?",
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
export function formatStatus(result: StatusResult): string {
|
|
215
|
+
const noColor = !!process.env.NO_COLOR;
|
|
216
|
+
|
|
217
|
+
const green = noColor ? (s: string) => s : (s: string) => colorize(s, "#788c5d");
|
|
218
|
+
const red = noColor ? (s: string) => s : (s: string) => colorize(s, "#cc4444");
|
|
219
|
+
const amber = noColor ? (s: string) => s : (s: string) => colorize(s, "#c49133");
|
|
220
|
+
|
|
221
|
+
const lines: string[] = [];
|
|
222
|
+
lines.push("selftune status");
|
|
223
|
+
lines.push("\u2550".repeat(15));
|
|
224
|
+
lines.push("");
|
|
225
|
+
|
|
226
|
+
// Skills table
|
|
227
|
+
const skillCount = result.skills.length;
|
|
228
|
+
lines.push(
|
|
229
|
+
`Skills (${skillCount})${" ".repeat(36 - `Skills (${skillCount})`.length)}Recent data`,
|
|
230
|
+
);
|
|
231
|
+
lines.push(" Name Pass Rate Trend Missed Status");
|
|
232
|
+
|
|
233
|
+
for (const skill of result.skills) {
|
|
234
|
+
const name = skill.name.padEnd(16);
|
|
235
|
+
const passRate =
|
|
236
|
+
skill.passRate !== null
|
|
237
|
+
? `${Math.round(skill.passRate * 100)}%`.padEnd(11)
|
|
238
|
+
: "\u2014".padEnd(11);
|
|
239
|
+
const trend = TREND_SYMBOLS[skill.trend].padEnd(7);
|
|
240
|
+
const missed = String(skill.missedQueries).padEnd(8);
|
|
241
|
+
const statusText =
|
|
242
|
+
skill.status === "REGRESSED"
|
|
243
|
+
? red(skill.status)
|
|
244
|
+
: skill.status === "HEALTHY"
|
|
245
|
+
? green(skill.status)
|
|
246
|
+
: amber(skill.status);
|
|
247
|
+
lines.push(` ${name}${passRate}${trend}${missed}${statusText}`);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
lines.push("");
|
|
251
|
+
|
|
252
|
+
// Summary stats
|
|
253
|
+
lines.push(`Unmatched queries: ${result.unmatchedQueries}`);
|
|
254
|
+
lines.push(`Pending proposals: ${result.pendingProposals}`);
|
|
255
|
+
|
|
256
|
+
// Last session
|
|
257
|
+
if (result.lastSession) {
|
|
258
|
+
const d = new Date(result.lastSession);
|
|
259
|
+
const formatted = d.toLocaleDateString("en-US", {
|
|
260
|
+
month: "short",
|
|
261
|
+
day: "numeric",
|
|
262
|
+
});
|
|
263
|
+
const time = d.toLocaleTimeString("en-US", {
|
|
264
|
+
hour: "2-digit",
|
|
265
|
+
minute: "2-digit",
|
|
266
|
+
hour12: false,
|
|
267
|
+
});
|
|
268
|
+
lines.push(`Last session: ${formatted}, ${time}`);
|
|
269
|
+
} else {
|
|
270
|
+
lines.push("Last session: \u2014");
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// System health
|
|
274
|
+
const { pass, fail, warn, healthy } = result.system;
|
|
275
|
+
const healthLabel = healthy ? green("HEALTHY") : red("UNHEALTHY");
|
|
276
|
+
lines.push(`System: ${healthLabel} (${pass} pass, ${fail} fail, ${warn} warn)`);
|
|
277
|
+
|
|
278
|
+
return lines.join("\n");
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// ---------------------------------------------------------------------------
|
|
282
|
+
// Terminal color helper using ANSI escapes
|
|
283
|
+
// ---------------------------------------------------------------------------
|
|
284
|
+
|
|
285
|
+
function colorize(text: string, hex: string): string {
|
|
286
|
+
// Expand 3-digit hex (#rgb) to 6-digit (#rrggbb)
|
|
287
|
+
const color =
|
|
288
|
+
hex.length === 4 && hex.startsWith("#")
|
|
289
|
+
? `#${hex[1]}${hex[1]}${hex[2]}${hex[2]}${hex[3]}${hex[3]}`
|
|
290
|
+
: hex;
|
|
291
|
+
const r = Number.parseInt(color.slice(1, 3), 16);
|
|
292
|
+
const g = Number.parseInt(color.slice(3, 5), 16);
|
|
293
|
+
const b = Number.parseInt(color.slice(5, 7), 16);
|
|
294
|
+
return `\x1b[38;2;${r};${g};${b}m${text}\x1b[0m`;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// ---------------------------------------------------------------------------
|
|
298
|
+
// cliMain — reads logs, runs doctor, prints output
|
|
299
|
+
// ---------------------------------------------------------------------------
|
|
300
|
+
|
|
301
|
+
export function cliMain(): void {
|
|
302
|
+
try {
|
|
303
|
+
const telemetry = readJsonl<SessionTelemetryRecord>(TELEMETRY_LOG);
|
|
304
|
+
const skillRecords = readJsonl<SkillUsageRecord>(SKILL_LOG);
|
|
305
|
+
const queryRecords = readJsonl<QueryLogRecord>(QUERY_LOG);
|
|
306
|
+
const auditEntries = readJsonl<EvolutionAuditEntry>(EVOLUTION_AUDIT_LOG);
|
|
307
|
+
const doctorResult = doctor();
|
|
308
|
+
|
|
309
|
+
const result = computeStatus(telemetry, skillRecords, queryRecords, auditEntries, doctorResult);
|
|
310
|
+
const output = formatStatus(result);
|
|
311
|
+
console.log(output);
|
|
312
|
+
process.exit(0);
|
|
313
|
+
} catch (err) {
|
|
314
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
315
|
+
console.error(`selftune status failed: ${message}`);
|
|
316
|
+
process.exit(1);
|
|
317
|
+
}
|
|
318
|
+
}
|
package/cli/selftune/types.ts
CHANGED